Merge pull request #2959 from easybuilders/4.8.x

release EasyBuild v4.8.0

.github/workflows/unit_tests.yml

@@ -13,12 +13,12 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
+        python: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', '3.11']
         modules_tool: [Lmod-6.6.3, Lmod-7.8.22, Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
         module_syntax: [Lua, Tcl]
         # exclude some configuration for non-Lmod modules tool:
         # - don't test with Lua module syntax (only supported in Lmod)
-        # - don't test with Python 3.5 and 3.7+ (only with 2.7 and 3.6), to limit test configurations
+        # - don't test with Python 3.5 and 3.7+ (only with 3.6), to limit test configurations
         exclude:
           - modules_tool: modules-tcl-1.147
             module_syntax: Lua
@@ -36,6 +36,8 @@ jobs:
             python: 3.9
           - modules_tool: modules-tcl-1.147
             python: '3.10'
+          - modules_tool: modules-tcl-1.147
+            python: '3.11'
           - modules_tool: modules-3.2.10
             python: 3.5
           - modules_tool: modules-3.2.10
@@ -46,6 +48,8 @@ jobs:
             python: 3.9
           - modules_tool: modules-3.2.10
             python: '3.10'
+          - modules_tool: modules-3.2.10
+            python: '3.11'
           - modules_tool: modules-4.1.4
             python: 3.5
           - modules_tool: modules-4.1.4
@@ -56,6 +60,8 @@ jobs:
             python: 3.9
           - modules_tool: modules-4.1.4
             python: '3.10'
+          - modules_tool: modules-4.1.4
+            python: '3.11'
       fail-fast: false
     steps:
     - uses: actions/checkout@v2

RELEASE_NOTES

@@ -3,7 +3,27 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
-The latest version of easybuild-easyblocks provides 248 software-specific easyblocks and 41 generic easyblocks.
+The latest version of easybuild-easyblocks provides 249 software-specific easyblocks and 42 generic easyblocks.
+
+
+v4.8.0 (7 July 2023)
+--------------------
+
+feature release
+
+- add `PerlBundle` generic easyblock for installing a bundle of Perl modules (#2945)
+- add custom easyblock for Rust, which makes sure that all shared libraries use RPATH rather than RUNPATH if --rpath is used (#2952)
+- minor enhancements and updates, including:
+  - add support for NVIDIA Hopper CC 9.0 in LAMMPS (#2941)
+  - add support for `install_cmds` in `Binary` easyblock (#2953)
+- various bug fixes, including:
+  - patch CMake's UnixPaths.cmake script if --sysroot is set (#2248)
+  - fix incorrect sanity_check_step for torchvision (#2938)
+  - update OpenBLAS easyblock since make shared is necessary and sufficient with OpenBLAS 0.3.23 + recent parallel build fixes (#2944)
+  - improve handling of `optarch` in `Cargo` easyblock (#2947)
+  - reset modules loaded by PythonPackage to let ExtensionEasyBlock handle `multi_deps` correctly (#2951)
+- other changes:
+  - stop running tests with Python 2.7 since it is no longer supported in GitHub Actions (#2943)
 
 
 v4.7.2 (27 May 2023)
@@ -23,6 +43,7 @@ v4.7.2 (27 May 2023)
   - force building torchvision with CUDA support if CUDA is included as dependency by setting `$FORCE_CUDA` (#2931)
   - fix exec permission on files in arch bindir for COMSOL (#2932)
 
+
 v4.7.1 (March 20th 2023)
 ------------------------
 

easybuild/easyblocks/__init__.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.7.2')
+VERSION = LooseVersion('4.8.0')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/c/cmake.py

@@ -29,7 +29,9 @@
 
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
 from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.config import build_option
+from easybuild.tools.filetools import apply_regex_substitutions
 from easybuild.tools.modules import get_software_root, get_software_libdir
 import easybuild.tools.environment as env
 from easybuild.tools.filetools import symlink
@@ -48,6 +50,39 @@ def extra_options():
         })
         return extra_vars
 
+    def patch_step(self):
+        """
+        Apply patch files, and do runtime patching (if needed).
+
+        Tweak UnixPaths.cmake if EasyBuild is configured with --sysroot.
+        """
+        super(EB_CMake, self).patch_step()
+
+        sysroot = build_option('sysroot')
+        if sysroot:
+            # prepend custom sysroot to all hardcoded paths like /lib and /usr
+            # see also patch applied by Gentoo:
+            # https://gitweb.gentoo.org/repo/gentoo.git/tree/dev-util/cmake/files/cmake-3.14.0_rc3-prefix-dirs.patch
+            srcdir = os.path.join(self.builddir, 'cmake-%s' % self.version)
+            unixpaths_cmake = os.path.join(srcdir, 'Modules', 'Platform', 'UnixPaths.cmake')
+            self.log.info("Patching %s to take into account --sysroot=%s", unixpaths_cmake, sysroot)
+
+            sysroot_lib_dirs = [os.path.join(sysroot, x) for x in ['/usr/lib64', '/usr/lib', '/lib64', '/lib']]
+            if os.path.exists(unixpaths_cmake):
+                regex_subs = [
+                    # add <sysroot>/usr/lib* and <sysroot>/lib* paths to CMAKE_SYSTEM_LIBRARY_PATH
+                    (r'(APPEND CMAKE_SYSTEM_LIBRARY_PATH)', r'\1 %s' % ' '.join(sysroot_lib_dirs)),
+                    # replace all hardcoded paths like /usr with <sysroot>/usr
+                    (r' /([a-z])', r' %s/\1' % sysroot),
+                    # also replace hardcoded '/' (root path)
+                    (r' / ', r' %s ' % sysroot),
+                    # also replace just '/' appearing at the end of a line
+                    (r' /$', r' %s' % sysroot),
+                ]
+                apply_regex_substitutions(unixpaths_cmake, regex_subs)
+            else:
+                raise EasyBuildError("File to patch %s not found", unixpaths_cmake)
+
     def configure_step(self):
         """
         Run qmake on the GUI, if necessary
@@ -91,13 +126,6 @@ def configure_step(self):
             if dep_name_upper in available_system_options and '-system-' + dep_name.lower() not in configure_opts:
                 add_cmake_opts['CMAKE_USE_SYSTEM_' + dep_name_upper] = 'ON'
 
-        sysroot = build_option('sysroot')
-        if sysroot:
-            self.log.info("Found sysroot '%s', adding it to $CMAKE_PREFIX_PATH and $CMAKE_LIBRARY_PATH", sysroot)
-            cmake_prefix_path.append(sysroot)
-            cmake_library_path.append(os.path.join(sysroot, 'usr', 'lib'))
-            cmake_include_path.append(os.path.join(sysroot, 'usr', 'include'))
-
         cmake_path_env_vars = {
             'CMAKE_PREFIX_PATH': cmake_prefix_path,
             'CMAKE_LIBRARY_PATH': cmake_library_path,

easybuild/easyblocks/generic/binary.py

@@ -59,6 +59,7 @@ def extra_options(extra_vars=None):
         extra_vars.update({
             'extract_sources': [False, "Whether or not to extract sources", CUSTOM],
             'install_cmd': [None, "Install command to be used.", CUSTOM],
+            'install_cmds': [None, "List of install commands to be used.", CUSTOM],
             # staged installation can help with the hard (potentially faulty) check on available disk space
             'staged_install': [False, "Perform staged installation via subdirectory of build directory", CUSTOM],
             'prepend_to_path': [PREPEND_TO_PATH_DEFAULT, "Prepend the given directories (relative to install-dir) to "
@@ -104,17 +105,31 @@ def build_step(self):
     def install_step(self):
         """Copy all files in build directory to the install directory"""
         install_cmd = self.cfg.get('install_cmd', None)
-        if install_cmd is None:
+        install_cmds = self.cfg.get('install_cmds', [])
+
+        if install_cmd is None and install_cmds is None:
             try:
                 # shutil.copytree doesn't allow the target directory to exist already
                 remove_dir(self.installdir)
                 shutil.copytree(self.cfg['start_dir'], self.installdir, symlinks=self.cfg['keepsymlinks'])
             except OSError as err:
                 raise EasyBuildError("Failed to copy %s to %s: %s", self.cfg['start_dir'], self.installdir, err)
         else:
-            cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']])
-            self.log.info("Installing %s using command '%s'..." % (self.name, cmd))
-            run_cmd(cmd, log_all=True, simple=True)
+            if install_cmd:
+                if not install_cmds:
+                    install_cmds = [install_cmd]
+                    install_cmd = None
+                else:
+                    raise EasyBuildError("Don't use both install_cmds and install_cmd, pick one!")
+
+            if isinstance(install_cmds, (list, tuple)):
+                for install_cmd in install_cmds:
+                    cmd = ' '.join([self.cfg['preinstallopts'], install_cmd, self.cfg['installopts']])
+                    self.log.info("Running install command for %s: '%s'..." % (self.name, cmd))
+                    run_cmd(cmd, log_all=True, simple=True)
+            else:
+                raise EasyBuildError("Incorrect value type for install_cmds, should be list or tuple: ",
+                                     install_cmds)
 
     def post_install_step(self):
         """Copy installation to actual installation directory in case of a staged installation."""

easybuild/easyblocks/generic/cargo.py

@@ -31,13 +31,15 @@
 import os
 
 import easybuild.tools.environment as env
+import easybuild.tools.systemtools as systemtools
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
 from easybuild.tools.filetools import extract_file, change_dir
 from easybuild.tools.run import run_cmd
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import write_file, compute_checksum
+from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC
 
 CRATESIO_SOURCE = "https://crates.io/api/v1/crates"
 
@@ -58,6 +60,35 @@ def extra_options(extra_vars=None):
 
         return extra_vars
 
+    def rustc_optarch(self):
+        """Determines what architecture to target.
+        Translates GENERIC optarch, and respects rustc specific optarch.
+        General optarchs are ignored as there is no direct translation.
+        """
+        if systemtools.X86_64 == systemtools.get_cpu_architecture():
+            generic = '-C target-cpu=x86-64'
+        else:
+            generic = '-C target-cpu=generic'
+
+        optimal = '-C target-cpu=native'
+
+        optarch = build_option('optarch')
+        if optarch:
+            if type(optarch) == dict:
+                if 'rustc' in optarch:
+                    rust_optarch = optarch['rustc']
+                    if rust_optarch == OPTARCH_GENERIC:
+                        return generic
+                    else:
+                        return '-' + rust_optarch
+                self.log.info("no rustc information in the optarch dict, so using %s" % optimal)
+            else:
+                if optarch == OPTARCH_GENERIC:
+                    return generic
+                else:
+                    self.log.warning("optarch is ignored as there is no translation for rustc, so using %s" % optimal)
+        return optimal
+
     def __init__(self, *args, **kwargs):
         """Constructor for Cargo easyblock."""
         super(Cargo, self).__init__(*args, **kwargs)
@@ -66,10 +97,7 @@ def __init__(self, *args, **kwargs):
         env.setvar('RUSTC', 'rustc')
         env.setvar('RUSTDOC', 'rustdoc')
         env.setvar('RUSTFMT', 'rustfmt')
-        optarch = build_option('optarch')
-        if not optarch:
-            optarch = 'native'
-        env.setvar('RUSTFLAGS', '-C target-cpu=%s' % optarch)
+        env.setvar('RUSTFLAGS', self.rustc_optarch())
         env.setvar('RUST_LOG', 'DEBUG')
         env.setvar('RUST_BACKTRACE', '1')
 

easybuild/easyblocks/generic/perlbundle.py

@@ -0,0 +1,94 @@
+##
+# Copyright 2018-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBuild support for installing a bundle of Perl modules, implemented as a generic easyblock
+
+@author: Mikael Öhman (Chalmers University of Technology)
+"""
+import os
+
+from easybuild.easyblocks.generic.bundle import Bundle
+from easybuild.easyblocks.generic.perlmodule import PerlModule
+from easybuild.easyblocks.perl import get_major_perl_version, get_site_suffix
+from easybuild.tools.config import build_option
+from easybuild.tools.environment import setvar
+
+
+class PerlBundle(Bundle):
+    """
+    Bundle of perl modules
+    """
+
+    @staticmethod
+    def extra_options():
+        """Easyconfig parameters specific to bundles of Perl modules."""
+        # combine custom easyconfig parameters of Bundle & PerlModule
+        extra_vars = PerlModule.extra_options()
+        return Bundle.extra_options(extra_vars)
+
+    def __init__(self, *args, **kwargs):
+        """Initialize PerlBundle easyblock."""
+        super(PerlBundle, self).__init__(*args, **kwargs)
+
+        self.cfg['exts_defaultclass'] = 'PerlModule'
+        self.cfg['exts_filter'] = ("perl -e 'require %(ext_name)s'", '')
+
+    def extensions_step(self, *args, **kwargs):
+        """Install extensions"""
+
+        setvar('INSTALLDIRS', 'site')
+        # define $OPENSSL_PREFIX to ensure that e.g. Net-SSLeay extension picks up OpenSSL
+        # from specified sysroot rather than from host OS
+        sysroot = build_option('sysroot')
+        if sysroot:
+            setvar('OPENSSL_PREFIX', sysroot)
+
+        super(PerlBundle, self).extensions_step(*args, **kwargs)
+
+    def test_step(self):
+        """No global test step for bundle of Perl modules."""
+        # required since runtest is set to True by default
+        pass
+
+    def sanity_check_step(self, *args, **kwargs):
+        """Custom sanity check for bundle of Perl modules."""
+
+        if not self.cfg['sanity_check_paths']:
+            majver = get_major_perl_version()
+            self.cfg['sanity_check_paths'] = {
+                'files': [],
+                'dirs': [os.path.join('lib', 'perl%s' % majver)],
+            }
+
+        super(Bundle, self).sanity_check_step(*args, **kwargs)
+
+    def make_module_extra(self):
+        """Extra module entries for Perl bundles."""
+        majver = get_major_perl_version()
+        sitelibsuffix = get_site_suffix('sitelib')
+
+        txt = super(Bundle, self).make_module_extra()
+        txt += self.module_generator.prepend_paths("PERL%sLIB" % majver, [sitelibsuffix])
+        return txt

easybuild/easyblocks/generic/pythonpackage.py

@@ -923,6 +923,12 @@ def sanity_check_step(self, *args, **kwargs):
             else:
                 raise EasyBuildError("Failed to determine pip version!")
 
+        # ExtensionEasyBlock handles loading modules correctly for multi_deps, so we clean up fake_mod_data
+        # and let ExtensionEasyBlock do its job
+        if 'Python' in self.cfg["multi_deps"] and self.fake_mod_data:
+            self.clean_up_fake_module(self.fake_mod_data)
+            self.sanity_check_module_loaded = False
+
         parent_success, parent_fail_msg = super(PythonPackage, self).sanity_check_step(*args, **kwargs)
 
         if parent_fail_msg:

easybuild/easyblocks/l/lammps.py

@@ -86,6 +86,7 @@
     'TURING75',  # NVIDIA Turing generation CC 7.5 GPU
     'AMPERE80',  # NVIDIA Ampere generation CC 8.0 GPU
     'AMPERE86',  # NVIDIA Ampere generation CC 8.6 GPU
+    'HOPPER90',  # NVIDIA Hopper generation CC 9.0 GPU
     'VEGA900',  # AMD GPU MI25 GFX900
     'VEGA906',  # AMD GPU MI50/MI60 GFX906
     'VEGA908',  # AMD GPU MI100 GFX908
@@ -135,6 +136,7 @@
     '7.5': 'TURING75',  # NVIDIA Turing generation CC 7.5
     '8.0': 'AMPERE80',  # NVIDIA Ampere generation CC 8.0
     '8.6': 'AMPERE86',  # NVIDIA Ampere generation CC 8.6
+    '9.0': 'HOPPER90',  # NVIDIA Hopper generation CC 9.0
 }
 
 # lammps version, which caused the most changes. This may not be precise, but it does work with existing easyconfigs