diff --git a/RELEASE_NOTES b/RELEASE_NOTES
index 96346e430c..f117aa07a1 100644
--- a/RELEASE_NOTES
+++ b/RELEASE_NOTES
@@ -3,7 +3,25 @@ For more detailed information, please see the git log.
These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
-The latest version of easybuild-easyblocks provides 248 software-specific easyblocks and 39 generic easyblocks.
+The latest version of easybuild-easyblocks provides 248 software-specific easyblocks and 41 generic easyblocks.
+
+
+v4.7.2 (27 May 2023)
+--------------------
+
+- new generic easyblock for installing Rust crates with cargo: Cargo and CargoPythonPackage (#2902, #2934)
+- minor enhancements and updates, including:
+ - let MATLAB easyblock raise an error if the MATLAB installation key is not provided (#2905)
+ - print message to inform that GPU package (instead of Kokkos) is used for LAMMPS (#2906)
+ - enhance PyTorch easyblock to use FlexiBLAS for PyTorch >= 1.11.0 (#2915)
+- various bug fixes, including:
+ - use custom RPATH sanity check for Go packages that doesn't actually check for an RPATH section in the binary (#2913)
+ - use string '0' to avoid problems when openssl version is not determined (#2914)
+ - update GCC easyblock to ensure that --sysroot is passed to linker (but only when it needs to be) (#2921)
+ - add output log to MATLAB installs, actually parse for common errors (#2924)
+ - enhance Gurobi easyblock to allow using $EB_GUROBI_LICENSE_FILE environment variable (#2926)
+ - force building torchvision with CUDA support if CUDA is included as dependency by setting `$FORCE_CUDA` (#2931)
+ - fix exec permission on files in arch bindir for COMSOL (#2932)
v4.7.1 (March 20th 2023)
------------------------
diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py
index d634d41d22..f740a2523e 100644
--- a/easybuild/easyblocks/__init__.py
+++ b/easybuild/easyblocks/__init__.py
@@ -43,7 +43,7 @@
# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
# This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.7.1')
+VERSION = LooseVersion('4.7.2')
UNKNOWN = 'UNKNOWN'
diff --git a/easybuild/easyblocks/c/comsol.py b/easybuild/easyblocks/c/comsol.py
index 09b6e44fcd..38d778a623 100644
--- a/easybuild/easyblocks/c/comsol.py
+++ b/easybuild/easyblocks/c/comsol.py
@@ -114,6 +114,11 @@ def install_step(self):
# make sure setup script is executable
adjust_permissions(setup_script, stat.S_IXUSR)
+ # make sure binaries in arch bindir is executable
+ archpath = os.path.join(self.start_dir, 'bin', 'glnxa64')
+ adjust_permissions(os.path.join(archpath, 'inflate'), stat.S_IXUSR)
+ adjust_permissions(os.path.join(archpath, 'setuplauncher'), stat.S_IXUSR)
+
# make sure $DISPLAY is not defined, which may lead to (hard to trace) problems
# this is a workaround for not being able to specify --nodisplay to the install scripts
env.unset_env_vars(['DISPLAY'])
diff --git a/easybuild/easyblocks/g/gcc.py b/easybuild/easyblocks/g/gcc.py
index 9069e3ea37..7ee469763e 100644
--- a/easybuild/easyblocks/g/gcc.py
+++ b/easybuild/easyblocks/g/gcc.py
@@ -48,7 +48,7 @@
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
from easybuild.tools.filetools import apply_regex_substitutions, change_dir, copy_file, move_file, symlink
-from easybuild.tools.filetools import which, write_file
+from easybuild.tools.filetools import which, read_file, write_file
from easybuild.tools.modules import get_software_root
from easybuild.tools.run import run_cmd
from easybuild.tools.systemtools import RISCV, check_os_dependency, get_cpu_architecture, get_cpu_family
@@ -357,8 +357,19 @@ def configure_step(self):
# (see https://gcc.gnu.org/install/configure.html)
self.cfg.update('configopts', '--with-sysroot=%s' % sysroot)
- # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c
- apply_regex_substitutions(os.path.join('gcc', 'gcc.c'), [('--sysroot=%R', '')])
+ libc_so_candidates = [os.path.join(sysroot, x, 'libc.so') for x in
+ ['lib', 'lib64', os.path.join('usr', 'lib'), os.path.join('usr', 'lib64')]]
+ for libc_so in libc_so_candidates:
+ if os.path.exists(libc_so):
+ # only patch gcc.c or gcc.cc if entries in libc.so are prefixed with sysroot
+ if '\nGROUP ( ' + sysroot in read_file(libc_so):
+ gccfile = os.path.join('gcc', 'gcc.c')
+ # renamed to gcc.cc in GCC 12
+ if not os.path.exists(gccfile):
+ gccfile = os.path.join('gcc', 'gcc.cc')
+ # avoid that --sysroot is passed to linker by patching value for SYSROOT_SPEC in gcc/gcc.c*
+ apply_regex_substitutions(gccfile, [('--sysroot=%R', '')])
+ break
# prefix dynamic linkers with sysroot
# this patches lines like:
diff --git a/easybuild/easyblocks/g/gurobi.py b/easybuild/easyblocks/g/gurobi.py
index a02938ac12..49147d9a2f 100644
--- a/easybuild/easyblocks/g/gurobi.py
+++ b/easybuild/easyblocks/g/gurobi.py
@@ -55,22 +55,25 @@ def __init__(self, *args, **kwargs):
"""Easyblock constructor, define custom class variables specific to Gurobi."""
super(EB_Gurobi, self).__init__(*args, **kwargs)
- self.license_file = self.cfg['license_file']
+ # make sure license file is available
+ self.orig_license_file = self.cfg['license_file']
+ if self.orig_license_file is None:
+ self.orig_license_file = os.getenv('EB_GUROBI_LICENSE_FILE', None)
if self.cfg['copy_license_file']:
self.license_file = os.path.join(self.installdir, 'gurobi.lic')
+ else:
+ self.license_file = self.orig_license_file
def install_step(self):
"""Install Gurobi and license file."""
-
- # make sure license file is available
- if self.cfg['license_file'] is None or not os.path.exists(self.cfg['license_file']):
- raise EasyBuildError("No existing license file specified: %s", self.cfg['license_file'])
-
super(EB_Gurobi, self).install_step()
if self.cfg['copy_license_file']:
- copy_file(self.cfg['license_file'], self.license_file)
+ if self.orig_license_file is None or not os.path.exists(self.orig_license_file):
+ raise EasyBuildError("No existing license file specified: %s", self.orig_license_file)
+
+ copy_file(self.orig_license_file, self.license_file)
if get_software_root('Python'):
run_cmd("python setup.py install --prefix=%s" % self.installdir)
diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py
new file mode 100644
index 0000000000..b511b851bc
--- /dev/null
+++ b/easybuild/easyblocks/generic/cargo.py
@@ -0,0 +1,251 @@
+##
+# Copyright 2009-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+EasyBuild support for installing Cargo packages (Rust lang package system)
+
+@author: Mikael Oehman (Chalmers University of Technology)
+"""
+
+import os
+
+import easybuild.tools.environment as env
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.framework.easyconfig import CUSTOM
+from easybuild.framework.extensioneasyblock import ExtensionEasyBlock
+from easybuild.tools.filetools import extract_file, change_dir
+from easybuild.tools.run import run_cmd
+from easybuild.tools.config import build_option
+from easybuild.tools.filetools import write_file, compute_checksum
+
+CRATESIO_SOURCE = "https://crates.io/api/v1/crates"
+
+
+class Cargo(ExtensionEasyBlock):
+ """Support for installing Cargo packages (Rust)"""
+
+ @staticmethod
+ def extra_options(extra_vars=None):
+ """Define extra easyconfig parameters specific to Cargo"""
+ extra_vars = ExtensionEasyBlock.extra_options(extra_vars)
+ extra_vars.update({
+ 'enable_tests': [True, "Enable building of tests", CUSTOM],
+ 'offline': [True, "Build offline", CUSTOM],
+ 'lto': [None, "Override default LTO flag ('fat', 'thin', 'off')", CUSTOM],
+ 'crates': [[], "List of (crate, version, [repo, rev]) tuples to use", CUSTOM],
+ })
+
+ return extra_vars
+
+ def __init__(self, *args, **kwargs):
+ """Constructor for Cargo easyblock."""
+ super(Cargo, self).__init__(*args, **kwargs)
+ self.cargo_home = os.path.join(self.builddir, '.cargo')
+ env.setvar('CARGO_HOME', self.cargo_home)
+ env.setvar('RUSTC', 'rustc')
+ env.setvar('RUSTDOC', 'rustdoc')
+ env.setvar('RUSTFMT', 'rustfmt')
+ optarch = build_option('optarch')
+ if not optarch:
+ optarch = 'native'
+ env.setvar('RUSTFLAGS', '-C target-cpu=%s' % optarch)
+ env.setvar('RUST_LOG', 'DEBUG')
+ env.setvar('RUST_BACKTRACE', '1')
+
+ # Populate sources from "crates" list of tuples (only once)
+ if self.cfg['crates']:
+ # copy list of crates, so we can wipe 'crates' easyconfig parameter,
+ # to avoid that creates are processed into 'sources' easyconfig parameter again
+ # when easyblock is initialized again using same parsed easyconfig
+ # (for example when check_sha256_checksums function is called, like in easyconfigs test suite)
+ self.crates = self.cfg['crates'][:]
+ sources = []
+ for crate_info in self.cfg['crates']:
+ if len(crate_info) == 2:
+ crate, version = crate_info
+ sources.append({
+ 'download_filename': crate + '/' + version + '/download',
+ 'filename': crate + '-' + version + '.tar.gz',
+ 'source_urls': [CRATESIO_SOURCE],
+ 'alt_location': 'crates.io',
+ })
+ else:
+ crate, version, repo, rev = crate_info
+ url, repo_name_git = repo.rsplit('/', maxsplit=1)
+ sources.append({
+ 'git_config': {'url': url, 'repo_name': repo_name_git[:-4], 'commit': rev},
+ 'filename': crate + '-' + version + '.tar.gz',
+ 'source_urls': [CRATESIO_SOURCE],
+ })
+
+ self.cfg.update('sources', sources)
+
+ # set 'crates' easyconfig parameter to empty list to prevent re-processing into sources
+ self.cfg['crates'] = []
+
+ def extract_step(self):
+ """
+ Unpack the source files and populate them with required .cargo-checksum.json if offline
+ """
+ if self.cfg['offline']:
+ self.log.info("Setting vendored crates dir")
+ # Replace crates-io with vendored sources using build dir wide toml file in CARGO_HOME
+ # because the rust source subdirectories might differ with python packages
+ config_toml = os.path.join(self.cargo_home, 'config.toml')
+ write_file(config_toml, '[source.vendored-sources]\ndirectory = "%s"\n\n' % self.builddir, append=True)
+ write_file(config_toml, '[source.crates-io]\nreplace-with = "vendored-sources"\n\n', append=True)
+
+ # also vendor sources from other git sources (could be many crates for one git source)
+ git_sources = set()
+ for crate_info in self.crates:
+ if len(crate_info) == 4:
+ _, _, repo, rev = crate_info
+ git_sources.add((repo, rev))
+ for repo, rev in git_sources:
+ write_file(config_toml, '[source."%s"]\ngit = "%s"\nrev = "%s"\n'
+ 'replace-with = "vendored-sources"\n\n' % (repo, repo, rev), append=True)
+
+ # Use environment variable since it would also be passed along to builds triggered via python packages
+ env.setvar('CARGO_NET_OFFLINE', 'true')
+
+ # More work is needed here for git sources to work, especially those repos with multiple packages.
+ for src in self.src:
+ existing_dirs = set(os.listdir(self.builddir))
+ self.log.info("Unpacking source %s" % src['name'])
+ srcdir = extract_file(src['path'], self.builddir, cmd=src['cmd'],
+ extra_options=self.cfg['unpack_options'], change_into_dir=False)
+ change_dir(srcdir)
+ if srcdir:
+ self.src[self.src.index(src)]['finalpath'] = srcdir
+ else:
+ raise EasyBuildError("Unpacking source %s failed", src['name'])
+
+ # Create checksum file for all sources required by vendored crates.io sources
+ new_dirs = set(os.listdir(self.builddir)) - existing_dirs
+ if self.cfg['offline'] and len(new_dirs) == 1:
+ cratedir = new_dirs.pop()
+ self.log.info('creating .cargo-checksums.json file for : %s', cratedir)
+ chksum = compute_checksum(src['path'], checksum_type='sha256')
+ chkfile = os.path.join(self.builddir, cratedir, '.cargo-checksum.json')
+ write_file(chkfile, '{"files":{},"package":"%s"}' % chksum)
+
+ def configure_step(self):
+ """Empty configuration step."""
+ pass
+
+ @property
+ def profile(self):
+ return 'debug' if self.toolchain.options.get('debug', None) else 'release'
+
+ def build_step(self):
+ """Build with cargo"""
+ parallel = ''
+ if self.cfg['parallel']:
+ parallel = "-j %s" % self.cfg['parallel']
+
+ tests = ''
+ if self.cfg['enable_tests']:
+ tests = "--tests"
+
+ lto = ''
+ if self.cfg['lto'] is not None:
+ lto = '--config profile.%s.lto=\\"%s\\"' % (self.profile, self.cfg['lto'])
+
+ run_cmd('rustc --print cfg', log_all=True, simple=True) # for tracking in log file
+ cmd = ' '.join([
+ self.cfg['prebuildopts'],
+ 'cargo build',
+ '--profile=' + self.profile,
+ lto,
+ tests,
+ parallel,
+ self.cfg['buildopts'],
+ ])
+ run_cmd(cmd, log_all=True, simple=True)
+
+ def test_step(self):
+ """Test with cargo"""
+ if self.cfg['enable_tests']:
+ cmd = ' '.join([
+ self.cfg['pretestopts'],
+ 'cargo test',
+ '--profile=' + self.profile,
+ self.cfg['testopts'],
+ ])
+ run_cmd(cmd, log_all=True, simple=True)
+
+ def install_step(self):
+ """Install with cargo"""
+ cmd = ' '.join([
+ self.cfg['preinstallopts'],
+ 'cargo install',
+ '--profile=' + self.profile,
+ '--root=' + self.installdir,
+ '--path=.',
+ self.cfg['installopts'],
+ ])
+ run_cmd(cmd, log_all=True, simple=True)
+
+
+def generate_crate_list(sourcedir):
+ """Helper for generating crate list"""
+ import toml
+
+ cargo_toml = toml.load(os.path.join(sourcedir, 'Cargo.toml'))
+ cargo_lock = toml.load(os.path.join(sourcedir, 'Cargo.lock'))
+
+ app_name = cargo_toml['package']['name']
+ deps = cargo_lock['package']
+
+ app_in_cratesio = False
+ crates = []
+ other_crates = []
+ for dep in deps:
+ name = dep['name']
+ version = dep['version']
+ if 'source' in dep:
+ if name == app_name:
+ app_in_cratesio = True # exclude app itself, needs to be first in crates list or taken from pypi
+ else:
+ if dep['source'] == 'registry+https://github.com/rust-lang/crates.io-index':
+ crates.append((name, version))
+ else:
+ # Lock file has revision#revision in the url for some reason.
+ crates.append((name, version, dep['source'].rsplit('#', maxsplit=1)[0]))
+ else:
+ other_crates.append((name, version))
+ return app_in_cratesio, crates, other_crates
+
+
+if __name__ == '__main__':
+ import sys
+ app_in_cratesio, crates, other = generate_crate_list(sys.argv[1])
+ print(other)
+ if app_in_cratesio or crates:
+ print('crates = [')
+ if app_in_cratesio:
+ print(' (name, version),')
+ for crate_info in crates:
+ print(" ('" + "', '".join(crate_info) + "'),")
+ print(']')
diff --git a/easybuild/easyblocks/generic/cargopythonpackage.py b/easybuild/easyblocks/generic/cargopythonpackage.py
new file mode 100644
index 0000000000..a935e190a0
--- /dev/null
+++ b/easybuild/easyblocks/generic/cargopythonpackage.py
@@ -0,0 +1,57 @@
+##
+# Copyright 2009-2023 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild. If not, see .
+##
+"""
+EasyBuild support for installing Cargo packages (Rust lang package system)
+
+@author: Mikael Oehman (Chalmers University of Technology)
+"""
+
+from easybuild.easyblocks.generic.cargo import Cargo
+from easybuild.easyblocks.generic.pythonpackage import PythonPackage
+
+
+class CargoPythonPackage(PythonPackage, Cargo): # PythonPackage must come first to take precedence
+ """Build a Python package with setup from Cargo but build/install step from PythonPackage
+
+ The cargo init step will set up the environment variables for rustc and vendor sources
+ but all the build steps are triggered via normal PythonPackage steps like normal.
+ """
+
+ @staticmethod
+ def extra_options(extra_vars=None):
+ """Define extra easyconfig parameters specific to Cargo"""
+ extra_vars = PythonPackage.extra_options(extra_vars)
+ extra_vars = Cargo.extra_options(extra_vars) # not all extra options here will used here
+
+ return extra_vars
+
+ def __init__(self, *args, **kwargs):
+ """Constructor for CargoPythonPackage easyblock."""
+ Cargo.__init__(self, *args, **kwargs)
+ PythonPackage.__init__(self, *args, **kwargs)
+
+ def extract_step(self):
+ """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums."""
+ return Cargo.extract_step(self)
diff --git a/easybuild/easyblocks/generic/gopackage.py b/easybuild/easyblocks/generic/gopackage.py
index c06f724437..5c2d96bdc9 100644
--- a/easybuild/easyblocks/generic/gopackage.py
+++ b/easybuild/easyblocks/generic/gopackage.py
@@ -149,3 +149,6 @@ def sanity_check_step(self):
custom_commands = ['%s --help' % self.name.lower()]
super(GoPackage, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
+
+ def sanity_check_rpath(self, rpath_dirs=None):
+ super(GoPackage, self).sanity_check_rpath(rpath_dirs=rpath_dirs, check_readelf_rpath=False)
diff --git a/easybuild/easyblocks/l/lammps.py b/easybuild/easyblocks/l/lammps.py
index 5c3d77fdb0..d026696a4d 100644
--- a/easybuild/easyblocks/l/lammps.py
+++ b/easybuild/easyblocks/l/lammps.py
@@ -346,7 +346,7 @@ def configure_step(self, **kwargs):
# https://lammps.sandia.gov/doc/Build_extras.html
# KOKKOS
if self.cfg['kokkos']:
- print_msg("Using Kokkos arch: CPU - %s, GPU - %s" % (processor_arch, gpu_arch))
+ print_msg("Using Kokkos package with arch: CPU - %s, GPU - %s" % (processor_arch, gpu_arch))
self.cfg.update('configopts', '-D%sKOKKOS=on' % self.pkg_prefix)
if self.toolchain.options.get('openmp', None):
@@ -372,6 +372,7 @@ def configure_step(self, **kwargs):
# CUDA only
elif self.cuda:
+ print_msg("Using GPU package (not Kokkos) with arch: CPU - %s, GPU - %s" % (processor_arch, gpu_arch))
self.cfg.update('configopts', '-D%sGPU=on' % self.pkg_prefix)
self.cfg.update('configopts', '-DGPU_API=cuda')
self.cfg.update('configopts', '-DGPU_ARCH=%s' % get_cuda_gpu_arch(cuda_cc))
diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py
index e1b65d77a8..b260e703de 100644
--- a/easybuild/easyblocks/m/matlab.py
+++ b/easybuild/easyblocks/m/matlab.py
@@ -55,6 +55,7 @@ def __init__(self, *args, **kwargs):
super(EB_MATLAB, self).__init__(*args, **kwargs)
self.comp_fam = None
self.configfile = os.path.join(self.builddir, 'my_installer_input.txt')
+ self.outputfile = os.path.join(self.builddir, 'my_installer_output.txt')
@staticmethod
def extra_options():
@@ -98,12 +99,14 @@ def configure_step(self):
regagree = re.compile(br"^# agreeToLicense=.*", re.M)
regmode = re.compile(br"^# mode=.*", re.M)
reglicpath = re.compile(br"^# licensePath=.*", re.M)
+ regoutfile = re.compile(br"^# outputFile=.*", re.M)
# must use byte-strings here when using Python 3, see above
config = regdest.sub(b"destinationFolder=%s" % self.installdir.encode('utf-8'), config)
config = regagree.sub(b"agreeToLicense=Yes", config)
config = regmode.sub(b"mode=silent", config)
config = reglicpath.sub(b"licensePath=%s" % licfile.encode('utf-8'), config)
+ config = regoutfile.sub(b"outputFile=%s" % self.outputfile.encode('utf-8'), config)
write_file(self.configfile, config)
@@ -159,7 +162,11 @@ def install_step(self):
keys = self.cfg['key']
if keys is None:
- keys = os.getenv('EB_MATLAB_KEY', '00000-00000-00000-00000-00000-00000-00000-00000-00000-00000')
+ try:
+ keys = os.environ['EB_MATLAB_KEY']
+ except KeyError:
+ raise EasyBuildError("The MATLAB install key is not set. This can be set either with the environment "
+ "variable EB_MATLAB_KEY or by the easyconfig variable 'key'.")
if isinstance(keys, string_type):
keys = keys.split(',')
@@ -184,6 +191,12 @@ def install_step(self):
# check installer output for known signs of trouble
patterns = [
"Error: You have entered an invalid File Installation Key",
+ "Not a valid key",
+ "All selected products are already installed",
+ "The application encountered an unexpected error and needs to close",
+ "Error: Unable to write to",
+ "Exiting with status -\\d",
+ "End - Unsuccessful",
]
for pattern in patterns:
@@ -191,6 +204,10 @@ def install_step(self):
if regex.search(out):
raise EasyBuildError("Found error pattern '%s' in output of installation command '%s': %s",
regex.pattern, cmd, out)
+ with open(self.outputfile) as f:
+ if regex.search(f.read()):
+ raise EasyBuildError("Found error pattern '%s' in output file of installer",
+ regex.pattern)
def sanity_check_step(self):
"""Custom sanity check for MATLAB."""
diff --git a/easybuild/easyblocks/o/openssl_wrapper.py b/easybuild/easyblocks/o/openssl_wrapper.py
index bd6c50341e..f73a5e5295 100644
--- a/easybuild/easyblocks/o/openssl_wrapper.py
+++ b/easybuild/easyblocks/o/openssl_wrapper.py
@@ -162,7 +162,7 @@ def __init__(self, *args, **kwargs):
for solib in solibs:
system_solib = find_library_path(solib)
if system_solib:
- openssl_version = 0
+ openssl_version = '0'
# get version of system library filename
try:
openssl_version = full_version_regex.search(os.path.realpath(system_solib)).group(0)
diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py
index 252bc09741..2783db4707 100644
--- a/easybuild/easyblocks/p/pytorch.py
+++ b/easybuild/easyblocks/p/pytorch.py
@@ -155,6 +155,9 @@ def configure_step(self):
if get_software_root('imkl'):
options.append('BLAS=MKL')
options.append('INTEL_MKL_DIR=$MKLROOT')
+ elif pytorch_version >= '1.11.0' and get_software_root('FlexiBLAS'):
+ options.append('BLAS=FlexiBLAS')
+ options.append('WITH_BLAS=flexi')
elif pytorch_version >= '1.9.0' and get_software_root('BLIS'):
options.append('BLAS=BLIS')
options.append('BLIS_HOME=' + get_software_root('BLIS'))
diff --git a/easybuild/easyblocks/t/torchvision.py b/easybuild/easyblocks/t/torchvision.py
index 197ff15c67..7ef971a8b9 100644
--- a/easybuild/easyblocks/t/torchvision.py
+++ b/easybuild/easyblocks/t/torchvision.py
@@ -26,12 +26,12 @@
EasyBuild support for building and installing torchvision, implemented as an easyblock
@author: Alexander Grund (TU Dresden)
+@author: Kenneth Hoste (HPC-UGent)
"""
-
from easybuild.easyblocks.generic.pythonpackage import PythonPackage
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.config import build_option
-from easybuild.tools.modules import get_software_root, get_software_version
+from easybuild.tools.modules import get_software_version
import easybuild.tools.environment as env
@@ -40,24 +40,60 @@ class EB_torchvision(PythonPackage):
@staticmethod
def extra_options():
- """Change some defaults."""
+ """Change some defaults for easyconfig parameters."""
extra_vars = PythonPackage.extra_options()
extra_vars['use_pip'][0] = True
extra_vars['download_dep_fail'][0] = True
extra_vars['sanity_pip_check'][0] = True
return extra_vars
+ def __init__(self, *args, **kwargs):
+ """Initialize torchvision easyblock."""
+ super(EB_torchvision, self).__init__(*args, **kwargs)
+
+ dep_names = set(dep['name'] for dep in self.cfg.dependencies())
+
+ # require that PyTorch is listed as dependency
+ if 'PyTorch' not in dep_names:
+ raise EasyBuildError('PyTorch not found as a dependency')
+
+ # enable building with GPU support if CUDA is included as dependency
+ if 'CUDA' in dep_names:
+ self.with_cuda = True
+ else:
+ self.with_cuda = False
+
def configure_step(self):
"""Set up torchvision config"""
- if not get_software_root('PyTorch'):
- raise EasyBuildError('PyTorch not found as a dependency')
# Note: Those can be overwritten by e.g. preinstallopts
env.setvar('BUILD_VERSION', self.version)
env.setvar('PYTORCH_VERSION', get_software_version('PyTorch'))
- if get_software_root('CUDA'):
+
+ if self.with_cuda:
+ # make sure that torchvision is installed with CUDA support by setting $FORCE_CUDA
+ env.setvar('FORCE_CUDA', '1')
+ # specify CUDA compute capabilities via $TORCH_CUDA_ARCH_LIST
cuda_cc = self.cfg['cuda_compute_capabilities'] or build_option('cuda_compute_capabilities')
if cuda_cc:
env.setvar('TORCH_CUDA_ARCH_LIST', ';'.join(cuda_cc))
super(EB_torchvision, self).configure_step()
+
+ def sanity_check_step(self):
+ """Custom sanity check for torchvision."""
+ custom_commands = []
+
+ # check whether torchvision was indeed built with CUDA support,
+ # cfr. https://discuss.pytorch.org/t/notimplementederror-could-not-run-torchvision-nms-with-arguments-from-\
+ # the-cuda-backend-this-could-be-because-the-operator-doesnt-exist-for-this-backend/132352/4
+ if self.with_cuda:
+ python_code = '; '.join([
+ "import torch, torchvision",
+ "boxes = torch.tensor([[0., 1., 2., 3.]]).to('cuda')",
+ "scores = torch.randn(1).to('cuda')",
+ "print(torchvision.ops.nms(boxes, scores, 0.5))",
+ ])
+ custom_commands.append('python -c "%s"' % python_code)
+
+ super(EB_torchvision, self).sanity_check_step(custom_commands=custom_commands)
diff --git a/test/easyblocks/init_easyblocks.py b/test/easyblocks/init_easyblocks.py
index 9f7a2a1a09..9a53f4b898 100644
--- a/test/easyblocks/init_easyblocks.py
+++ b/test/easyblocks/init_easyblocks.py
@@ -96,7 +96,7 @@ def tearDown(self):
self.log.error("Failed to remove %s: %s" % (self.eb_file, err))
-def template_init_test(self, easyblock, name='foo', version='1.3.2', toolchain=None):
+def template_init_test(self, easyblock, name='foo', version='1.3.2', toolchain=None, deps=None):
"""Test whether all easyblocks can be initialized."""
def check_extra_options_format(extra_options):
@@ -164,6 +164,9 @@ def check_extra_options_format(extra_options):
test_param = 'foo'
extra_txt += '%s = "%s"\n' % (key, test_param)
+ if deps:
+ extra_txt += 'dependencies = %s' % str(deps)
+
# write easyconfig file
self.write_ec(ebname, name=name, version=version, toolchain=toolchain, extratxt=extra_txt)
@@ -224,6 +227,9 @@ def innertest(self):
elif easyblock_fn == 'openssl_wrapper.py':
# easyblock to create OpenSSL wrapper expects an OpenSSL version
innertest = make_inner_test(easyblock, version='1.1')
+ elif easyblock_fn == 'torchvision.py':
+ # torchvision easyblock requires that PyTorch is listed as dependency
+ innertest = make_inner_test(easyblock, name='torchvision', deps=[('PyTorch', '1.12.1')])
else:
innertest = make_inner_test(easyblock)
diff --git a/test/easyblocks/module.py b/test/easyblocks/module.py
index 65b91f942c..db44834031 100644
--- a/test/easyblocks/module.py
+++ b/test/easyblocks/module.py
@@ -118,7 +118,7 @@ class ModuleOnlyTest(TestCase):
def writeEC(self, easyblock, name='foo', version='1.3.2', extratxt='', toolchain=None):
""" create temporary easyconfig file """
if toolchain is None:
- toolchain = {'name': 'dummy', 'version': 'dummy'}
+ toolchain = {'name': 'system', 'version': 'system'}
txt = '\n'.join([
'easyblock = "%s"',
@@ -333,7 +333,7 @@ def template_module_only_test(self, easyblock, name, version='1.3.2', extra_txt=
self.writeEC(ebname, name=name, version=version, extratxt=extra_txt, toolchain=toolchain)
# take into account that for some easyblock, particular dependencies are hard required early on
- # (in prepare_step for exampel);
+ # (in prepare_step for example);
# we just set the corresponding $EBROOT* environment variables here to fool it...
req_deps = {
# QScintilla easyblock requires that either PyQt or PyQt5 are available as dependency
@@ -434,8 +434,9 @@ def innertest(self):
for prgenv in ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi']:
write_file(os.path.join(TMPDIR, 'modules', 'all', prgenv, '1.2.3'), "#%Module")
- # add foo/1.3.2.1.1 module, required for testing ModuleAlias easyblock
- write_file(os.path.join(TMPDIR, 'modules', 'all', 'foo', '1.2.3.4.5'), "#%Module")
+ # add empty module files for dependencies that are required for testing easyblocks
+ for dep_mod_name in ('foo/1.2.3.4.5', 'PyTorch/1.12.1'):
+ write_file(os.path.join(TMPDIR, 'modules', 'all', dep_mod_name), "#%Module")
for easyblock in easyblocks:
eb_fn = os.path.basename(easyblock)
@@ -460,6 +461,10 @@ def innertest(self):
elif eb_fn == 'openssl_wrapper.py':
# easyblock to create OpenSSL wrapper expects an OpenSSL version
innertest = make_inner_test(easyblock, name='OpenSSL-wrapper', version='1.1')
+ elif eb_fn == 'torchvision.py':
+ # torchvision easyblock requires that PyTorch is listed as dependency
+ extra_txt = "dependencies = [('PyTorch', '1.12.1')]"
+ innertest = make_inner_test(easyblock, name='torchvision', extra_txt=extra_txt)
elif eb_fn == 'ucx_plugins.py':
# install fake ucx_info command (used in make_module_extra)
tmpdir = tempfile.mkdtemp()