From 446551bdd8534cc775b203ac7efa62097c13320b Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Mon, 1 Feb 2021 14:59:19 +0200 Subject: [PATCH 01/47] Use R_LIBS_SITE when installing RPackages. This makes R pick up user defined libraries before site installed ones. The order is: R_LIBS, R_LIBS_USER, R_LIBS_SITE. And the R install.packages only ever considers the first entry in the library path unless you explicitly specify a lib= to install to. --- easybuild/easyblocks/generic/rpackage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/rpackage.py b/easybuild/easyblocks/generic/rpackage.py index dadddf2b7b..fdf5e6bae6 100644 --- a/easybuild/easyblocks/generic/rpackage.py +++ b/easybuild/easyblocks/generic/rpackage.py @@ -232,7 +232,7 @@ def sanity_check_step(self, *args, **kwargs): return super(RPackage, self).sanity_check_step(EXTS_FILTER_R_PACKAGES, *args, **kwargs) def make_module_extra(self): - """Add install path to R_LIBS""" - # prepend R_LIBS with install path - extra = self.module_generator.prepend_paths("R_LIBS", [self.cfg['exts_subdir']]) + """Add install path to R_LIBS_SITE""" + # prepend R_LIBS_SITE with install path + extra = self.module_generator.prepend_paths("R_LIBS_SITE", [self.cfg['exts_subdir']]) return super(RPackage, self).make_module_extra(extra) From 38340aaa4e8cbd51520069321cd9c8fb68c96e32 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 23 Feb 2021 19:57:04 +0100 Subject: [PATCH 02/47] bump version to 4.3.4dev --- easybuild/easyblocks/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 26d3a57a8b..89173c17cd 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.3') +VERSION = LooseVersion('4.3.4.dev0') UNKNOWN = 'UNKNOWN' From 2dbc0fd03cbbf0a3708da9a44e424d2264e21b4f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 23 Feb 2021 21:05:38 +0100 Subject: [PATCH 03/47] move entry about enhanced Bazel easyblock to enhancements section in EasyBuild v4.3.3 release notes --- RELEASE_NOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 0191618ed9..ae32a08086 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -17,6 +17,7 @@ update/bugfix release - run motorBike tutorial case as sanity check for recent (community) OpenFOAM versions (#2201) - add foamMonitor to sanity checks of OpenFOAM (#2256) - create versioned symlinks for CMake commands + create symlink for 'cmake3' in PyTorch easyblock if cmake3 command is not found (#2259) + - improve Bazel easyblock: add support for running tests, enable static linking, use build dir rather than tmpdir, verbose output (#2285) - add support for skipping steps in Python packages installed as extension + print progress on individual steps for installing Python packages as extensions (#2290) - update BerkeleyGW easyblock to support GCC 10 and fftlib (#2297) - update QuantumESPRESSO easyblock to support GCC 10 (#2298) @@ -31,7 +32,6 @@ update/bugfix release - filter out user packages in LAMMPS easyblock if corresponding dependency isn't included + only set -DUSER-INTEL on x86_64 systems (#2254) - unify handling of pylibdirs and don't add duplicated $PYTHONPATH in PythonBundle (#2281) - enhance Amber easyblock to fix running of update_amber script when 'python' command is not available in OS (#2282) - - improve Bazel easyblock: add support for running tests, enable static linking, use build dir rather than tmpdir, verbose output (#2285) - guard 'module unload' statements in modules for Cray* toolchains (#2286) - set $PYTHONNOUSERSITE in PythonBundle.extensions_step to avoid picking up on Python packages installed in $HOME (#2289) - create less temporary directories for TensorFlow by (only) using --output_user_root (#2293) From 3f5b2d05f9bd785c83c54a7071e5ca3520b1b0f2 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 24 Feb 2021 10:17:21 +0100 Subject: [PATCH 04/47] add support for external libwebp and OpenEXR in opencv --- easybuild/easyblocks/o/opencv.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index a1f5b2bb50..3189e83be8 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -116,8 +116,8 @@ def configure_step(self): self.cfg.update('configopts', '-DWITH_CUDA=OFF') # configure for dependency libraries - for dep in ['JasPer', 'libjpeg-turbo', 'libpng', 'LibTIFF', 'zlib']: - if dep in ['libpng', 'LibTIFF']: + for dep in ['JasPer', 'libjpeg-turbo', 'libpng', 'LibTIFF', 'libwebp', 'OpenEXR', 'zlib']: + if dep in ['libpng', 'LibTIFF', 'libwebp']: # strip off 'lib' opt_name = dep[3:].upper() elif dep == 'libjpeg-turbo': @@ -128,6 +128,8 @@ def configure_step(self): shlib_ext = get_shared_lib_ext() if dep == 'zlib': lib_file = 'libz.%s' % shlib_ext + elif dep == 'OpenEXR': + lib_file = 'libIex.%s' % shlib_ext else: lib_file = 'lib%s.%s' % (opt_name.lower(), shlib_ext) From 24e759c0d3d54bd2ac8975a46c7f320750103a65 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 24 Feb 2021 10:22:47 +0100 Subject: [PATCH 05/47] disable bundled protobuf if it is a dependency in opencv --- easybuild/easyblocks/o/opencv.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 3189e83be8..024505d85a 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -115,6 +115,13 @@ def configure_step(self): else: self.cfg.update('configopts', '-DWITH_CUDA=OFF') + # disable bundled protobuf if it is a dependency + if 'BUILD_PROTOBUF' not in self.cfg['configopts']: + if get_software_root('protobuf'): + self.cfg.update('configopts', '-DBUILD_PROTOBUF=OFF') + else: + self.cfg.update('configopts', '-DBUILD_PROTOBUF=ON') + # configure for dependency libraries for dep in ['JasPer', 'libjpeg-turbo', 'libpng', 'LibTIFF', 'libwebp', 'OpenEXR', 'zlib']: if dep in ['libpng', 'LibTIFF', 'libwebp']: From 4d02973e325d9ae008676c73df2e0d4ec7d2a88e Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 24 Feb 2021 11:28:04 +0100 Subject: [PATCH 06/47] use OPENEXR_ROOT to set external OpenEXR in opencv --- easybuild/easyblocks/o/opencv.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 024505d85a..9b1db4b40f 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -135,16 +135,19 @@ def configure_step(self): shlib_ext = get_shared_lib_ext() if dep == 'zlib': lib_file = 'libz.%s' % shlib_ext - elif dep == 'OpenEXR': - lib_file = 'libIex.%s' % shlib_ext else: lib_file = 'lib%s.%s' % (opt_name.lower(), shlib_ext) dep_root = get_software_root(dep) if dep_root: - self.cfg.update('configopts', '-D%s_INCLUDE_DIR=%s' % (opt_name, os.path.join(dep_root, 'include'))) - libdir = get_software_libdir(dep, only_one=True) - self.cfg.update('configopts', '-D%s_LIBRARY=%s' % (opt_name, os.path.join(dep_root, libdir, lib_file))) + if dep == 'OpenEXR': + self.cfg.update('configopts', '-D%s_ROOT=%s' % (opt_name, dep_root)) + else: + inc_path = os.path.join(dep_root, 'include') + self.cfg.update('configopts', '-D%s_INCLUDE_DIR=%s' % (opt_name, inc_path)) + libdir = get_software_libdir(dep, only_one=True) + lib_path = os.path.join(dep_root, libdir, lib_file) + self.cfg.update('configopts', '-D%s_LIBRARY=%s' % (opt_name, lib_path)) # configure optimisation for CPU architecture # see https://github.com/opencv/opencv/wiki/CPU-optimizations-build-options From 073ff0b012bc6cdb3abaa1d2c9dde1c6735968d5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 24 Feb 2021 21:28:59 +0100 Subject: [PATCH 07/47] update PETSc easyblock to take into account that ScaLAPACK installation may not have header files + fix building in parallel --- easybuild/easyblocks/p/petsc.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index 8749ade5b6..d78a166f11 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -187,14 +187,16 @@ def configure_step(self): if LooseVersion(self.version) < LooseVersion("3.5"): deps.append("BLACS") for dep in deps: - inc = os.getenv('%s_INC_DIR' % dep.upper()) libdir = os.getenv('%s_LIB_DIR' % dep.upper()) libs = os.getenv('%s_STATIC_LIBS' % dep.upper()) - if inc and libdir and libs: + if libdir and libs: with_arg = "--with-%s" % dep.lower() self.cfg.update('configopts', '%s=1' % with_arg) - self.cfg.update('configopts', '%s-include=%s' % (with_arg, inc)) self.cfg.update('configopts', '%s-lib=[%s/%s]' % (with_arg, libdir, libs)) + + inc = os.getenv('%s_INC_DIR' % dep.upper()) + if inc: + self.cfg.update('configopts', '%s-include=%s' % (with_arg, inc)) else: self.log.info("Missing inc/lib info, so not enabling %s support." % dep) @@ -296,8 +298,9 @@ def configure_step(self): run_cmd(cmd, log_all=True, simple=True) # PETSc > 3.5, make does not accept -j + # to control parallel build, we need to specify MAKE_NP=... as argument to 'make' command if LooseVersion(self.version) >= LooseVersion("3.5"): - env.setvar('MAKE_NP', str(self.cfg['parallel'])) + self.cfg.update('buildopts', "MAKE_NP=%s" % self.cfg['parallel']) self.cfg['parallel'] = None # default make should be fine From f8dcb891df01982df23552ee8c0b8389533ba058 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 26 Feb 2021 20:28:28 +0100 Subject: [PATCH 08/47] stop using -nosvn option for CP2K regtest (removed in CP2K 8.1) --- easybuild/easyblocks/c/cp2k.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index e208459914..5d93714cd0 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -701,13 +701,20 @@ def test_step(self): break # location of do_regtest script - cfg_fn = "cp2k_regtest.cfg" + cfg_fn = 'cp2k_regtest.cfg' + regtest_script = os.path.join(self.cfg['start_dir'], 'tools', 'regtesting', 'do_regtest') - regtest_cmd = "%s -nosvn -nobuild -config %s" % (regtest_script, cfg_fn) + regtest_cmd = [regtest_script, '-nobuild', '-config', cfg_fn] + if LooseVersion(self.version) < LooseVersion('8.0'): + # -nosvn option was removed in CP2K 8.1 + regtest_cmd.insert(1, '-nosvn') + # older version of CP2K if not os.path.exists(regtest_script): regtest_script = os.path.join(self.cfg['start_dir'], 'tools', 'do_regtest') - regtest_cmd = "%s -nocvs -quick -nocompile -config %s" % (regtest_script, cfg_fn) + regtest_cmd = [regtest_script, '-nocvs', '-quick', '-nocompile', '-config', cfg_fn] + + regtest_cmd = ' '.join(regtest_cmd) # patch do_regtest so that reference output is used if regtest_refdir: From e0a4ee76a136b44d3614c65cb35a6dafa5dedeb1 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 26 Feb 2021 21:11:50 +0100 Subject: [PATCH 09/47] there are no more 'new' tests in CP2K 8.1 and newer --- easybuild/easyblocks/c/cp2k.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index 5d93714cd0..6d1d884929 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -819,9 +819,11 @@ def test_report(test_result): self.postmsg += test_report("FAILED") self.postmsg += test_report("WRONG") - # number of new tests, will be high if a non-suitable regtest reference was used - # will report error if count is positive (is that what we want?) - self.postmsg += test_report("NEW") + # there are no more 'new' tests from CP2K 8.1 onwards + if LooseVersion(self.version) < LooseVersion('8.0'): + # number of new tests, will be high if a non-suitable regtest reference was used + # will report error if count is positive (is that what we want?) + self.postmsg += test_report("NEW") # number of correct tests: just report test_report("CORRECT") From 85e5e5f925345d5587496eb22aee3ec6cf1bd5b1 Mon Sep 17 00:00:00 2001 From: Simon Branford Date: Sat, 27 Feb 2021 21:24:28 +0000 Subject: [PATCH 10/47] Disable CMake user package reposiory --- easybuild/easyblocks/generic/cmakemake.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 1e48725736..edef16934f 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -211,6 +211,9 @@ def configure_step(self, srcdir=None, builddir=None): # show what CMake is doing by default options.append('-DCMAKE_VERBOSE_MAKEFILE=ON') + # disable CMake user package repository + options.append('-DCMAKE_FIND_USE_PACKAGE_REGISTRY=FALSE') + if not self.cfg.get('allow_system_boost', False): # don't pick up on system Boost if Boost is included as dependency # - specify Boost location via -DBOOST_ROOT From 31f917bf57363b010c05e5f73d68e8761b0b2d35 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 2 Mar 2021 08:59:23 +0100 Subject: [PATCH 11/47] fix CP2K version check w.r.t. use of -nosvn option, was already removed in CP2K 7.1 --- easybuild/easyblocks/c/cp2k.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index 6d1d884929..e54d034a32 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -705,8 +705,8 @@ def test_step(self): regtest_script = os.path.join(self.cfg['start_dir'], 'tools', 'regtesting', 'do_regtest') regtest_cmd = [regtest_script, '-nobuild', '-config', cfg_fn] - if LooseVersion(self.version) < LooseVersion('8.0'): - # -nosvn option was removed in CP2K 8.1 + if LooseVersion(self.version) < LooseVersion('7.1'): + # -nosvn option was removed in CP2K 7.1 regtest_cmd.insert(1, '-nosvn') # older version of CP2K From 10e8c248873230647d46471931b9dea330f3c805 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Thu, 4 Mar 2021 11:50:05 +0200 Subject: [PATCH 12/47] Update gromacs.py for GROMACS/2021 with CUDA. --- easybuild/easyblocks/g/gromacs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/g/gromacs.py b/easybuild/easyblocks/g/gromacs.py index e9d891f3c5..49abc44fe2 100644 --- a/easybuild/easyblocks/g/gromacs.py +++ b/easybuild/easyblocks/g/gromacs.py @@ -177,7 +177,10 @@ def configure_step(self): self.log.info("skipping configure step") return - self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) + if LooseVersion(self.version) >= LooseVersion('2021'): + self.cfg.update('configopts', "-DGMX_GPU=CUDA -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) + else: + self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) else: # explicitly disable GPU support if CUDA is not available, # to avoid that GROMACS find and uses a system-wide CUDA compiler From 8f47c5d4f11356792b04bcd9ba6f86c8799b46c1 Mon Sep 17 00:00:00 2001 From: Ake Sandgren Date: Sat, 6 Mar 2021 00:16:32 +0200 Subject: [PATCH 13/47] vmd.py: Adjust call to python-config for Python >= 3.8. --- easybuild/easyblocks/v/vmd.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/v/vmd.py b/easybuild/easyblocks/v/vmd.py index 9c53ddc683..ad72ac3176 100644 --- a/easybuild/easyblocks/v/vmd.py +++ b/easybuild/easyblocks/v/vmd.py @@ -94,7 +94,8 @@ def configure_step(self): netcdflib = os.path.join(deps['netCDF'], 'lib') # Python locations - pymajver = get_software_version('Python').split('.')[0] + pyver = get_software_version('Python') + pymajver = pyver.split('.')[0] out, ec = run_cmd("python -c 'import sysconfig; print(sysconfig.get_path(\"include\"))'", simple=False) if ec: raise EasyBuildError("Failed to determine Python include path: %s", out) @@ -103,7 +104,10 @@ def configure_step(self): pylibdir = det_pylibdir() python_libdir = os.path.join(deps['Python'], os.path.dirname(pylibdir)) env.setvar('PYTHON_LIBRARY_DIR', python_libdir) - out, ec = run_cmd("python%s-config --libs" % pymajver, simple=False) + if LooseVersion(pyver) >= LooseVersion('3.8'): + out, ec = run_cmd("python%s-config --libs --embed" % pymajver, simple=False) + else: + out, ec = run_cmd("python%s-config --libs" % pymajver, simple=False) if ec: raise EasyBuildError("Failed to determine Python library name: %s", out) else: From cecb69b3a6b395cc49428a34fa74c2dd66b07479 Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Wed, 10 Mar 2021 08:17:17 +0000 Subject: [PATCH 14/47] cuDNN easyblock to allow for armsbsa (AARCH64). --- easybuild/easyblocks/c/cudnn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/cudnn.py b/easybuild/easyblocks/c/cudnn.py index 877c5060aa..b1c3c3e746 100644 --- a/easybuild/easyblocks/c/cudnn.py +++ b/easybuild/easyblocks/c/cudnn.py @@ -16,7 +16,7 @@ """ from easybuild.easyblocks.generic.tarball import Tarball from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.systemtools import POWER, X86_64, get_cpu_architecture +from easybuild.tools.systemtools import POWER, X86_64, AARCH64, get_cpu_architecture class EB_cuDNN(Tarball): @@ -29,6 +29,8 @@ def __init__(self, *args, **kwargs): cudnnarch = 'x64' elif myarch == POWER: cudnnarch = 'ppc64le' + elif myarch == AARCH64: + cudnnarch = 'aarch64sbsa' else: raise EasyBuildError("Architecture %s is not supported for cuDNN on EasyBuild", myarch) From 7ac5787bc664e474440018904d92af8e4e2b4b73 Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Thu, 11 Mar 2021 12:48:58 +0000 Subject: [PATCH 15/47] Adds support for different versions of java distribution, namelly, x64 and aarch64. --- easybuild/easyblocks/j/java.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 126573d12b..058fdb9e5e 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -36,6 +36,8 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, remove_dir from easybuild.tools.run import run_cmd +from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.systemtools import X86_64, AARCH64, get_cpu_architecture class EB_Java(PackedBinary): @@ -43,6 +45,21 @@ class EB_Java(PackedBinary): Use the PackedBinary easyblock and set some extra paths. """ + def __init__(self, *args, **kwargs): + """ Init the Java easyblock adding a new javaarch template var """ + myarch = get_cpu_architecture() + if myarch == X86_64: + javaarch = 'x64' + elif myarch == AARCH64: + javaarch = 'aarch64' + else: + raise EasyBuildError("Architecture %s is not supported for Java on EasyBuild", myarch) + + super(EB_cuDNN, self).__init__(*args, **kwargs) + + self.cfg.template_values['javaarch'] = javaarch + self.cfg.generate_template_values() + def extract_step(self): """Unpack the source""" if LooseVersion(self.version) < LooseVersion('1.7'): From 7ffc59775a659559e1a0ee8b3b4bb01222b6360b Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Thu, 11 Mar 2021 13:05:48 +0000 Subject: [PATCH 16/47] Fixes a noticed issue with wrongly named class for Java. --- easybuild/easyblocks/j/java.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 058fdb9e5e..ad74473df4 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -55,7 +55,7 @@ def __init__(self, *args, **kwargs): else: raise EasyBuildError("Architecture %s is not supported for Java on EasyBuild", myarch) - super(EB_cuDNN, self).__init__(*args, **kwargs) + super(EB_Java, self).__init__(*args, **kwargs) self.cfg.template_values['javaarch'] = javaarch self.cfg.generate_template_values() From a852150eaa7d502340a41aaf0936663a464d86a6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 11 Mar 2021 14:10:18 +0100 Subject: [PATCH 17/47] pass down compiler flags provided by EasyBuild in g2clib easyblock --- easybuild/easyblocks/g/g2clib.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/g/g2clib.py b/easybuild/easyblocks/g/g2clib.py index 2ac920194a..4754fcba4e 100644 --- a/easybuild/easyblocks/g/g2clib.py +++ b/easybuild/easyblocks/g/g2clib.py @@ -58,7 +58,12 @@ def build_step(self): raise EasyBuildError("JasPer module not loaded?") # beware: g2clib uses INC, while g2lib uses INCDIR ! - buildopts = 'CC="%s" FC="%s" INC="-I%s/include"' % (os.getenv('CC'), os.getenv('F90'), jasper) + buildopts = ' '.join([ + r'CC="%s"' % os.getenv('CC'), + r'CFLAGS="%s \$(INC) \$(DEF) -D__64BIT__"' % os.getenv('CFLAGS'), + r'FC="%s"' % os.getenv('F90'), + r'INC="-I%s"' % os.path.join(jasper, 'include'), + ]) self.cfg.update('buildopts', buildopts) super(EB_g2clib, self).build_step() From 2fbb7e981001c30baaca1653370a4b3905de3453 Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Thu, 11 Mar 2021 13:28:21 +0000 Subject: [PATCH 18/47] Replaces the header of cudnn with EasyBuild one and adds LuxProvide as one of contributors. --- easybuild/easyblocks/c/cudnn.py | 28 +++++++++++++++++++++------- easybuild/easyblocks/j/java.py | 1 + 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/c/cudnn.py b/easybuild/easyblocks/c/cudnn.py index b1c3c3e746..07b9abb1fc 100644 --- a/easybuild/easyblocks/c/cudnn.py +++ b/easybuild/easyblocks/c/cudnn.py @@ -1,18 +1,32 @@ ## -# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild +# Copyright 2012-2021 Ghent University # -# Copyright:: Copyright 2012-2019 Uni.Lu/LCSB, NTUA -# Authors:: Simon Branford -# License:: MIT/GPL -# $Id$ +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). # -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/ +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . ## """ EasyBuild support for cuDNN, implemented as an easyblock @author: Simon Branford (University of Birmingham) +@author: Robert Mijakovic (LuxProvide) """ from easybuild.easyblocks.generic.tarball import Tarball from easybuild.tools.build_log import EasyBuildError diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index ad74473df4..c43f414f46 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -27,6 +27,7 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) +@author: Robert Mijakovic (LuxProvide) """ import os From 27018c55c96e14022eb5956304a9c7b44febdead Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Thu, 11 Mar 2021 21:17:59 +0000 Subject: [PATCH 19/47] Adds AARCH64 support for CUDA easyblock, i.e., _sbsa file suffix. --- easybuild/easyblocks/c/cuda.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index ff2474c514..82c0ce9fdd 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -66,6 +66,8 @@ def __init__(self, *args, **kwargs): cudaarch = '' elif myarch == POWER: cudaarch = '_ppc64le' + elif myarch == AARCH64: + cudaarch = '_sbsa' else: raise EasyBuildError("Architecture %s is not supported for CUDA on EasyBuild", myarch) From d83fa06d9b23604d3312b94ae992d44629a47229 Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Thu, 11 Mar 2021 21:22:29 +0000 Subject: [PATCH 20/47] Adds the missing AARCH64 constant. --- easybuild/easyblocks/c/cuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index 82c0ce9fdd..7b0a9e10f6 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -32,7 +32,7 @@ from easybuild.tools.filetools import adjust_permissions, patch_perl_script_autoflush from easybuild.tools.filetools import remove_file, which, write_file from easybuild.tools.run import run_cmd, run_cmd_qa -from easybuild.tools.systemtools import POWER, X86_64, get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.systemtools import POWER, X86_64, AARCH64, get_cpu_architecture, get_shared_lib_ext # Wrapper script definition WRAPPER_TEMPLATE = """#!/bin/sh From 3a14cb16941ba8d0ceab598e165993fbd4532249 Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Thu, 11 Mar 2021 21:25:36 +0000 Subject: [PATCH 21/47] Adds the header and adds LuxProvide as one of the contributors. --- easybuild/easyblocks/c/cuda.py | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index 7b0a9e10f6..f5e69f6937 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -1,14 +1,26 @@ ## -# This file is an EasyBuild reciPY as per https://github.com/easybuilders/easybuild +# Copyright 2012-2021 Ghent University # -# Copyright:: Copyright 2012-2019 Cyprus Institute / CaSToRC, Uni.Lu, NTUA, Ghent University, -# Forschungszentrum Juelich GmbH -# Authors:: George Tsouloupas , Fotis Georgatos , Kenneth Hoste, Damian Alvarez -# License:: MIT/GPL -# $Id$ +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). # -# This work implements a part of the HPCBIOS project and is a component of the policy: -# http://hpcbios.readthedocs.org/en/latest/HPCBIOS_2012-99.html +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . ## """ EasyBuild support for CUDA, implemented as an easyblock @@ -20,6 +32,7 @@ @author: Kenneth Hoste (Ghent University) @author: Damian Alvarez (Forschungszentrum Juelich) @author: Ward Poelmans (Free University of Brussels) +@author: Robert Mijakovic (LuxProvide S.A.) """ import os import stat From 7d39c68de410fcf84dc1125dbe759f8552644136 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 12 Mar 2021 10:06:41 +0100 Subject: [PATCH 22/47] update LAPACK easyblock to keep control of compiler options for versions >= 3.9.0 --- easybuild/easyblocks/l/lapack.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/l/lapack.py b/easybuild/easyblocks/l/lapack.py index 3968201d28..f1854e6fcd 100644 --- a/easybuild/easyblocks/l/lapack.py +++ b/easybuild/easyblocks/l/lapack.py @@ -31,7 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ - +from distutils.version import LooseVersion import glob import os @@ -100,11 +100,30 @@ def configure_step(self): else: copy_file(src, dest) + # control compiler commands being used + self.cfg.update('buildopts', 'CC="%s"' % os.getenv('CC')) + # older versions use FORTRAN, newer versions use FC for Fortran compiler command + if LooseVersion(self.version) >= LooseVersion('3.9.0'): + fc_var = 'FC' + else: + fc_var = 'FORTRAN' + self.cfg.update('buildopts', '%s="%s"' % (fc_var, os.getenv('FC'))) + # set optimization flags + + self.cfg.update('buildopts', 'CFLAGS="%s -m64"' % os.getenv('CFLAGS')) + + # Fortran compiler flags are controlled via OPTS in older version, via FFLAGS in newer versions + if LooseVersion(self.version) >= LooseVersion('3.9.0'): + fflags_var = 'FFLAGS' + else: + fflags_var = 'OPTS' + self.cfg.update('buildopts', '%s="%s -m64"' % (fflags_var, os.getenv('FFLAGS'))) + fpic = '' if self.toolchain.options['pic']: fpic = '-fPIC' - self.cfg.update('buildopts', 'OPTS="$FFLAGS -m64" NOOPT="%s -m64 -O0"' % fpic) + self.cfg.update('buildopts', 'NOOPT="%s -m64 -O0"' % fpic) # prematurely exit configure when we're only testing if self.cfg['test_only']: From de34a0122a99546972eb8041ee2d478feabdfef5 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 12 Mar 2021 10:47:41 +0100 Subject: [PATCH 23/47] define FFLAGS_NOOPT rather than NOOPT for LAPACK 3.9.0+ --- easybuild/easyblocks/l/lapack.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/l/lapack.py b/easybuild/easyblocks/l/lapack.py index f1854e6fcd..0fd8ca08ee 100644 --- a/easybuild/easyblocks/l/lapack.py +++ b/easybuild/easyblocks/l/lapack.py @@ -116,14 +116,16 @@ def configure_step(self): # Fortran compiler flags are controlled via OPTS in older version, via FFLAGS in newer versions if LooseVersion(self.version) >= LooseVersion('3.9.0'): fflags_var = 'FFLAGS' + noopt_var = 'FFLAGS_NOOPT' else: fflags_var = 'OPTS' + noopt_var = 'NOOPT' self.cfg.update('buildopts', '%s="%s -m64"' % (fflags_var, os.getenv('FFLAGS'))) fpic = '' if self.toolchain.options['pic']: fpic = '-fPIC' - self.cfg.update('buildopts', 'NOOPT="%s -m64 -O0"' % fpic) + self.cfg.update('buildopts', '%s="%s -m64 -O0"' % (noopt_var, fpic)) # prematurely exit configure when we're only testing if self.cfg['test_only']: From a0f27fcc3a2d32661c429fed37b2a69860383476 Mon Sep 17 00:00:00 2001 From: Bart Oldeman Date: Fri, 12 Mar 2021 13:49:52 +0000 Subject: [PATCH 24/47] vtune: work with 2020 versions. "amplifier" was renamed to "profiler" --- easybuild/easyblocks/v/vtune.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/v/vtune.py b/easybuild/easyblocks/v/vtune.py index 003b9d77d2..660a984de9 100644 --- a/easybuild/easyblocks/v/vtune.py +++ b/easybuild/easyblocks/v/vtune.py @@ -45,10 +45,12 @@ def __init__(self, *args, **kwargs): # recent versions of VTune are installed to a subdirectory self.subdir = '' loosever = LooseVersion(self.version) - if loosever >= LooseVersion('2013_update12') and loosever < LooseVersion('2018'): - self.subdir = 'vtune_amplifier_xe' + if loosever >= LooseVersion('2020'): + self.subdir = 'vtune_profiler' elif loosever >= LooseVersion('2018'): self.subdir = 'vtune_amplifier' + elif loosever >= LooseVersion('2013_update12'): + self.subdir = 'vtune_amplifier_xe' def prepare_step(self, *args, **kwargs): """Since 2019u3 there is no license required.""" @@ -82,6 +84,9 @@ def make_module_req_guess(self): def sanity_check_step(self): """Custom sanity check paths for VTune.""" - binaries = ['amplxe-cl', 'amplxe-feedback', 'amplxe-gui', 'amplxe-runss'] + if LooseVersion(self.version) >= LooseVersion('2020'): + binaries = ['vtune', 'amplxe-feedback', 'vtune-gui', 'amplxe-runss'] + else: + binaries = ['amplxe-cl', 'amplxe-feedback', 'amplxe-gui', 'amplxe-runss'] custom_paths = self.get_custom_paths_tools(binaries) super(EB_VTune, self).sanity_check_step(custom_paths=custom_paths) From e6611a07933f0ddab33914e510dc91f17a9de7c6 Mon Sep 17 00:00:00 2001 From: James Moore <69149348+jkwmoore@users.noreply.github.com> Date: Mon, 15 Mar 2021 15:56:33 +0000 Subject: [PATCH 25/47] Adding preconfig and configopts to the configure commands. --- easybuild/easyblocks/w/wps.py | 2 +- easybuild/easyblocks/w/wrf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index 91f8626625..045051d4db 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -204,7 +204,7 @@ def configure_step(self): # fetch option number based on build type option and selected build type build_type_question = r"\s*(?P[0-9]+).\s*%s\s*\(?%s\)?\s*\n" % (build_type_option, knownbuildtypes[bt]) - cmd = "./configure" + cmd = ' '.join([ self.cfg['preconfigopts'],'./configure',self.cfg['configopts']]) qa = {} no_qa = [".*compiler is.*"] std_qa = { diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index 1f9d04142c..5637325d3e 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -186,7 +186,7 @@ def configure_step(self): build_type_question = r"\s*(?P[0-9]+).\s*%s\s*\(%s\)" % (build_type_option, bt) # run configure script - cmd = "./configure" + cmd = ' '.join([ self.cfg['preconfigopts'],'./configure',self.cfg['configopts']]) qa = { # named group in match will be used to construct answer "Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]:": "1", From 0470f9a9d1ae569c976527ba8ede9debcdc0da74 Mon Sep 17 00:00:00 2001 From: James Moore <69149348+jkwmoore@users.noreply.github.com> Date: Mon, 15 Mar 2021 16:00:22 +0000 Subject: [PATCH 26/47] Corrections for white space. --- easybuild/easyblocks/w/wps.py | 2 +- easybuild/easyblocks/w/wrf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/w/wps.py b/easybuild/easyblocks/w/wps.py index 045051d4db..1149ba4de2 100644 --- a/easybuild/easyblocks/w/wps.py +++ b/easybuild/easyblocks/w/wps.py @@ -204,7 +204,7 @@ def configure_step(self): # fetch option number based on build type option and selected build type build_type_question = r"\s*(?P[0-9]+).\s*%s\s*\(?%s\)?\s*\n" % (build_type_option, knownbuildtypes[bt]) - cmd = ' '.join([ self.cfg['preconfigopts'],'./configure',self.cfg['configopts']]) + cmd = ' '.join([self.cfg['preconfigopts'], './configure', self.cfg['configopts']]) qa = {} no_qa = [".*compiler is.*"] std_qa = { diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index 5637325d3e..3751a66c04 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -186,7 +186,7 @@ def configure_step(self): build_type_question = r"\s*(?P[0-9]+).\s*%s\s*\(%s\)" % (build_type_option, bt) # run configure script - cmd = ' '.join([ self.cfg['preconfigopts'],'./configure',self.cfg['configopts']]) + cmd = ' '.join([self.cfg['preconfigopts'], './configure', self.cfg['configopts']]) qa = { # named group in match will be used to construct answer "Compile for nesting? (1=basic, 2=preset moves, 3=vortex following) [default 1]:": "1", From 0b63ceb1aadab41252ab23a391e7bba4a08fd8f8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 16 Mar 2021 11:15:40 +0100 Subject: [PATCH 27/47] strip out -m64 compiler option for LAPACK --- easybuild/easyblocks/l/lapack.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/l/lapack.py b/easybuild/easyblocks/l/lapack.py index 0fd8ca08ee..79b534a163 100644 --- a/easybuild/easyblocks/l/lapack.py +++ b/easybuild/easyblocks/l/lapack.py @@ -111,7 +111,7 @@ def configure_step(self): # set optimization flags - self.cfg.update('buildopts', 'CFLAGS="%s -m64"' % os.getenv('CFLAGS')) + self.cfg.update('buildopts', 'CFLAGS="%s"' % os.getenv('CFLAGS')) # Fortran compiler flags are controlled via OPTS in older version, via FFLAGS in newer versions if LooseVersion(self.version) >= LooseVersion('3.9.0'): @@ -120,12 +120,12 @@ def configure_step(self): else: fflags_var = 'OPTS' noopt_var = 'NOOPT' - self.cfg.update('buildopts', '%s="%s -m64"' % (fflags_var, os.getenv('FFLAGS'))) + self.cfg.update('buildopts', '%s="%s"' % (fflags_var, os.getenv('FFLAGS'))) fpic = '' if self.toolchain.options['pic']: fpic = '-fPIC' - self.cfg.update('buildopts', '%s="%s -m64 -O0"' % (noopt_var, fpic)) + self.cfg.update('buildopts', '%s="%s -O0"' % (noopt_var, fpic)) # prematurely exit configure when we're only testing if self.cfg['test_only']: From d40aaaca4fabc533ed82587b95b1c32f7db99b3e Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Thu, 18 Mar 2021 10:21:47 +0800 Subject: [PATCH 28/47] add clangversion 12.0.0 for aocc 3.0.0 --- easybuild/easyblocks/a/aocc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index f9d36837b2..d281251fc9 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -60,6 +60,7 @@ def __init__(self, *args, **kwargs): def _aocc_guess_clang_version(self): map_aocc_to_clang_ver = { '2.3.0': '11.0.0', + '3.0.0': '12.0.0', } if self.version in map_aocc_to_clang_ver: From 24736b5724d9f00ebfd6b53ce90827516a8c8a34 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 18 Mar 2021 16:03:16 +0100 Subject: [PATCH 29/47] Set TORCH_CUDA_ARCH_LIST also for tests --- easybuild/easyblocks/p/pytorch.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index f06b26550e..32bd42e511 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -51,7 +51,7 @@ def extra_options(): extra_vars.update({ 'excluded_tests': [{}, 'Mapping of architecture strings to list of tests to be excluded', CUSTOM], 'custom_opts': [[], 'List of options for the build/install command. Can be used to change the defaults ' + - 'set by the PyTorch EasyBlock, for example ["USE_MKLDNN=0"].', CUSTOM] + 'set by the PyTorch EasyBlock, for example ["USE_MKLDNN=0"].', CUSTOM], }) extra_vars['download_dep_fail'][0] = True extra_vars['sanity_pip_check'][0] = True @@ -193,8 +193,12 @@ def configure_step(self): raise EasyBuildError('List of CUDA compute capabilities must be specified, either via ' 'cuda_compute_capabilities easyconfig parameter or via ' '--cuda-compute-capabilities') + self.log.info('Compiling with specified list of CUDA compute capabilities: %s', ', '.join(cuda_cc)) - options.append('TORCH_CUDA_ARCH_LIST="%s"' % ';'.join(cuda_cc)) + # This variable is also used at runtime (e.g. for tests) and if it is not set PyTorch will automatically + # determine the compute capability of a GPU in the system and use that which may fail tests if + # it is to new for the used nvcc + env.setvar('TORCH_CUDA_ARCH_LIST', ';'.join(cuda_cc)) else: # Disable CUDA options.append('USE_CUDA=0') From ee1777b75e55275d165c2f807277e53505eec27d Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 19 Mar 2021 12:13:30 +0100 Subject: [PATCH 30/47] Use PYPI_SOURCE as the default for source_urls of PythonPackage --- easybuild/easyblocks/generic/pythonpackage.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 678412ee41..27d34e5f36 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -42,6 +42,8 @@ from easybuild.base import fancylogger from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES from easybuild.framework.easyconfig import CUSTOM +from easybuild.framework.easyconfig.default import DEFAULT_CONFIG +from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option @@ -252,6 +254,13 @@ def extra_options(extra_vars=None): 'use_setup_py_develop': [False, "Install using '%s' (deprecated)" % SETUP_PY_DEVELOP_CMD, CUSTOM], 'zipped_egg': [False, "Install as a zipped eggs (requires use_easy_install)", CUSTOM], }) + # Use PYPI_SOURCE as the default for source_urls. + # This even works, when the name is not all lowercase, i.e. PYPI_LOWER_SOURCE doesn't seem to be required. + if 'source_urls' not in extra_vars: + src_urls = DEFAULT_CONFIG['source_urls'] + src_urls[0] = [url for name, url, _ in TEMPLATE_CONSTANTS if name == 'PYPI_SOURCE'] + extra_vars['source_urls'] = src_urls + return ExtensionEasyBlock.extra_options(extra_vars=extra_vars) def __init__(self, *args, **kwargs): From 155bfed4744109d8b02bfdec99c20e4d39135700 Mon Sep 17 00:00:00 2001 From: Robert Mijakovic Date: Fri, 19 Mar 2021 11:27:50 +0000 Subject: [PATCH 31/47] Updates easyblocks according to review. --- easybuild/easyblocks/c/cuda.py | 10 +++++----- easybuild/easyblocks/c/cudnn.py | 10 +++++----- easybuild/easyblocks/j/java.py | 18 ------------------ 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index f5e69f6937..50d4555793 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -45,7 +45,7 @@ from easybuild.tools.filetools import adjust_permissions, patch_perl_script_autoflush from easybuild.tools.filetools import remove_file, which, write_file from easybuild.tools.run import run_cmd, run_cmd_qa -from easybuild.tools.systemtools import POWER, X86_64, AARCH64, get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext # Wrapper script definition WRAPPER_TEMPLATE = """#!/bin/sh @@ -75,12 +75,12 @@ def extra_options(): def __init__(self, *args, **kwargs): """ Init the cuda easyblock adding a new cudaarch template var """ myarch = get_cpu_architecture() - if myarch == X86_64: - cudaarch = '' + if myarch == AARCH64: + cudaarch = '_sbsa' elif myarch == POWER: cudaarch = '_ppc64le' - elif myarch == AARCH64: - cudaarch = '_sbsa' + elif myarch == X86_64: + cudaarch = '' else: raise EasyBuildError("Architecture %s is not supported for CUDA on EasyBuild", myarch) diff --git a/easybuild/easyblocks/c/cudnn.py b/easybuild/easyblocks/c/cudnn.py index 07b9abb1fc..b52677a0dc 100644 --- a/easybuild/easyblocks/c/cudnn.py +++ b/easybuild/easyblocks/c/cudnn.py @@ -30,7 +30,7 @@ """ from easybuild.easyblocks.generic.tarball import Tarball from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.systemtools import POWER, X86_64, AARCH64, get_cpu_architecture +from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture class EB_cuDNN(Tarball): @@ -39,12 +39,12 @@ class EB_cuDNN(Tarball): def __init__(self, *args, **kwargs): """ Init the cuDNN easyblock adding a new cudnnarch template var """ myarch = get_cpu_architecture() - if myarch == X86_64: - cudnnarch = 'x64' + if myarch == AARCH64: + cudnnarch = 'aarch64sbsa' elif myarch == POWER: cudnnarch = 'ppc64le' - elif myarch == AARCH64: - cudnnarch = 'aarch64sbsa' + elif myarch == X86_64: + cudnnarch = 'x64' else: raise EasyBuildError("Architecture %s is not supported for cuDNN on EasyBuild", myarch) diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index c43f414f46..126573d12b 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -27,7 +27,6 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) -@author: Robert Mijakovic (LuxProvide) """ import os @@ -37,8 +36,6 @@ from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, remove_dir from easybuild.tools.run import run_cmd -from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.systemtools import X86_64, AARCH64, get_cpu_architecture class EB_Java(PackedBinary): @@ -46,21 +43,6 @@ class EB_Java(PackedBinary): Use the PackedBinary easyblock and set some extra paths. """ - def __init__(self, *args, **kwargs): - """ Init the Java easyblock adding a new javaarch template var """ - myarch = get_cpu_architecture() - if myarch == X86_64: - javaarch = 'x64' - elif myarch == AARCH64: - javaarch = 'aarch64' - else: - raise EasyBuildError("Architecture %s is not supported for Java on EasyBuild", myarch) - - super(EB_Java, self).__init__(*args, **kwargs) - - self.cfg.template_values['javaarch'] = javaarch - self.cfg.generate_template_values() - def extract_step(self): """Unpack the source""" if LooseVersion(self.version) < LooseVersion('1.7'): From 972bf5322df25e623875b4325a39fc0c2a61827b Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Mon, 22 Mar 2021 09:55:42 +0100 Subject: [PATCH 32/47] Improve comment --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 27d34e5f36..5d8339ca9b 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -255,7 +255,7 @@ def extra_options(extra_vars=None): 'zipped_egg': [False, "Install as a zipped eggs (requires use_easy_install)", CUSTOM], }) # Use PYPI_SOURCE as the default for source_urls. - # This even works, when the name is not all lowercase, i.e. PYPI_LOWER_SOURCE doesn't seem to be required. + # As PyPi ignores the casing in the path part of the URL (but not the filename) we can always use PYPI_SOURCE. if 'source_urls' not in extra_vars: src_urls = DEFAULT_CONFIG['source_urls'] src_urls[0] = [url for name, url, _ in TEMPLATE_CONSTANTS if name == 'PYPI_SOURCE'] From 2a99d99de42d64a9cb40b0d8f1f109031c9b7735 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 24 Mar 2021 15:27:32 +0100 Subject: [PATCH 33/47] Catch the case where a Python package has no version in the sanity check --- easybuild/easyblocks/generic/pythonpackage.py | 39 ++++++++++++++++++- easybuild/easyblocks/t/tensorflow.py | 21 ---------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 5d8339ca9b..a2b4dfdce0 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) """ +import json import os import re import sys @@ -49,7 +50,7 @@ from easybuild.tools.config import build_option from easybuild.tools.filetools import mkdir, remove_dir, which from easybuild.tools.modules import get_software_root -from easybuild.tools.py2vs3 import string_type +from easybuild.tools.py2vs3 import string_type, subprocess_popen_text from easybuild.tools.run import run_cmd from easybuild.tools.utilities import nub from easybuild.tools.hooks import CONFIGURE_STEP, BUILD_STEP, TEST_STEP, INSTALL_STEP @@ -408,6 +409,30 @@ def prepare_python(self): # set Python lib directories self.set_pylibdirs() + def get_installed_python_packages(self, names_only=True): + """Return list of Python packages that are installed + + When names_only is True then only the names are returned, else the full info from `pip list`. + Note that the names are reported by pip and might be different to the name that need to be used to import it + """ + # Check installed python packages but only check stdout, not stderr which might contain user facing warnings + cmd_list = [self.python_cmd, '-m', 'pip', 'list', '--isolated', '--disable-pip-version-check', + '--format', 'json'] + full_cmd = ' '.join(cmd_list) + self.log.info("Running command '%s'" % full_cmd) + proc = subprocess_popen_text(cmd_list, env=os.environ) + (stdout, stderr) = proc.communicate() + ec = proc.returncode + self.log.info("Command '%s' returned with %s: stdout: %s; stderr: %s" % (full_cmd, ec, stdout, stderr)) + if ec: + raise EasyBuildError('Failed to determine installed python packages: %s', stderr) + + pkgs = json.loads(stdout.strip()) + if names_only: + return [pkg['name'] for pkg in pkgs] + else: + return pkgs + def compose_install_command(self, prefix, extrapath=None, installopts=None): """Compose full install command.""" @@ -765,6 +790,18 @@ def sanity_check_step(self, *args, **kwargs): fake_mod_data = self.load_fake_module(purge=True) run_cmd("pip check", trace=False) + # Also check for a common issue where the package version shows up as 0.0.0 often caused + # by using setup.py as the installation method for a package which is released as a generic wheel + # named name-version-py2.py3-none-any.whl. `tox` creates those from version controlled source code + # so it will contain a version, but the raw tar.gz does not. + pkgs = self.get_installed_python_packages(names_only=False) + faulty_version = '0.0.0' + faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] + if faulty_pkg_names: + raise EasyBuildError("The following Python packages were not installed correctly and show a " + "version of '%s':\n%s\nThis may be solved by using the whl file instead " + "as the source. See e.g. the SOURCE*_WHL templates", + '\n'.join(faulty_pkg_names), faulty_version) if not self.is_extension: self.clean_up_fake_module(fake_mod_data) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index 6fbff8bdac..beb834d7ea 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -34,7 +34,6 @@ import re import stat import tempfile -import json from distutils.version import LooseVersion import easybuild.tools.environment as env @@ -50,7 +49,6 @@ from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_os_name, get_os_version -from easybuild.tools.py2vs3 import subprocess_popen_text CPU_DEVICE = 'cpu' @@ -269,25 +267,6 @@ def python_pkg_exists(self, name): self.log.debug('Existence check for %s returned %s with output: %s', name, ec, out) return ec == 0 - def get_installed_python_packages(self): - """Return list of Python package names that are installed - - Note that the names are reported by pip and might be different to the name that needs to be used to import it - """ - # Check installed python packages but only check stdout, not stderr which might contain user facing warnings - cmd_list = [self.python_cmd, '-m', 'pip', 'list', '--isolated', '--disable-pip-version-check', - '--format', 'json'] - full_cmd = ' '.join(cmd_list) - self.log.info("Running command '%s'" % full_cmd) - proc = subprocess_popen_text(cmd_list, env=os.environ) - (stdout, stderr) = proc.communicate() - ec = proc.returncode - self.log.info("Command '%s' returned with %s: stdout: %s; stderr: %s" % (full_cmd, ec, stdout, stderr)) - if ec: - raise EasyBuildError('Failed to determine installed python packages: %s', stderr) - - return [pkg['name'] for pkg in json.loads(stdout.strip())] - def handle_jemalloc(self): """Figure out whether jemalloc support should be enabled or not.""" if self.cfg['with_jemalloc'] is None: From 8eab6aa067e8a90800ac481d083d713ba4b3103f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 24 Mar 2021 16:01:33 +0100 Subject: [PATCH 34/47] Improve logging --- easybuild/easyblocks/generic/pythonpackage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index a2b4dfdce0..0f9c7e646f 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -423,10 +423,12 @@ def get_installed_python_packages(self, names_only=True): proc = subprocess_popen_text(cmd_list, env=os.environ) (stdout, stderr) = proc.communicate() ec = proc.returncode - self.log.info("Command '%s' returned with %s: stdout: %s; stderr: %s" % (full_cmd, ec, stdout, stderr)) + msg = "Command '%s' returned with %s: stdout: %s; stderr: %s" % (full_cmd, ec, stdout, stderr) if ec: + self.log.info(msg) raise EasyBuildError('Failed to determine installed python packages: %s', stderr) + self.log.debug(msg) pkgs = json.loads(stdout.strip()) if names_only: return [pkg['name'] for pkg in pkgs] @@ -797,6 +799,7 @@ def sanity_check_step(self, *args, **kwargs): pkgs = self.get_installed_python_packages(names_only=False) faulty_version = '0.0.0' faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] + self.log.info('Found %s invalid packages out of %s packages', len(faulty_pkg_names), len(pkgs)) if faulty_pkg_names: raise EasyBuildError("The following Python packages were not installed correctly and show a " "version of '%s':\n%s\nThis may be solved by using the whl file instead " From 08725a6325d87d73a1d1a0a71a19e051dcb7a449 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Wed, 24 Mar 2021 16:05:38 +0100 Subject: [PATCH 35/47] Handle multi-python installation sanity check --- easybuild/easyblocks/generic/pythonpackage.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 0f9c7e646f..46991e55b6 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -409,14 +409,16 @@ def prepare_python(self): # set Python lib directories self.set_pylibdirs() - def get_installed_python_packages(self, names_only=True): + def get_installed_python_packages(self, names_only=True, python_cmd=None): """Return list of Python packages that are installed When names_only is True then only the names are returned, else the full info from `pip list`. Note that the names are reported by pip and might be different to the name that need to be used to import it """ + if python_cmd is None: + python_cmd = self.python_cmd # Check installed python packages but only check stdout, not stderr which might contain user facing warnings - cmd_list = [self.python_cmd, '-m', 'pip', 'list', '--isolated', '--disable-pip-version-check', + cmd_list = [python_cmd, '-m', 'pip', 'list', '--isolated', '--disable-pip-version-check', '--format', 'json'] full_cmd = ' '.join(cmd_list) self.log.info("Running command '%s'" % full_cmd) @@ -770,11 +772,13 @@ def sanity_check_step(self, *args, **kwargs): # make sure 'exts_filter' is defined, which is used for sanity check if self.multi_python: # when installing for multiple Python versions, we must use 'python', not a full-path 'python' command! + python_cmd = 'python' if 'exts_filter' not in kwargs: kwargs.update({'exts_filter': EXTS_FILTER_PYTHON_PACKAGES}) else: # 'python' is replaced by full path to active 'python' command # (which is required especially when installing with system Python) + python_cmd = self.python_cmd if 'exts_filter' not in kwargs: orig_exts_filter = EXTS_FILTER_PYTHON_PACKAGES exts_filter = (orig_exts_filter[0].replace('python', self.python_cmd), orig_exts_filter[1]) @@ -796,7 +800,7 @@ def sanity_check_step(self, *args, **kwargs): # by using setup.py as the installation method for a package which is released as a generic wheel # named name-version-py2.py3-none-any.whl. `tox` creates those from version controlled source code # so it will contain a version, but the raw tar.gz does not. - pkgs = self.get_installed_python_packages(names_only=False) + pkgs = self.get_installed_python_packages(names_only=False, python_cmd=python_cmd) faulty_version = '0.0.0' faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] self.log.info('Found %s invalid packages out of %s packages', len(faulty_pkg_names), len(pkgs)) From 2192857f1a2ecdc730188baad704a913c92041e8 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 24 Mar 2021 19:45:17 +0100 Subject: [PATCH 36/47] enhance BWA easyblock: pass compiler flags + use filetools functions --- easybuild/easyblocks/b/bwa.py | 60 ++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/easybuild/easyblocks/b/bwa.py b/easybuild/easyblocks/b/bwa.py index 670e8bd2ab..212a6aab2a 100755 --- a/easybuild/easyblocks/b/bwa.py +++ b/easybuild/easyblocks/b/bwa.py @@ -18,13 +18,11 @@ @author: Kenneth Hoste (Ghent University) @author: George Tsouloupas """ - import os -import shutil from distutils.version import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import copy_file, mkdir class EB_BWA(ConfigureMake): @@ -35,18 +33,28 @@ class EB_BWA(ConfigureMake): def __init__(self, *args, **kwargs): """Add extra config options specific to BWA.""" super(EB_BWA, self).__init__(*args, **kwargs) - self.files = [] - def configure_step(self): - """ - Empty function as bwa comes with _no_ configure script - """ - self.files = ["bwa", "qualfa2fq.pl", "xa2multi.pl"] - if LooseVersion(self.version) < LooseVersion("0.7.0"): + self.files = ['bwa', 'qualfa2fq.pl', 'xa2multi.pl'] + if LooseVersion(self.version) < LooseVersion('0.7.0'): # solid2fastq was dropped in recent versions because the same functionality # is covered by other tools already # cfr. http://osdir.com/ml/general/2010-10/msg26205.html - self.files.append("solid2fastq.pl") + self.files.append('solid2fastq.pl') + + def configure_step(self): + """ + Empty function as BWA comes with _no_ configure script + """ + pass + + def build_step(self): + """Custom build procedure: pass down compiler command and options as arguments to 'make'.""" + + for env_var in ('CC', 'CFLAGS'): + if env_var + '=' not in self.cfg['buildopts']: + self.cfg.update('buildopts', env_var + '="$' + env_var + '"') + + super(EB_BWA, self).build_step() def install_step(self): """ @@ -54,26 +62,26 @@ def install_step(self): """ srcdir = self.cfg['start_dir'] destdir = os.path.join(self.installdir, 'bin') - mandir = os.path.join(self.installdir, 'man') - manman1dir = os.path.join(self.installdir, 'man/man1') + mkdir(destdir) + for filename in self.files: + srcfile = os.path.join(srcdir, filename) + copy_file(srcfile, destdir) + manfile = os.path.join(srcdir, 'bwa.1') - srcfile = None - try: - os.makedirs(destdir) - os.makedirs(mandir) - os.makedirs(manman1dir) - for filename in self.files: - srcfile = os.path.join(srcdir, filename) - shutil.copy2(srcfile, destdir) - shutil.copy2(manfile, manman1dir) - except OSError as err: - raise EasyBuildError("Copying %s to installation dir %s failed: %s", srcfile, destdir, err) + manman1dir = os.path.join(self.installdir, 'man', 'man1') + mkdir(manman1dir, parents=True) + copy_file(manfile, manman1dir) def sanity_check_step(self): """Custom sanity check for BWA.""" + custom_paths = { - 'files': ["bin/%s" % x for x in self.files], + 'files': [os.path.join('bin', x) for x in self.files], 'dirs': [] } - super(EB_BWA, self).sanity_check_step(custom_paths=custom_paths) + # 'bwa' command doesn't have a --help option, but it does print help-like information to stderr + # when run without arguments (and exits with exit code 1) + custom_commands = ["bwa 2>&1 | grep 'index sequences in the FASTA format'"] + + super(EB_BWA, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) From bd5c47a437947672139eecb5933853e0c0d9d5a9 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Fri, 26 Mar 2021 09:10:11 +0100 Subject: [PATCH 37/47] Fix accidental change of the default source_urls in the PythonPackage EB --- easybuild/easyblocks/generic/pythonpackage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 5d8339ca9b..624ab8b705 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -257,7 +257,8 @@ def extra_options(extra_vars=None): # Use PYPI_SOURCE as the default for source_urls. # As PyPi ignores the casing in the path part of the URL (but not the filename) we can always use PYPI_SOURCE. if 'source_urls' not in extra_vars: - src_urls = DEFAULT_CONFIG['source_urls'] + # Create a copy so the defaults are not modified by the following line + src_urls = DEFAULT_CONFIG['source_urls'][:] src_urls[0] = [url for name, url, _ in TEMPLATE_CONSTANTS if name == 'PYPI_SOURCE'] extra_vars['source_urls'] = src_urls From ab96fe33a19c7d92358a06fd9e798583ff4a107e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 26 Mar 2021 19:57:04 +0100 Subject: [PATCH 38/47] enhance Hadoop easyblock to avoid copying same native library twice --- easybuild/easyblocks/h/hadoop.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/h/hadoop.py b/easybuild/easyblocks/h/hadoop.py index 53bd019f05..d12415d173 100644 --- a/easybuild/easyblocks/h/hadoop.py +++ b/easybuild/easyblocks/h/hadoop.py @@ -30,11 +30,11 @@ import glob import os import re -import shutil from easybuild.easyblocks.generic.tarball import Tarball from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.filetools import copy_file from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import get_shared_lib_ext @@ -83,9 +83,17 @@ def post_install_step(self): lib_src = os.path.join(lib_root, lib_path) lib_dest = os.path.join(self.installdir, 'lib', 'native') self.log.info('Copying shared objects in "%s"', lib_src) + copied_libs = [] for lib in glob.glob(lib_src): - self.log.info('Copying "%s" to "%s"', lib, lib_dest) - shutil.copy2(lib, lib_dest) + lib_fn = os.path.basename(lib) + # avoid copying the same file again, which may cause problems due to read-only permissions of target + # this could happen when 'lib' is a symlink to 'lib64', which leads to duplicates in the glob result + if lib_fn in copied_libs: + self.log.info("Not copying '%s' to '%s', this file was already copied", lib, lib_dest) + else: + self.log.info("Copying '%s' to '%s'", lib, lib_dest) + copy_file(lib, lib_dest) + copied_libs.append(lib_fn) def sanity_check_step(self): """Custom sanity check for Hadoop.""" From 513d15a5a123446875930d10b57f88bebf341ee0 Mon Sep 17 00:00:00 2001 From: Samuel Moors Date: Mon, 29 Mar 2021 14:04:46 +0200 Subject: [PATCH 39/47] fix pip extension download pattern for pythonpackage easyblock --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 3530655ba9..b45d3b0a19 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -747,7 +747,7 @@ def sanity_check_step(self, *args, **kwargs): self.log.info("Detection of downloaded depenencies enabled, checking output of installation command...") patterns = [ 'Downloading .*/packages/.*', # setuptools - r'Collecting .* \(from.*', # pip + r'Collecting .*', # pip ] downloaded_deps = [] for pattern in patterns: From ddb284e318e6c7b0c5db7a8976dc3d5b7439bd5f Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 30 Mar 2021 11:34:23 +0200 Subject: [PATCH 40/47] Make the stub libs take preference over system libs on linking --- easybuild/easyblocks/c/cuda.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index ff2474c514..e93cb56594 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -29,8 +29,8 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, patch_perl_script_autoflush -from easybuild.tools.filetools import remove_file, which, write_file +from easybuild.tools.filetools import adjust_permissions, copy_dir, mkdir, patch_perl_script_autoflush +from easybuild.tools.filetools import remove_file, symlink, which, write_file from easybuild.tools.run import run_cmd, run_cmd_qa from easybuild.tools.systemtools import POWER, X86_64, get_cpu_architecture, get_shared_lib_ext @@ -179,7 +179,7 @@ def create_wrapper(wrapper_name, wrapper_comp): for comp in (self.cfg['host_compilers'] or []): create_wrapper('nvcc_%s' % comp, comp) - ldconfig = which('ldconfig') + ldconfig = which('ldconfig', log_ok=False, log_error=False) sbin_dirs = ['/sbin', '/usr/sbin'] if not ldconfig: # ldconfig is usually in /sbin or /usr/sbin @@ -195,10 +195,20 @@ def create_wrapper(wrapper_name, wrapper_comp): path = os.environ.get('PATH', '') raise EasyBuildError("Unable to find 'ldconfig' in $PATH (%s), nor in any of %s", path, sbin_dirs) + stubs_dir = os.path.join(self.installdir, 'lib64', 'stubs') # Run ldconfig to create missing symlinks in the stubs directory (libcuda.so.1, etc) - cmd = ' '.join([ldconfig, '-N', os.path.join(self.installdir, 'lib64', 'stubs')]) + cmd = ' '.join([ldconfig, '-N', stubs_dir]) run_cmd(cmd) + # GCC searches paths in LIBRARY_PATH and the system paths suffixed with ../lib64 or ../lib first + # This means stubs/../lib64 is searched before the system /lib64 folder containing a potentially older libcuda. + # See e.g. https://github.com/easybuilders/easybuild-easyconfigs/issues/12348 + # Workaround: Create a copy that matches this pattern + new_stubs_dir = os.path.join(self.installdir, 'stubs') + copy_dir(stubs_dir, os.path.join(new_stubs_dir, 'lib64')) + # Also create the lib dir as a symlink + symlink('lib64', os.path.join(new_stubs_dir, 'lib'), use_abspath_source=False) + super(EB_CUDA, self).post_install_step() def sanity_check_step(self): @@ -206,12 +216,7 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() - chk_libdir = ["lib64"] - - # Versions higher than 6 do not provide 32 bit libraries - if LooseVersion(self.version) < LooseVersion("6"): - chk_libdir += ["lib"] - + chk_libdir = ["lib64", "lib"] culibs = ["cublas", "cudart", "cufft", "curand", "cusparse"] custom_paths = { 'files': [os.path.join("bin", x) for x in ["fatbinary", "nvcc", "nvlink", "ptxas"]] + @@ -259,7 +264,7 @@ def make_module_req_guess(self): guesses.update({ 'PATH': bin_path, 'LD_LIBRARY_PATH': lib_path, - 'LIBRARY_PATH': ['lib64', os.path.join('lib64', 'stubs')], + 'LIBRARY_PATH': ['lib64', os.path.join('stubs', 'lib64')], 'CPATH': inc_path, }) From 1948df0c7f172c57a4918f2fa6e0bb8f08d617b2 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Tue, 30 Mar 2021 11:35:27 +0200 Subject: [PATCH 41/47] Remove unused import --- easybuild/easyblocks/c/cuda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index e93cb56594..4db624521d 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -29,7 +29,7 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, copy_dir, mkdir, patch_perl_script_autoflush +from easybuild.tools.filetools import adjust_permissions, copy_dir, patch_perl_script_autoflush from easybuild.tools.filetools import remove_file, symlink, which, write_file from easybuild.tools.run import run_cmd, run_cmd_qa from easybuild.tools.systemtools import POWER, X86_64, get_cpu_architecture, get_shared_lib_ext From 8a761bf5b4ec29684c878e457d4d359ae76de719 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 1 Apr 2021 09:08:11 +0200 Subject: [PATCH 42/47] Enhance and fix the error message for the 0.0.0 version --- easybuild/easyblocks/generic/pythonpackage.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 3530655ba9..476186a630 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -806,10 +806,15 @@ def sanity_check_step(self, *args, **kwargs): faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] self.log.info('Found %s invalid packages out of %s packages', len(faulty_pkg_names), len(pkgs)) if faulty_pkg_names: - raise EasyBuildError("The following Python packages were not installed correctly and show a " - "version of '%s':\n%s\nThis may be solved by using the whl file instead " - "as the source. See e.g. the SOURCE*_WHL templates", - '\n'.join(faulty_pkg_names), faulty_version) + msg = ( + "The following Python packages were likely not installed correctly because they show a " + "version of '%s':\n%s\n" + "This may be solved by using a *-none-any.whl file as the source instead. " + "See e.g. the SOURCE*_WHL templates.\n" + "Otherwise you could check if the package provides a version at all or if e.g. poetry is " + "required (check the source for a pyproject.toml and see PEP517 for details on that)." + ) % (faulty_version, '\n'.join(faulty_pkg_names)) + raise EasyBuildError(msg) if not self.is_extension: self.clean_up_fake_module(fake_mod_data) From 9eaadd01b4e1816bd8c99a14f5f82fd0fd75bc11 Mon Sep 17 00:00:00 2001 From: Alexander Grund Date: Thu, 1 Apr 2021 09:37:38 +0200 Subject: [PATCH 43/47] Add unversioned_packages EC param and report all errors together --- easybuild/easyblocks/generic/pythonpackage.py | 38 +++++++++++++++++-- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 476186a630..54d3a05e5b 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -241,9 +241,14 @@ def extra_options(extra_vars=None): 'pip_ignore_installed': [True, "Let pip ignore installed Python packages (i.e. don't remove them)", CUSTOM], 'req_py_majver': [None, "Required major Python version (only relevant when using system Python)", CUSTOM], 'req_py_minver': [None, "Required minor Python version (only relevant when using system Python)", CUSTOM], - 'sanity_pip_check': [False, "Run 'pip check' to ensure all required Python packages are installed", CUSTOM], + 'sanity_pip_check': [False, "Run 'pip check' to ensure all required Python packages are installed " + "and check for any package with an invalid (0.0.0) version.", CUSTOM], 'runtest': [True, "Run unit tests.", CUSTOM], # overrides default 'unpack_sources': [True, "Unpack sources prior to build/install", CUSTOM], + # A version of 0.0.0 is usually an error on installation unless the package does really not provide a + # version. Those would fail the (extended) sanity_pip_check. So as a last resort they can be added here + # and will be excluded from that check. Note that the display name is required, i.e. from `pip list`. + 'unversioned_packages': [[], "List of packages that don't have a version at all, i.e. show 0.0.0", CUSTOM], 'use_easy_install': [False, "Install using '%s' (deprecated)" % EASY_INSTALL_INSTALL_CMD, CUSTOM], 'use_pip': [None, "Install using '%s'" % PIP_INSTALL_CMD, CUSTOM], 'use_pip_editable': [False, "Install using 'pip install --editable'", CUSTOM], @@ -796,7 +801,14 @@ def sanity_check_step(self, *args, **kwargs): # is not "in view", and we will overlook missing dependencies... fake_mod_data = self.load_fake_module(purge=True) - run_cmd("pip check", trace=False) + pip_check_errors = [] + + pip_check_msg, ec = run_cmd("pip check", log_ok=False) + if ec: + pip_check_errors.append('`pip check` failed:\n%s' % pip_check_msg) + else: + self.log.info('`pip check` completed successfully') + # Also check for a common issue where the package version shows up as 0.0.0 often caused # by using setup.py as the installation method for a package which is released as a generic wheel # named name-version-py2.py3-none-any.whl. `tox` creates those from version controlled source code @@ -804,6 +816,23 @@ def sanity_check_step(self, *args, **kwargs): pkgs = self.get_installed_python_packages(names_only=False, python_cmd=python_cmd) faulty_version = '0.0.0' faulty_pkg_names = [pkg['name'] for pkg in pkgs if pkg['version'] == faulty_version] + + for unversioned_package in self.cfg.get('unversioned_packages', []): + try: + faulty_pkg_names.remove(unversioned_package) + self.log.debug('Excluding unversioned package %s from check', unversioned_package) + except ValueError: + try: + version = next(pkg['version'] for pkg in pkgs if pkg['name'] == unversioned_package) + except StopIteration: + msg = ('Package %s in unversioned_packages was not found in the installed packages. ' + 'Check that the name from `pip list` is used which may be different than the ' + 'module name.' % unversioned_package) + else: + msg = ('Package %s in unversioned_packages Has a version of %s which is valid. ' + 'Please remove it from unversioned_packages.' % (unversioned_package, version)) + pip_check_errors.append(msg) + self.log.info('Found %s invalid packages out of %s packages', len(faulty_pkg_names), len(pkgs)) if faulty_pkg_names: msg = ( @@ -814,10 +843,13 @@ def sanity_check_step(self, *args, **kwargs): "Otherwise you could check if the package provides a version at all or if e.g. poetry is " "required (check the source for a pyproject.toml and see PEP517 for details on that)." ) % (faulty_version, '\n'.join(faulty_pkg_names)) - raise EasyBuildError(msg) + pip_check_errors.append(msg) if not self.is_extension: self.clean_up_fake_module(fake_mod_data) + + if pip_check_errors: + raise EasyBuildError('\n'.join(pip_check_errors)) else: raise EasyBuildError("pip >= 9.0.0 is required for running 'pip check', found %s", pip_version) else: From 836473d50dde4d9ec4aeaa7dd7d1aca453cb61f9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 1 Apr 2021 17:47:06 +0200 Subject: [PATCH 44/47] fix tiny typo in error message --- easybuild/easyblocks/generic/pythonpackage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 54d3a05e5b..c4484f9e07 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -829,7 +829,7 @@ def sanity_check_step(self, *args, **kwargs): 'Check that the name from `pip list` is used which may be different than the ' 'module name.' % unversioned_package) else: - msg = ('Package %s in unversioned_packages Has a version of %s which is valid. ' + msg = ('Package %s in unversioned_packages has a version of %s which is valid. ' 'Please remove it from unversioned_packages.' % (unversioned_package, version)) pip_check_errors.append(msg) From 19ff0e358aa7a200e31e922eceaf79208b4d45fc Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 6 Apr 2021 22:13:24 +0200 Subject: [PATCH 45/47] fix comment in NVHPC/AOCC easyblocks --- easybuild/easyblocks/a/aocc.py | 2 +- easybuild/easyblocks/n/nvhpc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/easybuild/easyblocks/a/aocc.py b/easybuild/easyblocks/a/aocc.py index d281251fc9..086d038571 100644 --- a/easybuild/easyblocks/a/aocc.py +++ b/easybuild/easyblocks/a/aocc.py @@ -75,7 +75,7 @@ def _aocc_guess_clang_version(self): raise EasyBuildError('\n'.join(error_lines)) def install_step(self): - # EULA for AOCC must be accepted via --accept-eula EasyBuild configuration option, + # EULA for AOCC must be accepted via --accept-eula-for EasyBuild configuration option, # or via 'accept_eula = True' in easyconfig file self.check_accepted_eula(more_info='http://developer.amd.com/wordpress/media/files/AOCC_EULA.pdf') diff --git a/easybuild/easyblocks/n/nvhpc.py b/easybuild/easyblocks/n/nvhpc.py index e0e68c61d9..095d0d57cd 100644 --- a/easybuild/easyblocks/n/nvhpc.py +++ b/easybuild/easyblocks/n/nvhpc.py @@ -101,7 +101,7 @@ def __init__(self, *args, **kwargs): def install_step(self): """Install by running install command.""" - # EULA for NVHPC must be accepted via --accept-eula EasyBuild configuration option, + # EULA for NVHPC must be accepted via --accept-eula-for EasyBuild configuration option, # or via 'accept_eula = True' in easyconfig file self.check_accepted_eula(more_info='https://docs.nvidia.com/hpc-sdk/eula/index.html') From f0d8920d37f78eb5bb1baa085bd3d383ac7a3a0b Mon Sep 17 00:00:00 2001 From: Miguel Dias Costa Date: Wed, 7 Apr 2021 11:09:01 +0800 Subject: [PATCH 46/47] prepare release notes for EasyBuild v4.3.4 + bump version to 4.3.4 --- RELEASE_NOTES | 32 ++++++++++++++++++++++++++++++++ easybuild/easyblocks/__init__.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index ae32a08086..b6e46bdd1b 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -6,6 +6,38 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en The latest version of easybuild-easyblocks provides 229 software-specific easyblocks and 37 generic easyblocks. +v4.3.4 (April 9th 2021) +----------------------- + +update/bugfix release + +- minor enhancements, including: + - make OpenCV easyblock aware of protobuf, libwebp and OpenEXR dependencies provided via EasyBuild (#2346) + - update CP2K easyblock w.r.t. running regtest for CP2K v8.1 (#2350) + - update gromacs.py for GROMACS/2021 with CUDA (#2353) + - adjust call to python-config for Python >= 3.8 in VMD easyblock (#2355) + - enhance cuDNN and CUDA easyblocks to support aarch64 (#2356) + - pass down compiler flags provided by EasyBuild in g2clib easyblock (#2357) + - update VTune easyblock for version 2020 (#2359) + - make WRF and WPS easyblocks aware of (pre)configopts (#2361) + - add Clang version 12.0.0 for AOCC 3.0.0 to mapping in custom easyblock for AOCC (#2362) + - use PYPI_SOURCE as the default for source_urls of PythonPackage (#2364) + - enhance PythonPackage easyblock to catch faulty version (0.0.0) for installed Python packages (#2367) + - enhance BWA easyblock: pass compiler flags + use filetools functions (#2368) +- various bug fixes, including: + - set $R_LIBS_SITE rather than $R_LIBS when installing R packages (#2326) + - update PETSc easyblock to take into account that ScaLAPACK installation may not have header files + fix building in parallel (#2348) + - disable CMake user package repository (#2351) + - update LAPACK easyblock to keep control of compiler options for versions >= 3.9.0 (#2358) + - also set $TORCH_CUDA_ARCH_LIST for PyTorch tests (#2363) + - fix accidental change of the default source_urls in the PythonPackage EB (#2370) + - enhance Hadoop easyblock to avoid copying same native library twice (#2371) + - fix pip extension download pattern for pythonpackage easyblock (#2372) + - make the CUDA stub libs take preference over system libs when linking (#2373) + - improve Python package version check and add unversioned_packages easyconfig parameter (#2377) + - fix comment in NVHPC/AOCC easyblocks (#2378) + + v4.3.3 (February 23rd 2021) --------------------------- diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 89173c17cd..b74f75ca71 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.3.4.dev0') +VERSION = LooseVersion('4.3.4') UNKNOWN = 'UNKNOWN' From 669a35c4ccdb19d1485f689bc5cfc54f40eea6da Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 7 Apr 2021 11:00:59 +0200 Subject: [PATCH 47/47] tweak easyblocks release notes for EasyBuild v4.3.4 --- RELEASE_NOTES | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/RELEASE_NOTES b/RELEASE_NOTES index b6e46bdd1b..fe940815df 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -14,28 +14,26 @@ update/bugfix release - minor enhancements, including: - make OpenCV easyblock aware of protobuf, libwebp and OpenEXR dependencies provided via EasyBuild (#2346) - update CP2K easyblock w.r.t. running regtest for CP2K v8.1 (#2350) - - update gromacs.py for GROMACS/2021 with CUDA (#2353) + - update GROMACS easyblock for GROMACS/2021 with CUDA (#2353) - adjust call to python-config for Python >= 3.8 in VMD easyblock (#2355) - enhance cuDNN and CUDA easyblocks to support aarch64 (#2356) - pass down compiler flags provided by EasyBuild in g2clib easyblock (#2357) - update VTune easyblock for version 2020 (#2359) - make WRF and WPS easyblocks aware of (pre)configopts (#2361) - add Clang version 12.0.0 for AOCC 3.0.0 to mapping in custom easyblock for AOCC (#2362) - - use PYPI_SOURCE as the default for source_urls of PythonPackage (#2364) - - enhance PythonPackage easyblock to catch faulty version (0.0.0) for installed Python packages (#2367) + - use PYPI_SOURCE as the default for source_urls of PythonPackage (#2364, #2370) + - enhance PythonPackage easyblock to catch faulty version (0.0.0) for installed Python packages (#2367, #2377) - enhance BWA easyblock: pass compiler flags + use filetools functions (#2368) - various bug fixes, including: - set $R_LIBS_SITE rather than $R_LIBS when installing R packages (#2326) - update PETSc easyblock to take into account that ScaLAPACK installation may not have header files + fix building in parallel (#2348) - - disable CMake user package repository (#2351) + - disable CMake user package repository in CMakeMake generic easyblock (#2351) - update LAPACK easyblock to keep control of compiler options for versions >= 3.9.0 (#2358) - also set $TORCH_CUDA_ARCH_LIST for PyTorch tests (#2363) - - fix accidental change of the default source_urls in the PythonPackage EB (#2370) - enhance Hadoop easyblock to avoid copying same native library twice (#2371) - - fix pip extension download pattern for pythonpackage easyblock (#2372) + - fix pip extension download pattern for PythonPackage easyblock (#2372) - make the CUDA stub libs take preference over system libs when linking (#2373) - improve Python package version check and add unversioned_packages easyconfig parameter (#2377) - - fix comment in NVHPC/AOCC easyblocks (#2378) v4.3.3 (February 23rd 2021)