diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 96cd448b68..8dd90e0374 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -3,7 +3,47 @@ For more detailed information, please see the git log. These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html. -The latest version of easybuild-easyblocks provides 249 software-specific easyblocks and 42 generic easyblocks. +The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks. + + +v4.8.1 (11 September 2023) +-------------------------- + +update/bugfix release + +- new custom easyblock for sympy (#2949) and tensorflow-compression (#2990) +- minor enhancements and updates, including: + - drop unnecessary CUDA stub libraries from $LIBRARY_PATH (#2793) + - update Score-P easyblock to use `--with-nocross-compiler-suite=nvhpc` for recent software versions (#2928) + - unset `$CPPFLAGS`, `$LDFLAGS`, `$LIB` which may interfere with Score-P configure magic (#2928) + - update Clang easyblock for versions >= 16 + run tests only for final stage of bootstrap build (#2929) + - handle new directory structure for Intel Advisor (#2942) + - use `DCPU_BASELINE=DETECT` for OpenCV when default optarch compiler option is used (#2954) + - update MXNet easyblock + don't try to install R extension by default for MXNet >= 1.0 (#2955) + - use checkMCR.sh to determine if we have the correct MCR for FreeSurfer (#2962) + - add options to `MesonNinja` easyblock to customize `build_cmd`, `install_cmd`, `builddir` (#2963, #2993) + - add support for building CP2K with libvori support (#2967) + - enable system `pybind11` for PyTorch 1.10+ to make sure `pybind11` provided as dependency is used (#2968) + - update LLVM easyblock for LLVM v16: symlink `third-party` to `third-party-.src` (#2970, #2994) + - update scipy easyblock for scipy >= 1.11.0 (#2971, #2980) + - update sanity check for Mesa >= 22.3 (#2973) + - update sanity check for OpenFOAM 11 (#2978) + - add support to `PerlModule` easyblock to customize prefix option used in installation command (#2979) + - update TensorFlow easyblock for v2.13 since LMDB is no longer a dependency (#2982) + - enhance PyTorch easyblock to print individual failed tests (#2983) + - enhance PETSc easyblock to support using custom `$PETSC_ARCH` (#2987) +- various bug fixes, including: + - correctly determine path to active binutils in TensorFlow easyblock (#2218) + - patch Java binaries/libraries when using alternate sysroot to ensure correct glibc & co are picked up + add custom sanity check (#2557, #2995) + - update OpenMPI easyblock to fix sanity check for Clang-based compilers (#2774) + - improve depot management in `JuliaPackage` easyblock (#2935) + - disable disk space check in STAR-CCM+ installer (#2956) + - fix type check for `optarch` value in `Cargo` easyblock (#2969) + - conditionally add `-Wno-unused-command-line-argument` to `$CFLAGS` to fix error when installing `imkl-FFTW` with RPATH (#2975) + - enhance PythonPackage easyblock to deal with `posix_local` installation scheme used by Python in recent Debian/Ubuntu versions (#2977, #2988) + - don't add MATLAB libraries to `$LD_LIBRARY_PATH` (#2981) + - enhance Mesa easyblock to append EGL vendor library directory path to `$__EGL_VENDOR_LIBRARY_DIRS` (#2985) + - fix typo in TensorFlow easyblock when finding libdir of OpenSSL (#2989) v4.8.0 (7 July 2023) diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index 887ecbc220..d8bbee0eb0 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.0') +VERSION = LooseVersion('4.8.1') UNKNOWN = 'UNKNOWN' diff --git a/easybuild/easyblocks/a/advisor.py b/easybuild/easyblocks/a/advisor.py index 952c94642e..54246495fa 100644 --- a/easybuild/easyblocks/a/advisor.py +++ b/easybuild/easyblocks/a/advisor.py @@ -30,6 +30,7 @@ @author: Josef Dvoracek (Institute of Physics, Czech Academy of Sciences) """ +import os from distutils.version import LooseVersion from easybuild.easyblocks.generic.intelbase import IntelBase @@ -45,8 +46,10 @@ def __init__(self, *args, **kwargs): super(EB_Advisor, self).__init__(*args, **kwargs) if LooseVersion(self.version) < LooseVersion('2017'): self.subdir = 'advisor_xe' - else: + elif LooseVersion(self.version) < LooseVersion('2021'): self.subdir = 'advisor' + else: + self.subdir = os.path.join('advisor', 'latest') def prepare_step(self, *args, **kwargs): """Since 2019u3 there is no license required.""" diff --git a/easybuild/easyblocks/c/clang.py b/easybuild/easyblocks/c/clang.py index 172811566d..3fb1b006f5 100644 --- a/easybuild/easyblocks/c/clang.py +++ b/easybuild/easyblocks/c/clang.py @@ -550,14 +550,6 @@ def build_with_prev_stage(self, prev_obj, next_obj): # restore $PATH setvar('PATH', orig_path) - def run_clang_tests(self, obj_dir): - """Run Clang tests in specified directory (unless disabled).""" - if not self.cfg['skip_all_tests']: - change_dir(obj_dir) - - self.log.info("Running tests") - run_cmd("make %s check-all" % self.make_parallel_opts, log_all=True) - def build_step(self): """Build Clang stage 1, 2, 3""" @@ -567,23 +559,20 @@ def build_step(self): super(EB_Clang, self).build_step() if self.cfg['bootstrap']: - # Stage 1: run tests. - self.run_clang_tests(self.llvm_obj_dir_stage1) - self.log.info("Building stage 2") self.build_with_prev_stage(self.llvm_obj_dir_stage1, self.llvm_obj_dir_stage2) - self.run_clang_tests(self.llvm_obj_dir_stage2) self.log.info("Building stage 3") self.build_with_prev_stage(self.llvm_obj_dir_stage2, self.llvm_obj_dir_stage3) - # Don't run stage 3 tests here, do it in the test step. def test_step(self): - """Run Clang tests.""" - if self.cfg['bootstrap']: - self.run_clang_tests(self.llvm_obj_dir_stage3) - else: - self.run_clang_tests(self.llvm_obj_dir_stage1) + """Run Clang tests on final stage (unless disabled).""" + if not self.cfg['skip_all_tests']: + if self.cfg['bootstrap']: + change_dir(self.llvm_obj_dir_stage3) + else: + change_dir(self.llvm_obj_dir_stage1) + run_cmd("make %s check-all" % self.make_parallel_opts, log_all=True) def install_step(self): """Install stage 3 binaries.""" @@ -633,6 +622,11 @@ def sanity_check_step(self): custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags'] shlib_ext = get_shared_lib_ext() + # Clang v16+ only use the major version number for the resource dir + resdir_version = self.version + if LooseVersion(self.version) >= LooseVersion('16'): + resdir_version = self.version.split('.')[0] + # Detect OpenMP support for CPU architecture arch = get_cpu_architecture() # Check architecture explicitly since Clang uses potentially @@ -662,9 +656,9 @@ def sanity_check_step(self): 'files': [ "bin/clang", "bin/clang++", "bin/llvm-ar", "bin/llvm-nm", "bin/llvm-as", "bin/opt", "bin/llvm-link", "bin/llvm-config", "bin/llvm-symbolizer", "include/llvm-c/Core.h", "include/clang-c/Index.h", - "lib/libclang.%s" % shlib_ext, "lib/clang/%s/include/stddef.h" % self.version, + "lib/libclang.%s" % shlib_ext, "lib/clang/%s/include/stddef.h" % resdir_version, ], - 'dirs': ["include/clang", "include/llvm", "lib/clang/%s/lib" % self.version], + 'dirs': ["include/clang", "include/llvm", "lib/clang/%s/lib" % resdir_version], } if self.cfg['static_analyzer']: custom_paths['files'].extend(["bin/scan-build", "bin/scan-view"]) @@ -697,7 +691,7 @@ def sanity_check_step(self): custom_commands.extend(["%s --help" % flang_compiler]) if LooseVersion(self.version) >= LooseVersion('3.8'): - custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % self.version]) + custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % resdir_version]) if LooseVersion(self.version) >= LooseVersion('12'): omp_target_libs = ["lib/libomptarget.%s" % shlib_ext, "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)] diff --git a/easybuild/easyblocks/c/cp2k.py b/easybuild/easyblocks/c/cp2k.py index c9f4171921..a921523e07 100644 --- a/easybuild/easyblocks/c/cp2k.py +++ b/easybuild/easyblocks/c/cp2k.py @@ -440,6 +440,16 @@ def configure_common(self): else: self.log.info("libxc module not loaded, so building without libxc support") + libvori = get_software_root('libvori') + if libvori: + if LooseVersion(self.version) >= LooseVersion('8.1'): + options['LIBS'] += ' -lvori' + options['DFLAGS'] += ' -D__LIBVORI' + else: + raise EasyBuildError("This version of CP2K does not suppport libvori") + else: + self.log.info("libvori module not loaded, so building without support for Voronoi integration") + return options def configure_intel_based(self): diff --git a/easybuild/easyblocks/c/cuda.py b/easybuild/easyblocks/c/cuda.py index d5d6489bd0..32629aab40 100644 --- a/easybuild/easyblocks/c/cuda.py +++ b/easybuild/easyblocks/c/cuda.py @@ -234,6 +234,17 @@ def create_wrapper(wrapper_name, wrapper_comp): raise EasyBuildError("Unable to find 'ldconfig' in $PATH (%s), nor in any of %s", path, sbin_dirs) stubs_dir = os.path.join(self.installdir, 'lib64', 'stubs') + + # Remove stubs which are not required as the full library is in $EBROOTCUDA/lib64 because this duplication + # causes issues (e.g. CMake warnings) when using this module (see $LIBRARY_PATH & $LD_LIBRARY_PATH) + for stub_lib in expand_glob_paths([os.path.join(stubs_dir, '*.*')]): + real_lib = os.path.join(self.installdir, 'lib64', os.path.basename(stub_lib)) + if os.path.exists(real_lib): + self.log.debug("Removing unnecessary stub library %s", stub_lib) + remove_file(stub_lib) + else: + self.log.debug("Keeping stub library %s", stub_lib) + # Run ldconfig to create missing symlinks in the stubs directory (libcuda.so.1, etc) cmd = ' '.join([ldconfig, '-N', stubs_dir]) run_cmd(cmd) @@ -243,7 +254,7 @@ def create_wrapper(wrapper_name, wrapper_comp): # See e.g. https://github.com/easybuilders/easybuild-easyconfigs/issues/12348 # Workaround: Create a copy that matches this pattern new_stubs_dir = os.path.join(self.installdir, 'stubs') - copy_dir(stubs_dir, os.path.join(new_stubs_dir, 'lib64')) + copy_dir(stubs_dir, os.path.join(new_stubs_dir, 'lib64'), symlinks=True) # Also create the lib dir as a symlink symlink('lib64', os.path.join(new_stubs_dir, 'lib'), use_abspath_source=False) diff --git a/easybuild/easyblocks/f/freesurfer.py b/easybuild/easyblocks/f/freesurfer.py index c60059c149..cccb0fbd2b 100644 --- a/easybuild/easyblocks/f/freesurfer.py +++ b/easybuild/easyblocks/f/freesurfer.py @@ -30,6 +30,8 @@ import os +from distutils.version import LooseVersion + from easybuild.easyblocks.generic.tarball import Tarball from easybuild.framework.easyconfig import MANDATORY from easybuild.tools.filetools import write_file @@ -92,4 +94,8 @@ def sanity_check_step(self): 'dirs': ['bin', 'lib', 'mni'], } - super(EB_FreeSurfer, self).sanity_check_step(custom_paths=custom_paths) + custom_commands = [] + if LooseVersion(self.version) >= LooseVersion("7.2"): + custom_commands.append('checkMCR.sh') + + super(EB_FreeSurfer, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/generic/cargo.py b/easybuild/easyblocks/generic/cargo.py index 56cda395fb..c47f2e9f05 100644 --- a/easybuild/easyblocks/generic/cargo.py +++ b/easybuild/easyblocks/generic/cargo.py @@ -74,7 +74,7 @@ def rustc_optarch(self): optarch = build_option('optarch') if optarch: - if type(optarch) == dict: + if isinstance(optarch, dict): if 'rustc' in optarch: rust_optarch = optarch['rustc'] if rust_optarch == OPTARCH_GENERIC: @@ -82,11 +82,10 @@ def rustc_optarch(self): else: return '-' + rust_optarch self.log.info("no rustc information in the optarch dict, so using %s" % optimal) + elif optarch == OPTARCH_GENERIC: + return generic else: - if optarch == OPTARCH_GENERIC: - return generic - else: - self.log.warning("optarch is ignored as there is no translation for rustc, so using %s" % optimal) + self.log.warning("optarch is ignored as there is no translation for rustc, so using %s" % optimal) return optimal def __init__(self, *args, **kwargs): diff --git a/easybuild/easyblocks/generic/cmakeninja.py b/easybuild/easyblocks/generic/cmakeninja.py index db344470f1..f14acb9989 100644 --- a/easybuild/easyblocks/generic/cmakeninja.py +++ b/easybuild/easyblocks/generic/cmakeninja.py @@ -38,9 +38,13 @@ class CMakeNinja(CMakeMake, MesonNinja): @staticmethod def extra_options(extra_vars=None): - """Define extra easyconfig parameters specific to CMakeMake.""" + """Define extra easyconfig parameters specific to CMakeNinja.""" extra_vars = CMakeMake.extra_options(extra_vars) extra_vars['generator'][0] = 'Ninja' + extra_vars.update({ + key: value for key, value in MesonNinja.extra_options().items() + if key.startswith('build_') or key.startswith('install_') + }) return extra_vars def configure_step(self, *args, **kwargs): diff --git a/easybuild/easyblocks/generic/juliabundle.py b/easybuild/easyblocks/generic/juliabundle.py index 28c8035068..0f316092c4 100644 --- a/easybuild/easyblocks/generic/juliabundle.py +++ b/easybuild/easyblocks/generic/juliabundle.py @@ -86,9 +86,12 @@ def prepare_step(self, *args, **kwargs): raise EasyBuildError("Julia not included as dependency!") def make_module_extra(self, *args, **kwargs): - """Prepend installation directory to JULIA_DEPOT_PATH in module file.""" + """ + Module has to append installation directory to JULIA_DEPOT_PATH to keep + the user depot in the top entry. See issue easybuilders/easybuild-easyconfigs#17455 + """ txt = super(JuliaBundle, self).make_module_extra() - txt += self.module_generator.prepend_paths('JULIA_DEPOT_PATH', ['']) + txt += self.module_generator.append_paths('JULIA_DEPOT_PATH', ['']) return txt def sanity_check_step(self, *args, **kwargs): diff --git a/easybuild/easyblocks/generic/juliapackage.py b/easybuild/easyblocks/generic/juliapackage.py index 2e32047665..6e535d7f44 100644 --- a/easybuild/easyblocks/generic/juliapackage.py +++ b/easybuild/easyblocks/generic/juliapackage.py @@ -28,6 +28,7 @@ @author: Alex Domingo (Vrije Universiteit Brussel) """ import os +import re from distutils.version import LooseVersion @@ -40,6 +41,7 @@ from easybuild.tools.run import run_cmd EXTS_FILTER_JULIA_PACKAGES = ("julia -e 'using %(ext_name)s'", "") +USER_DEPOT_PATTERN = re.compile(r"\/\.julia\/?$") class JuliaPackage(ExtensionEasyBlock): @@ -56,6 +58,26 @@ def extra_options(extra_vars=None): }) return extra_vars + def set_depot_path(self): + """ + Top directory in JULIA_DEPOT_PATH is target installation directory + Prepend installation directory to JULIA_DEPOT_PATH + Remove user depot from JULIA_DEPOT_PATH during installation + see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH + """ + depot_path = os.getenv('JULIA_DEPOT_PATH', []) + + if depot_path: + depot_path = depot_path.split(os.pathsep) + if len(depot_path) > 0: + # strip user depot path (top entry by definition) + if USER_DEPOT_PATTERN.search(depot_path[0]): + self.log.debug('Temporary disabling Julia user depot: %s', depot_path[0]) + del depot_path[0] + + depot_path.insert(0, self.installdir) + env.setvar('JULIA_DEPOT_PATH', os.pathsep.join(depot_path)) + def set_pkg_offline(self): """Enable offline mode of Julia Pkg""" if get_software_root('Julia') is None: @@ -79,6 +101,7 @@ def prepare_step(self, *args, **kwargs): """Prepare for installing Julia package.""" super(JuliaPackage, self).prepare_step(*args, **kwargs) self.set_pkg_offline() + self.set_depot_path() def configure_step(self): """No separate configuration for JuliaPackage.""" @@ -95,16 +118,6 @@ def test_step(self): def install_step(self): """Install Julia package with Pkg""" - # prepend installation directory to Julia DEPOT_PATH - # extensions in a bundle can share their DEPOT_PATH - # see https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH - depot_path = os.getenv('JULIA_DEPOT_PATH', []) - if depot_path: - depot_path = depot_path.split(os.pathsep) - if self.installdir not in depot_path: - depot_path = os.pathsep.join([depot for depot in [self.installdir] + depot_path if depot]) - env.setvar('JULIA_DEPOT_PATH', depot_path) - # command sequence for Julia.Pkg julia_pkg_cmd = ['using Pkg'] if os.path.isdir(os.path.join(self.start_dir, '.git')): @@ -147,6 +160,7 @@ def run(self): ExtensionEasyBlock.run(self, unpack_src=True) self.set_pkg_offline() + self.set_depot_path() # all extensions share common depot in installdir self.install_step() def sanity_check_step(self, *args, **kwargs): @@ -163,7 +177,10 @@ def sanity_check_step(self, *args, **kwargs): return ExtensionEasyBlock.sanity_check_step(self, EXTS_FILTER_JULIA_PACKAGES, *args, **kwargs) def make_module_extra(self): - """Prepend installation directory to JULIA_DEPOT_PATH in module file.""" + """ + Module has to append installation directory to JULIA_DEPOT_PATH to keep + the user depot in the top entry. See issue easybuilders/easybuild-easyconfigs#17455 + """ txt = super(JuliaPackage, self).make_module_extra() - txt += self.module_generator.prepend_paths('JULIA_DEPOT_PATH', ['']) + txt += self.module_generator.append_paths('JULIA_DEPOT_PATH', ['']) return txt diff --git a/easybuild/easyblocks/generic/mesonninja.py b/easybuild/easyblocks/generic/mesonninja.py index c8ad0af51b..8ce6b530e4 100644 --- a/easybuild/easyblocks/generic/mesonninja.py +++ b/easybuild/easyblocks/generic/mesonninja.py @@ -37,6 +37,8 @@ from easybuild.tools.run import run_cmd DEFAULT_CONFIGURE_CMD = 'meson' +DEFAULT_BUILD_CMD = 'ninja' +DEFAULT_INSTALL_CMD = 'ninja' class MesonNinja(EasyBlock): @@ -49,7 +51,10 @@ def extra_options(extra_vars=None): """Define extra easyconfig parameters specific to MesonNinja.""" extra_vars = EasyBlock.extra_options(extra_vars) extra_vars.update({ + 'build_dir': [None, "build_dir to pass to meson", CUSTOM], + 'build_cmd': [DEFAULT_BUILD_CMD, "Build command to use", CUSTOM], 'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM], + 'install_cmd': [DEFAULT_INSTALL_CMD, "Install command to use", CUSTOM], 'separate_build_dir': [True, "Perform build in a separate directory", CUSTOM], }) return extra_vars @@ -85,12 +90,14 @@ def configure_step(self, cmd_prefix=''): configure_cmd == DEFAULT_CONFIGURE_CMD): configure_cmd += ' setup' - cmd = "%(preconfigopts)s %(configure_cmd)s --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { + build_dir = self.cfg.get('build_dir') or self.start_dir + + cmd = "%(preconfigopts)s %(configure_cmd)s --prefix %(installdir)s %(configopts)s %(source_dir)s" % { 'configopts': self.cfg['configopts'], 'configure_cmd': configure_cmd, 'installdir': self.installdir, 'preconfigopts': self.cfg['preconfigopts'], - 'sourcedir': self.start_dir, + 'source_dir': build_dir, } (out, _) = run_cmd(cmd, log_all=True, simple=False) return out @@ -99,12 +106,15 @@ def build_step(self, verbose=False, path=None): """ Build with Ninja. """ + build_cmd = self.cfg.get('build_cmd', DEFAULT_BUILD_CMD) + parallel = '' if self.cfg['parallel']: parallel = "-j %s" % self.cfg['parallel'] - cmd = "%(prebuildopts)s ninja %(parallel)s %(buildopts)s" % { + cmd = "%(prebuildopts)s %(build_cmd)s %(parallel)s %(buildopts)s" % { 'buildopts': self.cfg['buildopts'], + 'build_cmd': build_cmd, 'parallel': parallel, 'prebuildopts': self.cfg['prebuildopts'], } @@ -124,13 +134,16 @@ def install_step(self): """ Install with 'ninja install'. """ + install_cmd = self.cfg.get('install_cmd', DEFAULT_INSTALL_CMD) + parallel = '' if self.cfg['parallel']: parallel = "-j %s" % self.cfg['parallel'] - cmd = "%(preinstallopts)s ninja %(parallel)s %(installopts)s install" % { + cmd = "%(preinstallopts)s %(install_cmd)s %(parallel)s %(installopts)s install" % { 'installopts': self.cfg['installopts'], 'parallel': parallel, + 'install_cmd': install_cmd, 'preinstallopts': self.cfg['preinstallopts'], } (out, _) = run_cmd(cmd, log_all=True, simple=False) diff --git a/easybuild/easyblocks/generic/perlmodule.py b/easybuild/easyblocks/generic/perlmodule.py index f58163d6e8..e10b6d5b1c 100644 --- a/easybuild/easyblocks/generic/perlmodule.py +++ b/easybuild/easyblocks/generic/perlmodule.py @@ -47,6 +47,7 @@ def extra_options(): """Easyconfig parameters specific to Perl modules.""" extra_vars = { 'runtest': ['test', "Run unit tests.", CUSTOM], # overrides default + 'prefix_opt': [None, "String to use for option to set installation prefix (default is 'PREFIX')", CUSTOM], } return ExtensionEasyBlock.extra_options(extra_vars) @@ -62,14 +63,20 @@ def __init__(self, *args, **kwargs): def install_perl_module(self): """Install procedure for Perl modules: using either Makefile.Pl or Build.PL.""" + prefix_opt = self.cfg.get('prefix_opt') + # Perl modules have two possible installation procedures: using Makefile.PL and Build.PL # configure, build, test, install if os.path.exists('Makefile.PL'): + + if prefix_opt is None: + prefix_opt = 'PREFIX' + install_cmd = ' '.join([ self.cfg['preconfigopts'], 'perl', 'Makefile.PL', - 'PREFIX=%s' % self.installdir, + '%s=%s' % (prefix_opt, self.installdir), self.cfg['configopts'], ]) run_cmd(install_cmd) @@ -79,11 +86,15 @@ def install_perl_module(self): ConfigureMake.install_step(self) elif os.path.exists('Build.PL'): + + if prefix_opt is None: + prefix_opt = '--prefix' + install_cmd = ' '.join([ self.cfg['preconfigopts'], 'perl', 'Build.PL', - '--prefix', + prefix_opt, self.installdir, self.cfg['configopts'], ]) diff --git a/easybuild/easyblocks/generic/pythonpackage.py b/easybuild/easyblocks/generic/pythonpackage.py index 412d005270..29c7a8d803 100644 --- a/easybuild/easyblocks/generic/pythonpackage.py +++ b/easybuild/easyblocks/generic/pythonpackage.py @@ -49,7 +49,7 @@ from easybuild.framework.extensioneasyblock import ExtensionEasyBlock from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option -from easybuild.tools.filetools import mkdir, remove_dir, which +from easybuild.tools.filetools import change_dir, mkdir, remove_dir, symlink, which from easybuild.tools.modules import get_software_root from easybuild.tools.py2vs3 import string_type, subprocess_popen_text from easybuild.tools.run import run_cmd @@ -64,6 +64,17 @@ SETUP_PY_INSTALL_CMD = "%(python)s setup.py %(install_target)s --prefix=%(prefix)s %(installopts)s" UNKNOWN = 'UNKNOWN' +# Python installation schemes, see https://docs.python.org/3/library/sysconfig.html#installation-paths; +# posix_prefix is the default upstream installation scheme (and the want to want) +PY_INSTALL_SCHEME_POSIX_PREFIX = 'posix_prefix' +# posix_local is custom installation scheme on Debian/Ubuntu which implies additional action, +# see https://github.com/easybuilders/easybuild-easyblocks/issues/2976 +PY_INSTALL_SCHEME_POSIX_LOCAL = 'posix_local' +PY_INSTALL_SCHEMES = [ + PY_INSTALL_SCHEME_POSIX_PREFIX, + PY_INSTALL_SCHEME_POSIX_LOCAL, +] + def det_python_version(python_cmd): """Determine version of specified 'python' command.""" @@ -224,6 +235,88 @@ def det_pip_version(python_cmd='python'): return pip_version +def det_py_install_scheme(python_cmd='python'): + """ + Try to determine active installation scheme used by Python. + """ + # default installation scheme is 'posix_prefix', + # see also https://docs.python.org/3/library/sysconfig.html#installation-paths; + # on Debian/Ubuntu, we may be getting 'posix_local' as custom installation scheme, + # which injects /local as a subdirectory and cause trouble + # (see also https://github.com/easybuilders/easybuild-easyblocks/issues/2976) + + log = fancylogger.getLogger('det_py_install_scheme', fname=False) + + # sysconfig._get_default_scheme was renamed to sysconfig.get_default_scheme in Python 3.10 + pyver = det_python_version(python_cmd) + if LooseVersion(pyver) >= LooseVersion('3.10'): + get_default_scheme = 'get_default_scheme' + else: + get_default_scheme = '_get_default_scheme' + + cmd = "%s -c 'import sysconfig; print(sysconfig.%s())'" % (python_cmd, get_default_scheme) + log.debug("Determining active Python installation scheme with: %s", cmd) + out, _ = run_cmd(cmd, verbose=False, simple=False, trace=False) + py_install_scheme = out.strip() + + if py_install_scheme in PY_INSTALL_SCHEMES: + log.info("Active Python installation scheme: %s", py_install_scheme) + else: + log.warning("Unknown Python installation scheme: %s", py_install_scheme) + + return py_install_scheme + + +def handle_local_py_install_scheme(install_dir): + """ + Handle situation in which 'posix_local' installation scheme was used, + which implies that /local/' rather than / was used as installation prefix... + """ + # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2976 + + log = fancylogger.getLogger('handle_local_py_install_scheme', fname=False) + + install_dir_local = os.path.join(install_dir, 'local') + if os.path.exists(install_dir_local): + subdirs = os.listdir(install_dir) + log.info("Found 'local' subdirectory in installation prefix %s: %s", install_dir, subdirs) + + local_subdirs = os.listdir(install_dir_local) + log.info("Subdirectories of %s: %s", install_dir_local, local_subdirs) + + # symlink subdirectories of /local directly into + cwd = change_dir(install_dir) + for local_subdir in local_subdirs: + srcpath = os.path.join('local', local_subdir) + symlink(srcpath, os.path.join(install_dir, local_subdir), use_abspath_source=False) + change_dir(cwd) + + +def symlink_dist_site_packages(install_dir, pylibdirs): + """ + Symlink site-packages to dist-packages if only the latter is available in the specified directories. + """ + # in some situations, for example when the default installation scheme is not the upstream default posix_prefix, + # as is the case in Ubuntu 22.04 (cfr. https://github.com/easybuilders/easybuild-easyblocks/issues/2976), + # Python packages may get installed in /.../dist-packages rather than /.../site-packages; + # we try to determine all possible paths in get_pylibdirs but we still may get it wrong, + # mostly because distutils.sysconfig.get_python_lib(..., prefix=...) isn't correct when posix_prefix + # is not the active installation scheme; + # so taking the coward way out: just symlink site-packages to dist-packages if only latter is available + dist_pkgs = 'dist-packages' + for pylibdir in pylibdirs: + dist_pkgs_path = os.path.join(install_dir, os.path.dirname(pylibdir), dist_pkgs) + site_pkgs_path = os.path.join(os.path.dirname(dist_pkgs_path), 'site-packages') + + # site-packages may be there as empty directory (see mkdir loop in install_step); + # just remove it if that's the case so we can symlink to dist-packages + if os.path.exists(site_pkgs_path) and not os.listdir(site_pkgs_path): + remove_dir(site_pkgs_path) + + if os.path.exists(dist_pkgs_path) and not os.path.exists(site_pkgs_path): + symlink(dist_pkgs, site_pkgs_path, use_abspath_source=False) + + class PythonPackage(ExtensionEasyBlock): """Builds and installs a Python package, and provides a dedicated module file.""" @@ -469,11 +562,32 @@ def get_installed_python_packages(self, names_only=True, python_cmd=None): else: return pkgs + def using_pip_install(self): + """ + Check whether 'pip install --prefix' is being used to install Python packages. + """ + if self.install_cmd.startswith(PIP_INSTALL_CMD): + self.log.debug("Using 'pip install' for installing Python packages: %s" % self.install_cmd) + return True + else: + self.log.debug("Not using 'pip install' for installing Python packages (install command template: %s)", + self.install_cmd) + return False + + def using_local_py_install_scheme(self): + """ + Determine whether the custom 'posix_local' Python installation scheme is actually used. + This requires that 'pip install --prefix' is used, since the active Python installation scheme + doesn't matter when using 'python setup.py install --prefix'. + """ + # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2976 + py_install_scheme = det_py_install_scheme(python_cmd=self.python_cmd) + return py_install_scheme == PY_INSTALL_SCHEME_POSIX_LOCAL and self.using_pip_install() + def compose_install_command(self, prefix, extrapath=None, installopts=None): """Compose full install command.""" - using_pip = self.install_cmd.startswith(PIP_INSTALL_CMD) - if using_pip: + if self.using_pip_install(): pip_version = det_pip_version(python_cmd=self.python_cmd) if pip_version: @@ -510,7 +624,7 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): # otherwise, self.src is a list of dicts, one element per source file loc = self.src[0]['path'] - if using_pip: + if self.using_pip_install(): extras = self.cfg.get('use_pip_extras') if extras: loc += '[%s]' % extras @@ -539,6 +653,22 @@ def compose_install_command(self, prefix, extrapath=None, installopts=None): return ' '.join(cmd) + def py_post_install_shenanigans(self, install_dir): + """ + Run post-installation shenanigans on specified installation directory, incl: + * dealing with 'local' subdirectory in install directory in case 'posix_local' installation scheme was used; + * symlinking site-packages to dist-packages if only the former is available; + """ + if self.using_local_py_install_scheme(): + self.log.debug("Looks like the active Python installation scheme injected a 'local' subdirectory...") + handle_local_py_install_scheme(install_dir) + else: + self.log.debug("Looks like active Python installation scheme did not inject a 'local' subdirectory, good!") + + py_install_scheme = det_py_install_scheme(python_cmd=self.python_cmd) + if py_install_scheme != PY_INSTALL_SCHEME_POSIX_PREFIX: + symlink_dist_site_packages(install_dir, self.all_pylibdirs) + def extract_step(self): """Unpack source files, unless instructed otherwise.""" if self._should_unpack_source(): @@ -658,7 +788,7 @@ def test_step(self, return_output_ec=False): if self.cfg['runtest'] and self.testcmd is not None: extrapath = "" - testinstalldir = None + test_installdir = None out, ec = (None, None) @@ -666,21 +796,32 @@ def test_step(self, return_output_ec=False): # install in test directory and export PYTHONPATH try: - testinstalldir = tempfile.mkdtemp() + test_installdir = tempfile.mkdtemp() + + # if posix_local is the active installation scheme there will be + # a 'local' subdirectory in the specified prefix; + if self.using_local_py_install_scheme(): + actual_installdir = os.path.join(test_installdir, 'local') + else: + actual_installdir = test_installdir + + self.log.debug("Pre-creating subdirectories in %s: %s", actual_installdir, self.all_pylibdirs) for pylibdir in self.all_pylibdirs: - mkdir(os.path.join(testinstalldir, pylibdir), parents=True) + mkdir(os.path.join(actual_installdir, pylibdir), parents=True) except OSError as err: raise EasyBuildError("Failed to create test install dir: %s", err) # print Python search path (just debugging purposes) run_cmd("%s -c 'import sys; print(sys.path)'" % self.python_cmd, verbose=False, trace=False) - abs_pylibdirs = [os.path.join(testinstalldir, pylibdir) for pylibdir in self.all_pylibdirs] + abs_pylibdirs = [os.path.join(actual_installdir, pylibdir) for pylibdir in self.all_pylibdirs] extrapath = "export PYTHONPATH=%s &&" % os.pathsep.join(abs_pylibdirs + ['$PYTHONPATH']) - cmd = self.compose_install_command(testinstalldir, extrapath=extrapath) + cmd = self.compose_install_command(test_installdir, extrapath=extrapath) run_cmd(cmd, log_all=True, simple=True, verbose=False) + self.py_post_install_shenanigans(test_installdir) + if self.testcmd: testcmd = self.testcmd % {'python': self.python_cmd} cmd = ' '.join([ @@ -697,8 +838,8 @@ def test_step(self, return_output_ec=False): else: run_cmd(cmd, log_all=True, simple=True) - if testinstalldir: - remove_dir(testinstalldir) + if test_installdir: + remove_dir(test_installdir) if return_output_ec: return (out, ec) @@ -706,12 +847,21 @@ def test_step(self, return_output_ec=False): def install_step(self): """Install Python package to a custom path using setup.py""" + # if posix_local is the active installation scheme there will be + # a 'local' subdirectory in the specified prefix; + # see also https://github.com/easybuilders/easybuild-easyblocks/issues/2976 + if self.using_local_py_install_scheme(): + actual_installdir = os.path.join(self.installdir, 'local') + else: + actual_installdir = self.installdir + # create expected directories - abs_pylibdirs = [os.path.join(self.installdir, pylibdir) for pylibdir in self.all_pylibdirs] + abs_pylibdirs = [os.path.join(actual_installdir, pylibdir) for pylibdir in self.all_pylibdirs] + self.log.debug("Pre-creating subdirectories %s in %s...", abs_pylibdirs, actual_installdir) for pylibdir in abs_pylibdirs: mkdir(pylibdir, parents=True) - abs_bindir = os.path.join(self.installdir, 'bin') + abs_bindir = os.path.join(actual_installdir, 'bin') # set PYTHONPATH and PATH as expected old_values = dict() @@ -731,6 +881,8 @@ def install_step(self): # (for iterated installations over multiply Python versions) self.install_cmd_output += out + self.py_post_install_shenanigans(self.installdir) + # fix shebangs if specified self.fix_shebang() diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index c2bb129b2e..978b42137c 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -46,6 +46,7 @@ from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.config import build_option from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir, move_file, remove_dir, write_file from easybuild.tools.modules import get_software_root from easybuild.tools.run import run_cmd @@ -261,6 +262,13 @@ def build_mkl_fftw_interfaces(self, libdir): tmpbuild = tempfile.mkdtemp(dir=self.builddir) self.log.debug("Created temporary directory %s" % tmpbuild) + # Avoid unused command line arguments (-Wl,rpath...) causing errors when using RPATH + # See https://github.com/easybuilders/easybuild-easyconfigs/pull/18439#issuecomment-1662671054 + if build_option('rpath') and os.getenv('CC') in ('icx', 'clang'): + cflags = flags + ' -Wno-unused-command-line-argument' + else: + cflags = flags + # always set INSTALL_DIR, SPEC_OPT, COPTS and CFLAGS # fftw2x(c|f): use $INSTALL_DIR, $CFLAGS and $COPTS # fftw3x(c|f): use $CFLAGS @@ -268,7 +276,7 @@ def build_mkl_fftw_interfaces(self, libdir): env.setvar('INSTALL_DIR', tmpbuild) env.setvar('SPEC_OPT', flags) env.setvar('COPTS', flags) - env.setvar('CFLAGS', flags) + env.setvar('CFLAGS', cflags) intdir = os.path.join(interfacedir, lib) change_dir(intdir) diff --git a/easybuild/easyblocks/j/java.py b/easybuild/easyblocks/j/java.py index 63e0499321..9eca14cf72 100644 --- a/easybuild/easyblocks/j/java.py +++ b/easybuild/easyblocks/j/java.py @@ -28,16 +28,18 @@ @author: Jens Timmerman (Ghent University) @author: Kenneth Hoste (Ghent University) """ - +import glob import os import stat from distutils.version import LooseVersion from easybuild.easyblocks.generic.packedbinary import PackedBinary from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, remove_dir +from easybuild.tools.config import build_option +from easybuild.tools.filetools import adjust_permissions, change_dir, copy_dir, copy_file, remove_dir, which from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture +from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_shared_lib_ext +from easybuild.tools.utilities import nub class EB_Java(PackedBinary): @@ -76,15 +78,138 @@ def extract_step(self): adjust_permissions(self.builddir, stat.S_IWUSR, add=True, recursive=True) def install_step(self): + """Custom install step: just copy unpacked installation files.""" if LooseVersion(self.version) < LooseVersion('1.7'): remove_dir(self.installdir) copy_dir(os.path.join(self.builddir, 'jdk%s' % self.version), self.installdir) else: PackedBinary.install_step(self) + def post_install_step(self): + """ + Custom post-installation step: + - ensure correct glibc is used when installing into custom sysroot and using RPATH + """ + super(EB_Java, self).post_install_step() + + # patch binaries and libraries when using alternate sysroot in combination with RPATH + sysroot = build_option('sysroot') + if sysroot and self.toolchain.use_rpath: + if not which('patchelf'): + error_msg = "patchelf not found via $PATH, required to patch RPATH section in binaries/libraries" + raise EasyBuildError(error_msg) + + try: + # list of paths in sysroot to consider for adding to RPATH section + sysroot_lib_paths = glob.glob(os.path.join(sysroot, 'lib*')) + sysroot_lib_paths += glob.glob(os.path.join(sysroot, 'usr', 'lib*')) + sysroot_lib_paths += glob.glob(os.path.join(sysroot, 'usr', 'lib*', 'gcc', '*', '*')) + if sysroot_lib_paths: + self.log.info("List of library paths in %s to add to RPATH section: %s", sysroot, sysroot_lib_paths) + + # find path to ELF interpreter + elf_interp = None + + for ld_glob_pattern in (r'ld-linux-*.so.*', r'ld*.so.*'): + res = glob.glob(os.path.join(sysroot, 'lib*', ld_glob_pattern)) + self.log.debug("Paths for ELF interpreter via '%s' pattern: %s", ld_glob_pattern, res) + if res: + # if there are multiple hits, make sure they resolve to the same paths, + # but keep using the symbolic link, not the resolved path! + real_paths = nub([os.path.realpath(x) for x in res]) + if len(real_paths) == 1: + elf_interp = res[0] + self.log.info("ELF interpreter found at %s", elf_interp) + break + else: + raise EasyBuildError("Multiple different unique ELF interpreters found: %s", real_paths) + + if elf_interp is None: + raise EasyBuildError("Failed to isolate ELF interpreter!") + + module_guesses = self.make_module_req_guess() + + bindirs = [os.path.join(self.installdir, bindir) for bindir in module_guesses['PATH'] if + os.path.exists(os.path.join(self.installdir, bindir))] + # Make sure these are unique real paths + bindirs = list(set([os.path.realpath(path) for path in bindirs])) + for bindir in bindirs: + for path in os.listdir(bindir): + path = os.path.join(bindir, path) + out, _ = run_cmd("file %s" % path, trace=False) + if "dynamically linked" in out: + + out, _ = run_cmd("patchelf --print-interpreter %s" % path, trace=False) + self.log.debug("ELF interpreter for %s: %s" % (path, out)) + + run_cmd("patchelf --set-interpreter %s %s" % (elf_interp, path), trace=False) + + out, _ = run_cmd("patchelf --print-interpreter %s" % path, trace=False) + self.log.debug("ELF interpreter for %s: %s" % (path, out)) + + out, _ = run_cmd("patchelf --print-rpath %s" % path, simple=False, trace=False) + curr_rpath = out.strip() + self.log.debug("RPATH for %s: %s" % (path, curr_rpath)) + + new_rpath = ':'.join([curr_rpath] + sysroot_lib_paths) + # note: it's important to wrap the new RPATH value in single quotes, + # to avoid magic values like $ORIGIN being resolved by the shell + run_cmd("patchelf --set-rpath '%s' %s" % (new_rpath, path), trace=False) + + curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % path, simple=False, trace=False) + self.log.debug("RPATH for %s (prior to shrinking): %s" % (path, curr_rpath)) + + run_cmd("patchelf --shrink-rpath %s" % path, trace=False) + + curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % path, simple=False, trace=False) + self.log.debug("RPATH for %s (after shrinking): %s" % (path, curr_rpath)) + + libdirs = [os.path.join(self.installdir, libdir) for libdir in module_guesses['LIBRARY_PATH'] if + os.path.exists(os.path.join(self.installdir, libdir))] + # Make sure these are unique real paths + libdirs = list(set([os.path.realpath(path) for path in libdirs])) + shlib_ext = '.' + get_shared_lib_ext() + for libdir in libdirs: + for path, _, filenames in os.walk(libdir): + shlibs = [os.path.join(path, x) for x in filenames if x.endswith(shlib_ext)] + for shlib in shlibs: + out, _ = run_cmd("patchelf --print-rpath %s" % shlib, simple=False, trace=False) + curr_rpath = out.strip() + self.log.debug("RPATH for %s: %s" % (shlib, curr_rpath)) + + new_rpath = ':'.join([curr_rpath] + sysroot_lib_paths) + # note: it's important to wrap the new RPATH value in single quotes, + # to avoid magic values like $ORIGIN being resolved by the shell + run_cmd("patchelf --set-rpath '%s' %s" % (new_rpath, shlib), trace=False) + + curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % shlib, simple=False, trace=False) + self.log.debug("RPATH for %s (prior to shrinking): %s" % (path, curr_rpath)) + + run_cmd("patchelf --shrink-rpath %s" % shlib, trace=False) + + curr_rpath, _ = run_cmd("patchelf --print-rpath %s" % shlib, simple=False, trace=False) + self.log.debug("RPATH for %s (after shrinking): %s" % (path, curr_rpath)) + + except OSError as err: + raise EasyBuildError("Failed to patch RPATH section in binaries/libraries: %s", err) + + def sanity_check_step(self): + """Custom sanity check for Java.""" + custom_paths = { + 'files': ['bin/java', 'bin/javac'], + 'dirs': ['lib'], + } + + custom_commands = [ + "java -help", + "javac -help", + ] + + super(EB_Java, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) + def make_module_extra(self): """ - Set JAVA_HOME to install dir + Set $JAVA_HOME to installation directory """ txt = PackedBinary.make_module_extra(self) txt += self.module_generator.set_environment('JAVA_HOME', self.installdir) diff --git a/easybuild/easyblocks/l/llvm.py b/easybuild/easyblocks/l/llvm.py index c7f06d6b6e..5bced9a4c2 100644 --- a/easybuild/easyblocks/l/llvm.py +++ b/easybuild/easyblocks/l/llvm.py @@ -34,7 +34,7 @@ from easybuild.easyblocks.generic.cmakemake import CMakeMake from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.filetools import change_dir, symlink +from easybuild.tools.filetools import move_file from easybuild.tools.modules import get_software_root from easybuild.tools.systemtools import get_cpu_architecture from distutils.version import LooseVersion @@ -104,13 +104,21 @@ def configure_step(self): if LooseVersion(self.version) >= LooseVersion('15.0'): # make sure that CMake modules are available in build directory, - # and if so make a 'cmake' symlink so LLVM can find them + # by moving the extracted folder to the expected location cmake_modules_path = os.path.join(self.builddir, 'cmake-%s.src' % self.version) if os.path.exists(cmake_modules_path): - cwd = change_dir(self.builddir) - symlink('cmake-%s.src' % self.version, 'cmake') - change_dir(cwd) + move_file(cmake_modules_path, os.path.join(self.builddir, 'cmake')) else: raise EasyBuildError("Failed to find unpacked CMake modules directory at %s", cmake_modules_path) + if LooseVersion(self.version) >= LooseVersion('16.0'): + # make sure that third-party modules are available in build directory, + # by moving the extracted folder to the expected location + third_party_modules_path = os.path.join(self.builddir, 'third-party-%s.src' % self.version) + if os.path.exists(third_party_modules_path): + move_file(third_party_modules_path, os.path.join(self.builddir, 'third-party')) + else: + raise EasyBuildError("Failed to find unpacked 'third-party' modules directory at %s", + third_party_modules_path) + super(EB_LLVM, self).configure_step() diff --git a/easybuild/easyblocks/m/matlab.py b/easybuild/easyblocks/m/matlab.py index b260e703de..ec2f43fa39 100644 --- a/easybuild/easyblocks/m/matlab.py +++ b/easybuild/easyblocks/m/matlab.py @@ -221,11 +221,6 @@ def make_module_extra(self): """Extend PATH and set proper _JAVA_OPTIONS (e.g., -Xmx).""" txt = super(EB_MATLAB, self).make_module_extra() - # make MATLAB runtime available - if LooseVersion(self.version) >= LooseVersion('2017a'): - for ldlibdir in ['runtime', 'bin', os.path.join('sys', 'os')]: - libdir = os.path.join(ldlibdir, 'glnxa64') - txt += self.module_generator.prepend_paths('LD_LIBRARY_PATH', libdir) if self.cfg['java_options']: txt += self.module_generator.set_environment('_JAVA_OPTIONS', self.cfg['java_options']) return txt diff --git a/easybuild/easyblocks/m/mesa.py b/easybuild/easyblocks/m/mesa.py index 684f37c2be..897faacbaa 100644 --- a/easybuild/easyblocks/m/mesa.py +++ b/easybuild/easyblocks/m/mesa.py @@ -154,7 +154,11 @@ def sanity_check_step(self): shlib_ext = get_shared_lib_ext() if LooseVersion(self.version) >= LooseVersion('20.0'): - header_files = [os.path.join('include', 'EGL', x) for x in ['eglmesaext.h', 'eglextchromium.h']] + header_files = [os.path.join('include', 'EGL', 'eglmesaext.h')] + if LooseVersion(self.version) >= LooseVersion('22.3'): + header_files.extend([os.path.join('include', 'EGL', 'eglext_angle.h')]) + else: + header_files.extend([os.path.join('include', 'EGL', 'eglextchromium.h')]) header_files.extend([ os.path.join('include', 'GL', 'osmesa.h'), os.path.join('include', 'GL', 'internal', 'dri_interface.h'), @@ -175,3 +179,11 @@ def sanity_check_step(self): custom_paths['files'].extend(swr_arch_libs) super(EB_Mesa, self).sanity_check_step(custom_paths=custom_paths) + + def make_module_extra(self, *args, **kwargs): + """ Append to EGL vendor library path, + so that any NVidia libraries take precedence. """ + txt = super(EB_Mesa, self).make_module_extra(*args, **kwargs) + # Append rather than prepend path to ensure that system NVidia drivers have priority. + txt += self.module_generator.append_paths('__EGL_VENDOR_LIBRARY_DIRS', 'share/glvnd/egl_vendor.d') + return txt diff --git a/easybuild/easyblocks/m/mxnet.py b/easybuild/easyblocks/m/mxnet.py index 7dee5aeed6..10bf26cc5d 100644 --- a/easybuild/easyblocks/m/mxnet.py +++ b/easybuild/easyblocks/m/mxnet.py @@ -32,6 +32,7 @@ import shutil from distutils.version import LooseVersion +import easybuild.tools.environment as env from easybuild.easyblocks.generic.makecp import MakeCp from easybuild.easyblocks.generic.pythonpackage import PythonPackage from easybuild.easyblocks.generic.rpackage import RPackage @@ -63,11 +64,15 @@ class EB_MXNet(MakeCp): """Easyblock to build and install MXNet""" @staticmethod - def extra_options(extra_vars=None): + def extra_options(): """Change default values of options""" - extra = MakeCp.extra_options() + extra_vars = { + 'install_r_ext': [None, "Enable installation of R extensions", CUSTOM], + } + extra = MakeCp.extra_options(extra_vars) # files_to_copy is not mandatory here extra['files_to_copy'][2] = CUSTOM + return extra def __init__(self, *args, **kwargs): @@ -79,6 +84,13 @@ def __init__(self, *args, **kwargs): self.py_ext.module_generator = self.module_generator self.r_ext = RPackage(self, {'name': self.name, 'version': self.version}) self.r_ext.module_generator = self.module_generator + # auto-enable building of R extensions only for old versions of MXNet (< 1.0), + # since for newer versions is broken + if self.cfg['install_r_ext'] is None: + if LooseVersion(self.version) < LooseVersion('1.0'): + self.cfg['install_r_ext'] = True + else: + self.cfg['install_r_ext'] = False def extract_step(self): """ @@ -97,20 +109,36 @@ def extract_step(self): for srcdir in [d for d in os.listdir(self.builddir) if d != os.path.basename(self.mxnet_src_dir)]: submodule, _, _ = srcdir.rpartition('-') - newdir = os.path.join(self.mxnet_src_dir, submodule) - olddir = os.path.join(self.builddir, srcdir) + + if LooseVersion(self.version) >= LooseVersion('1.0'): + # if newdir starts with 'oneDNN-', we rename it to mkldnn: + if submodule == 'oneDNN': + submodule = 'mkldnn' + # rename the file to 'mkldnn': + old_srcdir = srcdir + srcdir = srcdir.replace('oneDNN', 'mkldnn') + os.rename(os.path.join(self.builddir, old_srcdir), os.path.join(self.builddir, srcdir)) + + olddir = os.path.join(self.builddir, srcdir) + newdir = os.path.join(self.mxnet_src_dir, '3rdparty', submodule) + else: + olddir = os.path.join(self.builddir, srcdir) + newdir = os.path.join(self.mxnet_src_dir, submodule) + # first remove empty existing directory remove_dir(newdir) + try: shutil.move(olddir, newdir) except IOError as err: raise EasyBuildError("Failed to move %s to %s: %s", olddir, newdir, err) - # the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place. - newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core") - olddir = os.path.join(self.mxnet_src_dir, "dmlc-core") - remove_dir(newdir) - symlink(olddir, newdir) + if LooseVersion(self.version) < LooseVersion('1.0'): + # the nnvm submodules has dmlc-core as a submodule too. Let's put a symlink in place. + newdir = os.path.join(self.mxnet_src_dir, "nnvm", "dmlc-core") + olddir = os.path.join(self.mxnet_src_dir, "dmlc-core") + remove_dir(newdir) + symlink(olddir, newdir) def prepare_step(self, *args, **kwargs): """Prepare for building and installing MXNet.""" @@ -133,6 +161,9 @@ def configure_step(self): blas = "atlas" elif toolchain_blas == 'OpenBLAS': blas = "openblas" + elif toolchain_blas == 'FlexiBLAS': + blas = "flexiblas" + env.setvar('CFLAGS', "%s -lflexiblas" % os.getenv('CFLAGS')) elif toolchain_blas is None: raise EasyBuildError("No BLAS library found in the toolchain") @@ -145,8 +176,9 @@ def configure_step(self): def install_step(self): """Specify list of files to copy""" - self.cfg['files_to_copy'] = ['bin', 'include', 'lib', - (['dmlc-core/include/dmlc', 'nnvm/include/nnvm'], 'include')] + self.cfg['files_to_copy'] = ['bin', 'include', 'lib'] + if LooseVersion(self.version) < LooseVersion('1.0'): + self.cfg.update('files_to_copy', [(['dmlc-core/include/dmlc', 'nnvm/include/nnvm'], 'include')]) super(EB_MXNet, self).install_step() def extensions_step(self): @@ -159,6 +191,17 @@ def extensions_step(self): self.py_ext.run(unpack_src=False) self.py_ext.postrun() + if self.cfg['install_r_ext']: + # This is off by default, because it's been working in the old version of MXNet and now it's not. + # Also, from the website of MXNet, Python bindings seem to be the preferred ones so we'll focus on that. + self.install_r_ext() + else: + self.log.info("Skipping R extension installation") + + def install_r_ext(self): + """ + Also install R extension for MXNet. + """ # next up, the R bindings self.r_ext.src = os.path.join(self.mxnet_src_dir, "R-package") change_dir(self.r_ext.src) diff --git a/easybuild/easyblocks/o/opencv.py b/easybuild/easyblocks/o/opencv.py index 4b1511af1b..69a97a903d 100644 --- a/easybuild/easyblocks/o/opencv.py +++ b/easybuild/easyblocks/o/opencv.py @@ -164,8 +164,23 @@ def configure_step(self): # configure optimisation for CPU architecture # see https://github.com/opencv/opencv/wiki/CPU-optimizations-build-options if self.toolchain.options.get('optarch') and 'CPU_BASELINE' not in self.cfg['configopts']: + optimal_arch_option = self.toolchain.COMPILER_OPTIMAL_ARCHITECTURE_OPTION.get( + (self.toolchain.arch, self.toolchain.cpu_family), '') optarch = build_option('optarch') + optarch_detect = False if not optarch: + optarch_detect = True + elif isinstance(optarch, str): + optarch_detect = optimal_arch_option in optarch + elif isinstance(optarch, dict): + optarch_gcc = optarch.get('GCC') + optarch_intel = optarch.get('Intel') + gcc_detect = get_software_root('GCC') and (not optarch_gcc or optimal_arch_option in optarch_gcc) + intel_root = get_software_root('iccifort') or get_software_root('intel-compilers') + intel_detect = intel_root and (not optarch_intel or optimal_arch_option in optarch_intel) + optarch_detect = gcc_detect or intel_detect + + if optarch_detect: # optimize for host arch (let OpenCV detect it) self.cfg.update('configopts', '-DCPU_BASELINE=DETECT') elif optarch == OPTARCH_GENERIC: diff --git a/easybuild/easyblocks/o/openfoam.py b/easybuild/easyblocks/o/openfoam.py index 7c8717b455..68589af669 100644 --- a/easybuild/easyblocks/o/openfoam.py +++ b/easybuild/easyblocks/o/openfoam.py @@ -438,14 +438,14 @@ def sanity_check_step(self): if self.is_dot_org and self.looseversion >= LooseVersion('7'): tools.remove("buoyantBoussinesqSimpleFoam") tools.remove("sonicFoam") - # buoyantSimpleFoam replaced by buoyantFoam in versions 10+ - if self.is_dot_org and self.looseversion >= LooseVersion("10"): + # engineFoam replaced by reactingFoam and buoyantSimpleFoam replaced by buoyantFoam in version 10 + if self.is_dot_org and LooseVersion("10") <= self.looseversion: tools.remove("buoyantSimpleFoam") - tools.append("buoyantFoam") - # engineFoam replaced by reactingFoam in versions 10+ - if self.is_dot_org and self.looseversion >= LooseVersion("10"): tools.remove("engineFoam") - tools.append("reactingFoam") + # both removed in version 11 + if self.looseversion < LooseVersion("11"): + tools.append("buoyantFoam") + tools.append("reactingFoam") bins = [os.path.join(self.openfoamdir, "bin", x) for x in ["paraFoam"]] + \ [os.path.join(toolsdir, x) for x in tools] diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index dc510f723f..cf7c73151f 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -199,6 +199,10 @@ def sanity_check_step(self): # for PGI, correct pattern is "pgfortran" with mpif90 if expected['mpif90'] == 'pgf90': expected['mpif90'] = 'pgfortran' + # for Clang the pattern is always clang + for key in ['mpicxx', 'mpifort', 'mpif90']: + if expected[key] in ['clang++', 'flang']: + expected[key] = 'clang' custom_commands = ["%s --version | grep '%s'" % (key, expected[key]) for key in sorted(expected.keys())] diff --git a/easybuild/easyblocks/p/petsc.py b/easybuild/easyblocks/p/petsc.py index a335132c6b..23c18c169c 100644 --- a/easybuild/easyblocks/p/petsc.py +++ b/easybuild/easyblocks/p/petsc.py @@ -52,7 +52,7 @@ def __init__(self, *args, **kwargs): """Initialize PETSc specific variables.""" super(EB_PETSc, self).__init__(*args, **kwargs) - self.petsc_arch = "" + self.petsc_arch = self.cfg['petsc_arch'] self.petsc_subdir = "" self.prefix_inc = '' self.prefix_lib = '' @@ -61,8 +61,6 @@ def __init__(self, *args, **kwargs): self.with_python = False if self.cfg['sourceinstall']: - self.prefix_inc = self.petsc_subdir - self.prefix_lib = os.path.join(self.petsc_subdir, self.petsc_arch) self.build_in_installdir = True if LooseVersion(self.version) >= LooseVersion("3.9"): @@ -73,6 +71,7 @@ def extra_options(): """Add extra config options specific to PETSc.""" extra_vars = { 'sourceinstall': [False, "Indicates whether a source installation should be performed", CUSTOM], + 'petsc_arch': ['', "Custom PETSC_ARCH for sourceinstall", CUSTOM], 'shared_libs': [False, "Build shared libraries", CUSTOM], 'with_papi': [False, "Enable PAPI support", CUSTOM], 'papi_inc': ['/usr/include', "Path for PAPI include files", CUSTOM], @@ -285,6 +284,9 @@ def configure_step(self): self.cfg.update('buildopts', 'PETSC_DIR=%s' % self.cfg['start_dir']) if self.cfg['sourceinstall']: + if self.petsc_arch: + env.setvar('PETSC_ARCH', self.cfg['petsc_arch']) + # run configure without --prefix (required) cmd = "%s ./configure %s" % (self.cfg['preconfigopts'], self.cfg['configopts']) (out, _) = run_cmd(cmd, log_all=True, simple=False) @@ -306,7 +308,12 @@ def configure_step(self): else: raise EasyBuildError("Failed to determine PETSC_ARCH setting.") - self.petsc_subdir = '%s-%s' % (self.name.lower(), self.version) + self.petsc_subdir = self.name.lower() + self.prefix_lib = os.path.join(self.petsc_subdir, self.petsc_arch) + self.prefix_inc = os.path.join(self.petsc_subdir, self.petsc_arch) + self.prefix_bin = os.path.join(self.petsc_subdir, self.petsc_arch) + else: + self.petsc_subdir = '%s-%s' % (self.name.lower(), self.version) else: # old versions (< 3.x) diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 2783db4707..86da42a51a 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -114,6 +114,7 @@ def is_version_ok(version_range): ('USE_SYSTEM_FXDIV=1', None, '1.6.0:'), ('USE_SYSTEM_BENCHMARK=1', None, '1.6.0:'), # Google Benchmark ('USE_SYSTEM_ONNX=1', None, '1.6.0:'), + ('USE_SYSTEM_PYBIND11=1', 'pybind11', '1.10.0:'), ('USE_SYSTEM_XNNPACK=1', None, '1.6.0:'), ) return [(enable_opt, dep_name) for enable_opt, dep_name, version_range in available_libs @@ -279,6 +280,28 @@ def test_step(self): tests_out, tests_ec = test_result + # Show failed subtests to aid in debugging failures + # I.e. patterns like + # === FAIL: test_add_scalar_relu (quantization.core.test_quantized_op.TestQuantizedOps) === + # --- ERROR: test_all_to_all_group_cuda (__main__.TestDistBackendWithSpawn) --- + regex = r"^[=-]+\n(FAIL|ERROR): (test_.*?)\s\(.*\n[=-]+\n" + failed_test_cases = re.findall(regex, tests_out, re.M) + # And patterns like: + # FAILED test_ops_gradients.py::TestGradientsCPU::test_fn_grad_linalg_det_singular_cpu_complex128 - [snip] + regex = r"^(FAILED) \w+\.py.*::(test_.*?) - " + failed_test_cases.extend(re.findall(regex, tests_out, re.M)) + if failed_test_cases: + errored_test_cases = sorted(m[1] for m in failed_test_cases if m[0] == 'ERROR') + failed_test_cases = sorted(m[1] for m in failed_test_cases if m[0] != 'ERROR') + msg = [] + if errored_test_cases: + msg.append("Found %d individual tests that exited with an error: %s" + % (len(errored_test_cases), ', '.join(errored_test_cases))) + if failed_test_cases: + msg.append("Found %d individual tests with failed assertions: %s" + % (len(failed_test_cases), ', '.join(failed_test_cases))) + self.log.warning("\n".join(msg)) + def get_count_for_pattern(regex, text): """Match the regexp containing a single group and return the integer value of the matched group. Return zero if no or more than 1 match was found and warn for the latter case @@ -307,7 +330,7 @@ def get_count_for_pattern(regex, text): # test_fx failed! regex = (r"^Ran (?P[0-9]+) tests.*$\n\n" r"FAILED \((?P.*)\)$\n" - r"(?:^(?:(?!failed!).)*$\n)*" + r"(?:^(?:(?!failed!).)*$\n){0,5}" r"(?P.*) failed!(?: Received signal: \w+)?\s*$") for m in re.finditer(regex, tests_out, re.M): @@ -323,7 +346,17 @@ def get_count_for_pattern(regex, text): # Grep for patterns like: # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 3.43s ===================== - regex = r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* =+$\n(?P.*) failed!$" + # test_quantization failed! + # OR: + # ===================== 2 failed, 128 passed, 2 skipped, 2 warnings in 63.43s (01:03:43) ========= + # FINISHED PRINTING LOG FILE + # test_quantization failed! + + regex = ( + r"^=+ (?P.*) in [0-9]+\.*[0-9]*[a-zA-Z]* (\([0-9]+:[0-9]+:[0-9]+\) )?=+$\n" + r"(?:^(?:(?!failed!).)*$\n){0,5}" + r"(?P.*) failed!$" + ) for m in re.finditer(regex, tests_out, re.M): # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings' diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index 823116e1e8..fa82d0a866 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Jasper Grimm (University of York) +@author: Sebastian Achilles (Juelich Supercomputing Centre) """ import os import tempfile @@ -95,11 +96,21 @@ def __init__(self, *args, **kwargs): # see https://github.com/easybuilders/easybuild-easyblocks/issues/2237 self.testcmd = "cd .. && %(python)s -c 'import numpy; import scipy; scipy.test(verbose=2)'" else: - self.testcmd = " && ".join([ - "cd ..", - "touch %(srcdir)s/.coveragerc", - "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", - ]) + if LooseVersion(self.version) >= LooseVersion('1.11'): + self.testcmd = " && ".join([ + "cd ..", + # note: beware of adding --parallel here to speed up running the tests: + # in some contexts the test suite could hang because pytest-xdist doesn't deal well with cgroups + # cfr. https://github.com/pytest-dev/pytest-xdist/issues/658 + "%(python)s %(srcdir)s/dev.py --no-build --install-prefix %(installdir)s test -v ", + ]) + else: + self.testcmd = " && ".join([ + "cd ..", + "touch %(srcdir)s/.coveragerc", + "%(python)s %(srcdir)s/runtests.py -v --no-build --parallel %(parallel)s", + ]) + if self.cfg['enable_slow_tests']: self.testcmd += " -m full " @@ -165,10 +176,17 @@ def test_step(self): change_dir(tmp_builddir) # reconfigure (to update prefix), and install to tmpdir - MesonNinja.configure_step(self, cmd_prefix=tmp_installdir) + orig_builddir = self.builddir + orig_installdir = self.installdir + self.builddir = tmp_builddir + self.installdir = tmp_installdir + MesonNinja.configure_step(self) MesonNinja.install_step(self) + self.builddir = orig_builddir + self.installdir = orig_installdir + MesonNinja.configure_step(self) - tmp_pylibdir = [os.path.join(tmp_installdir, det_pylibdir())] + tmp_pylibdir = os.path.join(tmp_installdir, det_pylibdir()) self.prepare_python() self.cfg['pretestopts'] = " && ".join([ @@ -181,6 +199,7 @@ def test_step(self): self.cfg['runtest'] = self.testcmd % { 'python': self.python_cmd, 'srcdir': self.cfg['start_dir'], + 'installdir': tmp_installdir, 'parallel': self.cfg['parallel'], } @@ -190,6 +209,7 @@ def test_step(self): self.testcmd = self.testcmd % { 'python': '%(python)s', 'srcdir': self.cfg['start_dir'], + 'installdir': '', 'parallel': self.cfg['parallel'], } FortranPythonPackage.test_step(self) diff --git a/easybuild/easyblocks/s/score_p.py b/easybuild/easyblocks/s/score_p.py index 410fea4cb7..1c473d2099 100644 --- a/easybuild/easyblocks/s/score_p.py +++ b/easybuild/easyblocks/s/score_p.py @@ -30,10 +30,13 @@ @author: Bernd Mohr (Juelich Supercomputing Centre) @author: Markus Geimer (Juelich Supercomputing Centre) @author: Alexander Grund (TU Dresden) +@author: Christian Feld (Juelich Supercomputing Centre) """ import easybuild.tools.toolchain as toolchain from easybuild.easyblocks.generic.configuremake import ConfigureMake +from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError +from easybuild.tools.environment import unset_env_vars from easybuild.tools.modules import get_software_root, get_software_libdir @@ -45,6 +48,10 @@ class EB_Score_minus_P(ConfigureMake): def configure_step(self, *args, **kwargs): """Configure the build, set configure options for compiler, MPI and dependencies.""" + # Remove some settings from the environment, as they interfere with + # Score-P's configure magic... + unset_env_vars(['CPPFLAGS', 'LDFLAGS', 'LIBS']) + # On non-cross-compile platforms, specify compiler and MPI suite explicitly. This is much quicker and safer # than autodetection. In Score-P build-system terms, the following platforms are considered cross-compile # architectures: @@ -56,16 +63,27 @@ def configure_step(self, *args, **kwargs): # Of those, only Cray is supported right now. tc_fam = self.toolchain.toolchain_family() if tc_fam != toolchain.CRAYPE: - # --with-nocross-compiler-suite=(gcc|ibm|intel|pgi|studio) + # since 2022/12 releases: --with-nocross-compiler-suite=(gcc|ibm|intel|oneapi|nvhpc|pgi|clang|aocc|amdclang) comp_opts = { # assume that system toolchain uses a system-provided GCC toolchain.SYSTEM: 'gcc', toolchain.GCC: 'gcc', toolchain.IBMCOMP: 'ibm', toolchain.INTELCOMP: 'intel', - toolchain.NVHPC: 'pgi', + toolchain.NVHPC: 'nvhpc', toolchain.PGI: 'pgi', } + nvhpc_since = { + 'Score-P': '8.0', + 'Scalasca': '2.6.1', + 'OTF2': '3.0.2', + 'CubeWriter': '4.8', + 'CubeLib': '4.8', + 'CubeGUI': '4.8', + } + if LooseVersion(self.version) < LooseVersion(nvhpc_since.get(self.name, '0')): + comp_opts[toolchain.NVHPC] = 'pgi' + comp_fam = self.toolchain.comp_family() if comp_fam in comp_opts: self.cfg.update('configopts', "--with-nocross-compiler-suite=%s" % comp_opts[comp_fam]) diff --git a/easybuild/easyblocks/s/star_ccm.py b/easybuild/easyblocks/s/star_ccm.py index 7f981e9b36..b4993bb23a 100644 --- a/easybuild/easyblocks/s/star_ccm.py +++ b/easybuild/easyblocks/s/star_ccm.py @@ -65,7 +65,8 @@ def install_step(self): # depending of the target filesystem the check for available disk space may fail, so disable it; # note that this makes the installer exit with non-zero exit code... - # env.setvar('CHECK_DISK_SPACE', 'OFF') + env.setvar('CHECK_DISK_SPACE', 'OFF') + env.setvar('IATEMPDIR', tempfile.mkdtemp()) cmd = ' '.join([ @@ -77,7 +78,10 @@ def install_step(self): "-DADDSYSTEMPATH=false", self.cfg['installopts'], ]) - run_cmd(cmd, log_all=True, simple=True) + + # ignore exit code of command, since there's always a non-zero exit if $CHECK_DISK_SPACE is set to OFF; + # rely on sanity check to catch problems with the installation + run_cmd(cmd, log_all=False, log_ok=False, simple=False) def find_starccm_subdirs(self): """Determine subdirectory of install directory in which STAR-CCM+ was installed.""" diff --git a/easybuild/easyblocks/s/sympy.py b/easybuild/easyblocks/s/sympy.py new file mode 100644 index 0000000000..15e5186231 --- /dev/null +++ b/easybuild/easyblocks/s/sympy.py @@ -0,0 +1,81 @@ +## +# Copyright 2009-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for sympy, implemented as an easyblock + +@author: Caspar van Leeuwen (SURF) +@author: Kenneth Hoste (HPC-UGent) +""" + +import os +import tempfile + +from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_pylibdir + + +class EB_sympy(PythonPackage): + """Custom easyblock for installing the sympy Python package.""" + + @staticmethod + def extra_options(extra_vars=None): + """Customize default value for easyconfig parameters for sympy""" + extra_vars = PythonPackage.extra_options(extra_vars=extra_vars) + extra_vars['use_pip'][0] = True + extra_vars['sanity_pip_check'][0] = True + extra_vars['download_dep_fail'][0] = True + return extra_vars + + def test_step(self): + """Custom test step for sympy""" + + self.cfg['runtest'] = "python setup.py test" + + # we need to make sure that the temporary directory being used is not a symlinked path; + # see https://github.com/easybuilders/easybuild-easyconfigs/issues/17593 + original_tmpdir = tempfile.gettempdir() + tempfile.tempdir = os.path.realpath(tempfile.gettempdir()) + msg = "Temporary directory set to resolved path %s (was %s), " % (original_tmpdir, tempfile.gettempdir()) + msg += "to avoid failing tests due to the temporary directory being a symlinked path..." + self.log.info(msg) + + super(EB_sympy, self).test_step(self) + + # restore original temporary directory + tempfile.tempdir = original_tmpdir + self.log.debug("Temporary directory restored to %s", tempfile.gettempdir()) + + def sanity_check_step(self, *args, **kwargs): + """Custom sanity check for sympy.""" + + # can't use self.pylibdir here, need to determine path on the fly using currently active 'python' command; + # this is important for sympy installations for multiple Python version (via multi_deps) + custom_paths = { + 'files': [os.path.join('bin', 'isympy')], + 'dirs': [os.path.join(det_pylibdir(), 'sympy')], + } + + custom_commands = ["isympy --help"] + + return super(EB_sympy, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands) diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index d63a4bc3f6..de9b97296a 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -146,7 +146,7 @@ def is_version_ok(version_range): ('libjpeg-turbo', '2.2.0:'): 'libjpeg_turbo', ('libpng', '2.0.0:2.1.0'): 'png_archive', ('libpng', '2.1.0:'): 'png', - ('LMDB', '2.0.0:'): 'lmdb', + ('LMDB', '2.0.0:2.13.0'): 'lmdb', ('NASM', '2.0.0:'): 'nasm', ('nsync', '2.0.0:'): 'nsync', ('PCRE', '2.0.0:2.6.0'): 'pcre', @@ -161,11 +161,11 @@ def is_version_ok(version_range): # Software recognized by TF but which is always disabled (usually because no EC is known) # Format: : unused_system_libs = { - 'boringssl': '2.0.0:', + 'boringssl': '2.0.0:', # Implied by cURL and existence of OpenSSL anywhere in the dependency chain 'com_github_googleapis_googleapis': '2.0.0:2.5.0', 'com_github_googlecloudplatform_google_cloud_cpp': '2.0.0:', # Not used due to $TF_NEED_GCP=0 'com_github_grpc_grpc': '2.2.0:', - 'com_googlesource_code_re2': '2.0.0:', + 'com_googlesource_code_re2': '2.0.0:', # Requires the RE2 version with Abseil (or 2023-06-01+) 'grpc': '2.0.0:2.2.0', } # Python packages installed as extensions or in the Python module @@ -335,10 +335,10 @@ def verify_system_libs_info(self): msg = 'Values for $TF_SYSTEM_LIBS in the TensorFlow EasyBlock are incomplete.\n' if missing_libs: # Libs available according to TF sources but not listed in this EasyBlock - msg += 'Missing entries for $TF_SYSTEM_LIBS: %s\n' % missing_libs + msg += 'Missing entries for $TF_SYSTEM_LIBS: %s\n' % sorted(missing_libs) if unknown_libs: # Libs listed in this EasyBlock but not present in the TF sources -> Removed? - msg += 'Unrecognized entries for $TF_SYSTEM_LIBS: %s\n' % unknown_libs + msg += 'Unrecognized entries for $TF_SYSTEM_LIBS: %s\n' % sorted(unknown_libs) msg += 'The EasyBlock needs to be updated to fully work with TensorFlow version %s' % self.version if build_option('strict') == run.ERROR: raise EasyBuildError(msg) @@ -416,7 +416,7 @@ def get_system_libs(self): incpath = os.path.join(openssl_root, 'include') if os.path.exists(incpath): cpaths.append(incpath) - libpath = get_software_libdir(dep_name) + libpath = get_software_libdir('OpenSSL') if libpath: libpaths.append(os.path.join(openssl_root, libpath)) @@ -457,10 +457,10 @@ def configure_step(self): bazel_max = 64 if get_bazel_version() < '3.0.0' else 128 self.cfg['parallel'] = min(self.cfg['parallel'], bazel_max) - binutils_root = get_software_root('binutils') - if not binutils_root: - raise EasyBuildError("Failed to determine installation prefix for binutils") - self.binutils_bin_path = os.path.join(binutils_root, 'bin') + # determine location where binutils' ld command is installed + # note that this may be an RPATH wrapper script (when EasyBuild is configured with --rpath) + ld_path = which('ld') + self.binutils_bin_path = os.path.dirname(ld_path) # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that # might conflict with dependencies on the system and/or installed with EB. For example: protobuf diff --git a/easybuild/easyblocks/t/tensorflow_compression.py b/easybuild/easyblocks/t/tensorflow_compression.py new file mode 100644 index 0000000000..c13acb7fca --- /dev/null +++ b/easybuild/easyblocks/t/tensorflow_compression.py @@ -0,0 +1,160 @@ +## +# Copyright 2017-2023 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be), +# Flemish Research Foundation (FWO) (http://www.fwo.be/en) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# https://github.com/easybuilders/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +EasyBuild support for building and installing tensorflow-compresssion, implemented as an easyblock + +@author: Ake Sandgren (Umea University) +""" +import glob +import os + +import easybuild.tools.environment as env +from easybuild.easyblocks.generic.pythonpackage import PythonPackage +from easybuild.tools import LooseVersion +from easybuild.tools.modules import get_software_version +from easybuild.tools.run import run_cmd + + +class EB_tensorflow_minus_compression(PythonPackage): + + def setup_build_dirs(self): + """Setup temporary build directories""" + # This is either the builddir (for standalone builds) or the extension sub folder when TFC is an extension + # Either way this folder only contains the folder with the sources and hence we can use fixed names + # for the subfolders + parent_dir = os.path.dirname(self.start_dir) + # Path where Bazel will store its output, build artefacts etc. + self.output_user_root_dir = os.path.join(parent_dir, 'bazel-root') + # Folder where wrapper binaries can be placed, where required. TODO: Replace by --action_env cmds + self.wrapper_dir = os.path.join(parent_dir, 'wrapper_bin') + + def configure_step(self): + """Custom configuration procedure for TensorFlow-Compression.""" + + self.setup_build_dirs() + + # Options passed to the target (build/test), e.g. --config arguments + self.target_opts = [] + + def build_step(self): + """Custom build procedure for TensorFlow-Compression.""" + + bazel_version = get_software_version('Bazel') + + # Options passed to the bazel command + self.bazel_opts = [ + '--output_user_root=%s' % self.output_user_root_dir, + ] + + # Environment variables and values needed for Bazel actions. + action_env = {} + # A value of None is interpreted as using the invoking environments value + INHERIT = None # For better readability + + if self.toolchain.options.get('debug', None): + self.target_opts.append('--strip=never') + self.target_opts.append('--compilation_mode=dbg') + + for flag in os.getenv('CFLAGS', '').split(' '): + self.target_opts.append('--copt="%s"' % flag) + + # make Bazel print full command line + make it verbose on failures + # https://docs.bazel.build/versions/master/user-manual.html#flag--subcommands + # https://docs.bazel.build/versions/master/user-manual.html#flag--verbose_failures + self.target_opts.extend(['--subcommands', '--verbose_failures']) + + self.target_opts.append('--jobs=%s' % self.cfg['parallel']) + + # include install location of Python packages in $PYTHONPATH, + # and specify that value of $PYTHONPATH should be passed down into Bazel build environment, + # this is required to make sure that Python packages included as extensions are found at build time, + # see also https://github.com/tensorflow/tensorflow/issues/22395 + pythonpath = os.getenv('PYTHONPATH', '') + env.setvar('PYTHONPATH', os.pathsep.join([os.path.join(self.installdir, self.pylibdir), pythonpath])) + + # Make TFC find our modules. LD_LIBRARY_PATH gets automatically added + action_env['CPATH'] = INHERIT + action_env['LIBRARY_PATH'] = INHERIT + action_env['PYTHONPATH'] = INHERIT + # Also export $EBPYTHONPREFIXES to handle the multi-deps python setup + # See https://github.com/easybuilders/easybuild-easyblocks/pull/1664 + if 'EBPYTHONPREFIXES' in os.environ: + action_env['EBPYTHONPREFIXES'] = INHERIT + + # Ignore user environment for Python + action_env['PYTHONNOUSERSITE'] = '1' + + # Use the same configuration (i.e. environment) for compiling and using host tools + # This means that our action_envs are (almost) always passed + # Fully removed in Bazel 6.0 and limited effect after at least 3.7 (see --host_action_env) + if LooseVersion(bazel_version) < LooseVersion('6.0.0'): + self.target_opts.append('--distinct_host_configuration=false') + + for key, value in sorted(action_env.items()): + option = key + if value is not None: + option += "='%s'" % value + + self.target_opts.append('--action_env=' + option) + if bazel_version >= '3.7.0': + # Since Bazel 3.7 action_env only applies to the "target" environment, not the "host" environment + # As we are not cross-compiling we need both be the same -> Duplicate the setting to host_action_env + # See https://github.com/bazelbuild/bazel/commit/a463d9095386b22c121d20957222dbb44caef7d4 + self.target_opts.append('--host_action_env=' + option) + + # Compose final command + cmd = ( + [self.cfg['prebuildopts']] + + ['bazel'] + + self.bazel_opts + + ['build'] + + self.target_opts + + [self.cfg['buildopts']] + # specify target of the build command as last argument + + [':build_pip_pkg'] + ) + + run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True) + + # run generated 'build_pip_pkg' script to build the .whl + cmd = ( + 'python', + 'build_pip_pkg.py', + 'bazel-bin/build_pip_pkg.runfiles/tensorflow_compression', + self.builddir, + self.version + ) + run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True) + + def install_step(self): + """Custom install procedure for TensorFlow-Compression.""" + + whl_paths = glob.glob(os.path.join(self.builddir, 'tensorflow_compression-%s-*.whl' % self.version)) + if not whl_paths: + whl_paths = glob.glob(os.path.join(self.builddir, 'tensorflow_compression-*.whl')) + if len(whl_paths) == 1: + self.cfg['install_src'] = whl_paths[0] + + super(EB_tensorflow_minus_compression, self).install_step() diff --git a/test/easyblocks/easyblock_specific.py b/test/easyblocks/easyblock_specific.py index 4e4d10b4d2..d9e90408ee 100644 --- a/test/easyblocks/easyblock_specific.py +++ b/test/easyblocks/easyblock_specific.py @@ -37,6 +37,7 @@ from test.easyblocks.module import cleanup import easybuild.tools.options as eboptions +import easybuild.easyblocks.generic.pythonpackage as pythonpackage from easybuild.base.testing import TestCase from easybuild.easyblocks.generic.cmakemake import det_cmake_version from easybuild.easyblocks.generic.toolchain import Toolchain @@ -46,7 +47,7 @@ from easybuild.tools.build_log import EasyBuildError from easybuild.tools.config import GENERAL_CLASS, get_module_syntax from easybuild.tools.environment import modify_env -from easybuild.tools.filetools import adjust_permissions, remove_dir, write_file +from easybuild.tools.filetools import adjust_permissions, mkdir, move_file, remove_dir, symlink, write_file from easybuild.tools.modules import modules_tool from easybuild.tools.options import set_tmpdir from easybuild.tools.py2vs3 import StringIO @@ -265,6 +266,99 @@ def test_det_cmake_version(self): """)) self.assertEqual(det_cmake_version(), '1.2.3-rc4') + def test_det_py_install_scheme(self): + """Test det_py_install_scheme function provided by PythonPackage easyblock.""" + res = pythonpackage.det_py_install_scheme(sys.executable) + self.assertTrue(isinstance(res, str)) + + # symlink currently used python command to 'python', so we can also test det_py_install_scheme with default; + # this is required because 'python' command may not be available + symlink(sys.executable, os.path.join(self.tmpdir, 'python')) + os.environ['PATH'] = '%s:%s' % (self.tmpdir, os.getenv('PATH')) + + res = pythonpackage.det_py_install_scheme() + self.assertTrue(isinstance(res, str)) + + def test_handle_local_py_install_scheme(self): + """Test handle_local_py_install_scheme function provided by PythonPackage easyblock.""" + + # test with empty dir, should be fine + pythonpackage.handle_local_py_install_scheme(self.tmpdir) + self.assertEqual(os.listdir(self.tmpdir), []) + + # create normal structure (no 'local' subdir), shouldn't cause trouble + bindir = os.path.join(self.tmpdir, 'bin') + mkdir(bindir) + write_file(os.path.join(bindir, 'test'), 'test') + libdir = os.path.join(self.tmpdir, 'lib') + pyshortver = '.'.join(str(x) for x in sys.version_info[:2]) + pylibdir = os.path.join(libdir, 'python' + pyshortver, 'site-packages') + mkdir(pylibdir, parents=True) + write_file(os.path.join(pylibdir, 'test.py'), "import os") + + pythonpackage.handle_local_py_install_scheme(self.tmpdir) + self.assertEqual(sorted(os.listdir(self.tmpdir)), ['bin', 'lib']) + + # move bin + lib into local/, check whether expected symlinks are created + local_subdir = os.path.join(self.tmpdir, 'local') + mkdir(local_subdir) + for subdir in (bindir, libdir): + move_file(subdir, os.path.join(local_subdir, os.path.basename(subdir))) + self.assertEqual(os.listdir(self.tmpdir), ['local']) + + pythonpackage.handle_local_py_install_scheme(self.tmpdir) + self.assertEqual(sorted(os.listdir(self.tmpdir)), ['bin', 'lib', 'local']) + self.assertTrue(os.path.islink(bindir) and os.path.samefile(bindir, os.path.join(local_subdir, 'bin'))) + self.assertTrue(os.path.islink(libdir) and os.path.samefile(libdir, os.path.join(local_subdir, 'lib'))) + self.assertTrue(os.path.exists(os.path.join(bindir, 'test'))) + local_test_py = os.path.join(libdir, 'python' + pyshortver, 'site-packages', 'test.py') + self.assertTrue(os.path.exists(local_test_py)) + + def test_symlink_dist_site_packages(self): + """Test symlink_dist_site_packages provided by PythonPackage easyblock.""" + pyshortver = '.'.join(str(x) for x in sys.version_info[:2]) + pylibdir_lib_dist = os.path.join('lib', 'python' + pyshortver, 'dist-packages') + pylibdir_lib64_site = os.path.join('lib64', 'python' + pyshortver, 'site-packages') + pylibdirs = [pylibdir_lib_dist, pylibdir_lib64_site] + + # first test on empty dir + pythonpackage.symlink_dist_site_packages(self.tmpdir, pylibdirs) + self.assertEqual(os.listdir(self.tmpdir), []) + + # check intended usage: dist-packages exists, site-packages doesn't => symlink created + lib64_site_path = os.path.join(self.tmpdir, pylibdir_lib_dist) + mkdir(os.path.join(self.tmpdir, pylibdir_lib_dist), parents=True) + # also create (empty) site-packages directory, which should get replaced by a symlink to dist-packages + mkdir(os.path.join(self.tmpdir, os.path.dirname(pylibdir_lib_dist), 'site-packages'), parents=True) + + lib64_site_path = os.path.join(self.tmpdir, pylibdir_lib64_site) + mkdir(lib64_site_path, parents=True) + # populate site-packages under lib64, because it'll get removed if empty + write_file(os.path.join(lib64_site_path, 'test.py'), "import os") + + pythonpackage.symlink_dist_site_packages(self.tmpdir, pylibdirs) + + # check for expected directory structure + self.assertEqual(sorted(os.listdir(self.tmpdir)), ['lib', 'lib64']) + path = os.path.join(self.tmpdir, 'lib') + self.assertEqual(os.listdir(path), ['python' + pyshortver]) + path = os.path.join(path, 'python' + pyshortver) + self.assertEqual(sorted(os.listdir(path)), ['dist-packages', 'site-packages']) + + # check for site-packages -> dist-packages symlink + dist_pkgs = os.path.join(path, 'dist-packages') + self.assertTrue(os.path.isdir(dist_pkgs)) + self.assertFalse(os.path.islink(dist_pkgs)) + site_pkgs = os.path.join(path, 'site-packages') + self.assertTrue(os.path.isdir(site_pkgs)) + self.assertTrue(os.path.islink(site_pkgs)) + + # if (non-empty) site-packages dir was there, no changes were made + lib64_path = os.path.dirname(lib64_site_path) + self.assertEqual(sorted(os.listdir(lib64_path)), ['site-packages']) + self.assertTrue(os.path.isdir(lib64_site_path)) + self.assertFalse(os.path.islink(lib64_site_path)) + def suite(): """Return all easyblock-specific tests."""