Merge pull request #2277 from easybuilders/4.3.x

release EasyBuild v4.3.2

Author: migueldiascosta

RELEASE_NOTES

@@ -3,7 +3,49 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
-The latest version of easybuild-easyblocks provides 225 software-specific easyblocks and 37 generic easyblocks.
+The latest version of easybuild-easyblocks provides 227 software-specific easyblocks and 37 generic easyblocks.
+
+v4.3.2 (December 10th 2020)
+---------------------------
+
+update/bugfix release
+
+- 2 new software-specific easyblocks:
+  - code-server (#2255), Metagenome-Atlas (#2219)
+- minor enhancements, including:
+  - add -fallow-argument-mismatch option when building CP2K 7.1 or older with GCC 10.x (#2223)
+  - update TensorFlow easyblock for upcoming TensorFlow 2.4 (#2225)
+  - add support for building Clang with OpenMP offload support (#2229)
+  - enhance OpenMPI easyblock to catch any --with-ucx* configure options (#2230)
+  - take into account preinstallopts and installopts in custom easyblock for NCL (#2234)
+  - add support for withnvptx easyconfig parameter, to enable GPU offloading, in GCC easyblock (#2235)
+  - take into account versions like '4.x' in OpenFOAM easyblock (#2239)
+  - also add 'bin' subdir to $PATH when installing a Python package (#2244)
+- various bug fixes, including:
+  - fix two bugs in GROMACS easyblock when using GCC & MKL for FFT and BLAS/LAPACK (#2212)
+  - fix version check in Qt5 easyblock w.r.t. disabling features on old Linux kernel versions (#2220)
+  - always define $FCCPP in QuantumESPRESSO easyblock (not just when using Intel compilers) (#2221)
+  - allow wxPython to be installed as an extension (#2227, #2275)
+  - only configure Python with --enable-optimizations when compiling Python with (recent) GCC compiler (#2228)
+  - fix sanity check for Boost MT libraries (#2231)
+  - fix hardcoded path in NVHPC easyblock to support multiple architectures (#2233)
+  - fix CPASSERT test faults on RHEL8 in CP2K easyblock (#2236)
+  - stop silently ignoring failing numpy tests, but include support for ignoring (failing) numpy tests (#2238, #2271)
+  - append to module guesses in easyblocks for Chapel, icc, imkl and impi (rather than overwriting guesses from parent easyblock) (#2242)
+  - weed out duplicates when determining paths to include-fixed subdirectory in GCC easyblock (#2245)
+  - prepend all hardcoded /usr/* paths with sysroot in Python's setup.py installation script (#2246)
+  - don't try to patch newer versions of Bazel where the patches won't apply (#2249)
+  - fix setting of $RUNPARALLEL in HDF5 easyblock (#2250)
+  - move --build and --host logic to run_configure_cmd in GCC easyblock (#2252)
+  - set $UCX_TLS in module for impi installation on top of UCX, and allow it to be overwritten in impi easyconfig file (#2253, #2258)
+  - enhance PyTorch easyblock to ensure it finds MKL (via $MKLROOT) (#2257)
+  - use integer division to determine number MPI ranks to use in WRF test step (#2266)
+  - also specify locincpth and glibpth configure options for Perl based on --sysroot (#2268)
+- other changes, including:
+  - add link to GCC mailing list thread confirming that binutils should not be configured with --with-sysroot=$EASYBUILD_SYSROOT when GCC is being configured like that (#2215)
+  - pass paths to patch one by one to apply_regex_substitutions in GCC easyblock when --sysroot is set (#2217)
+    - workaround for regression in apply_regex_substitutions introduced in EasyBuild v4.3.1 (which was fixed for v4.3.2)
+
 
 v4.3.1 (October 29th 2020)
 --------------------------

easybuild/easyblocks/__init__.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.3.1')
+VERSION = LooseVersion('4.3.2')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/b/bazel.py

@@ -40,16 +40,32 @@
 class EB_Bazel(EasyBlock):
     """Support for building/installing Bazel."""
 
-    def configure_step(self):
-        """Custom configuration procedure for Bazel."""
-
+    def fixup_hardcoded_paths(self):
+        """Patch out hard coded paths to compiler and binutils tools"""
         binutils_root = get_software_root('binutils')
         gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
         gcc_ver = get_software_version('GCCcore') or get_software_version('GCC')
 
         # only patch Bazel scripts if binutils & GCC installation prefix could be determined
-        if binutils_root and gcc_root:
+        if not binutils_root or not gcc_root:
+            self.log.info("Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
+            return
+
+        # replace hardcoded paths in (unix_)cc_configure.bzl
+        # hard-coded paths in (unix_)cc_configure.bzl were removed in 0.19.0
+        if LooseVersion(self.version) < LooseVersion('0.19.0'):
+            regex_subs = [
+                (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
+                (r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
+            ]
+            for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
+                filepath = os.path.join('tools', 'cpp', conf_bzl)
+                if os.path.exists(filepath):
+                    apply_regex_substitutions(filepath, regex_subs)
 
+        # replace hardcoded paths in CROSSTOOL
+        # CROSSTOOL script is no longer there in Bazel 0.24.0
+        if LooseVersion(self.version) < LooseVersion('0.24.0'):
             res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
             if res and len(res) == 1:
                 gcc_lib_inc = res[0]
@@ -65,36 +81,27 @@ def configure_step(self):
             if not os.path.exists(gcc_cplusplus_inc):
                 raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc)
 
-            # replace hardcoded paths in CROSSTOOL
-
-            # CROSSTOOL script is no longer there in Bazel 0.24.0
-            if LooseVersion(self.version) < LooseVersion('0.24.0'):
-                regex_subs = [
-                    (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
-                    (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc),
-                    (r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_bis),
-                    (r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc),
-                ]
-                for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
-                    path = which(tool)
-                    if path:
-                        regex_subs.append((os.path.join('/usr', 'bin', tool), path))
-                    else:
-                        raise EasyBuildError("Failed to determine path to '%s'", tool)
-
-                apply_regex_substitutions(os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
-
-            # replace hardcoded paths in (unix_)cc_configure.bzl
             regex_subs = [
                 (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
-                (r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
+                (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc),
+                (r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_bis),
+                (r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc),
             ]
-            for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
-                filepath = os.path.join('tools', 'cpp', conf_bzl)
-                if os.path.exists(filepath):
-                    apply_regex_substitutions(filepath, regex_subs)
-        else:
-            self.log.info("Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
+            for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
+                path = which(tool)
+                if path:
+                    regex_subs.append((os.path.join('/usr', 'bin', tool), path))
+                else:
+                    raise EasyBuildError("Failed to determine path to '%s'", tool)
+
+            apply_regex_substitutions(os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
+
+    def configure_step(self):
+        """Custom configuration procedure for Bazel."""
+
+        # Last instance of hardcoded paths was removed in 0.24.0
+        if LooseVersion(self.version) < LooseVersion('0.24.0'):
+            self.fixup_hardcoded_paths()
 
         # enable building in parallel
         bazel_args = '--jobs=%d' % self.cfg['parallel']

easybuild/easyblocks/b/binutils.py

@@ -125,6 +125,7 @@ def configure_step(self):
         # we should *not* use the value of the EasyBuild --sysroot configuration option here,
         # since that leads to weird errors where the sysroot path is duplicated, like:
         #   /bin/ld.gold: error: cannot open /<sysroot>/<sysroot>/lib64/libc.so.6: No such file or directory
+        # (see also https://gcc.gnu.org/legacy-ml/gcc-help/2006-08/msg00212.html)
         self.cfg.update('configopts', '--with-sysroot=/')
 
         # build both static and shared libraries for recent binutils versions (default is only static)

easybuild/easyblocks/b/boost.py

@@ -276,11 +276,16 @@ def sanity_check_step(self):
                 suffix = ''
             custom_paths['files'].append(os.path.join('lib', 'libboost_python%s.%s' % (suffix, shlib_ext)))
 
+        lib_mt_suffix = '-mt'
+        # MT libraries gained an extra suffix from v1.69.0 onwards
+        if LooseVersion(self.version) >= LooseVersion("1.69.0"):
+            lib_mt_suffix += '-x64'
+
         if self.cfg['boost_multi_thread']:
-            custom_paths['files'].append(os.path.join('lib', 'libboost_thread-mt.%s' % shlib_ext))
+            custom_paths['files'].append(os.path.join('lib', 'libboost_thread%s.%s' % (lib_mt_suffix, shlib_ext)))
 
         if self.cfg['boost_mpi'] and self.cfg['boost_multi_thread']:
-            custom_paths['files'].append(os.path.join('lib', 'libboost_mpi-mt.%s' % shlib_ext))
+            custom_paths['files'].append(os.path.join('lib', 'libboost_mpi%s.%s' % (lib_mt_suffix, shlib_ext)))
 
         super(EB_Boost, self).sanity_check_step(custom_paths=custom_paths)
 

easybuild/easyblocks/c/chapel.py

@@ -55,7 +55,11 @@ def make_module_req_guess(self):
         """
         A dictionary of possible directories to look for; this is needed since bin/linux64 of chapel is non standard
         """
-        return {
+        guesses = super(EB_Chapel, self).make_module_req_guess()
+        lib_paths = ['lib', 'lib/linux64', 'lib64']
+        guesses.update({
             'PATH': ['bin', 'bin/linux64', 'bin64'],
-            'LD_LIBRARY_PATH': ['lib', 'lib/linux64', 'lib64'],
-        }
+            'LD_LIBRARY_PATH': lib_paths,
+            'LIBRARY_PATH': lib_paths,
+        })
+        return guesses

easybuild/easyblocks/c/chimera.py

@@ -85,6 +85,8 @@ def extra_options():
             'skip_all_tests': [False, "Skip running of tests", CUSTOM],
             # The sanitizer tests often fail on HPC systems due to the 'weird' environment.
             'skip_sanitizer_tests': [True, "Do not run the sanitizer tests", CUSTOM],
+            'default_cuda_capability': [None, "Default CUDA capability specified for clang, e.g. '7.5'", CUSTOM],
+            'cuda_compute_capabilities': [[], "List of CUDA compute capabilities to build with", CUSTOM],
         })
         # disable regular out-of-source build, too simplistic for Clang to work
         extra_vars['separate_build_dir'][0] = False
@@ -282,9 +284,28 @@ def configure_step(self):
         if self.cfg['parallel']:
             self.make_parallel_opts = "-j %s" % self.cfg['parallel']
 
-        # If we don't want to build with CUDA (not in dependencies) trick CMakes FindCUDA module into
-        # not finding it by using the environment variable which is used as-is and later checked
-        # for a falsy value when determining wether CUDA was found
+        # If 'NVPTX' is in the build targets we assume the user would like OpenMP offload support as well
+        if 'NVPTX' in build_targets:
+            # list of CUDA compute capabilities to use can be specifed in two ways (where (2) overrules (1)):
+            # (1) in the easyconfig file, via the custom cuda_compute_capabilities;
+            # (2) in the EasyBuild configuration, via --cuda-compute-capabilities configuration option;
+            ec_cuda_cc = self.cfg['cuda_compute_capabilities']
+            cfg_cuda_cc = build_option('cuda_compute_capabilities')
+            cuda_cc = cfg_cuda_cc or ec_cuda_cc or []
+            if not cuda_cc:
+                raise EasyBuildError("Can't build Clang with CUDA support "
+                                     "without specifying 'cuda-compute-capabilities'")
+            default_cc = self.cfg['default_cuda_capability'] or min(cuda_cc)
+            if not self.cfg['default_cuda_capability']:
+                print_warning("No default CUDA capability defined! "
+                              "Using '%s' taken as minimum from 'cuda_compute_capabilities'" % default_cc)
+            cuda_cc = [cc.replace('.', '') for cc in cuda_cc]
+            default_cc = default_cc.replace('.', '')
+            self.cfg.update('configopts', '-DCLANG_OPENMP_NVPTX_DEFAULT_ARCH=sm_%s' % default_cc)
+            self.cfg.update('configopts', '-DLIBOMPTARGET_NVPTX_COMPUTE_CAPABILITIES=%s' % ','.join(cuda_cc))
+        # If we don't want to build with CUDA (not in dependencies) trick CMakes FindCUDA module into not finding it by
+        # using the environment variable which is used as-is and later checked for a falsy value when determining
+        # whether CUDA was found
         if not get_software_root('CUDA'):
             setvar('CUDA_NVCC_EXECUTABLE', 'IGNORE')
 

easybuild/easyblocks/c/clang.py

@@ -0,0 +1,85 @@
+##
+# Copyright 2012-2020 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# https://github.com/easybuilders/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+EasyBlock for installing code-server, implemented as an easyblock
+@author: Alan O'Cais (Juelich Supercomputing Centre)
+"""
+
+from easybuild.framework.easyblock import EasyBlock
+from easybuild.easyblocks.generic.packedbinary import PackedBinary
+from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.systemtools import AARCH64, X86_64, get_cpu_architecture
+
+
+class EB_code_minus_server(PackedBinary, EasyBlock):
+    """
+    Support for installing code-server.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """ Init the easyblock adding a new mapped_arch template var """
+        myarch = get_cpu_architecture()
+        if myarch == X86_64:
+            self.mapped_arch = 'amd64'
+        elif myarch == AARCH64:
+            self.mapped_arch = 'arm64'
+        else:
+            raise EasyBuildError("Architecture %s is not supported for code-server on EasyBuild", myarch)
+
+        super(EB_code_minus_server, self).__init__(*args, **kwargs)
+
+        self.cfg.template_values['mapped_arch'] = self.mapped_arch
+        self.cfg.generate_template_values()
+
+    def install_step(self):
+        """Custom install step for code-server."""
+        install_cmd = self.cfg.get('install_cmd', None)
+        if install_cmd is None:
+            cmd = 'cp -a code-server-%s-linux-%s/* %s' % (self.version, self.mapped_arch, self.installdir)
+            # set the install command to a default
+            self.log.info("For %s, using default installation command '%s'..." % (self.name, cmd))
+            self.cfg['install_cmd'] = cmd
+        super(EB_code_minus_server, self).install_step()
+
+    def sanity_check_step(self):
+        """Custom sanity check for code-server."""
+        custom_paths = {
+            'files': ['bin/code-server'],
+            'dirs': ['bin', 'lib', 'node_modules'],
+        }
+
+        custom_commands = ["code-server --help"]
+
+        res = super(EB_code_minus_server, self).sanity_check_step(
+            custom_paths=custom_paths,
+            custom_commands=custom_commands
+        )
+
+        return res
+
+    def make_module_extra(self):
+        """Add the default directories to the PATH."""
+        txt = EasyBlock.make_module_extra(self)
+        return txt

easybuild/easyblocks/c/code_server.py

@@ -476,6 +476,17 @@ def configure_intel_based(self):
 
         ifortver = LooseVersion(get_software_version('ifort'))
 
+        # Required due to memory leak that occurs if high optimizations are used (from CP2K 7.1 intel-popt-makefile)
+        if ifortver >= LooseVersion("2018.5"):
+            self.make_instructions += "mp2_optimize_ri_basis.o: mp2_optimize_ri_basis.F\n" \
+                "\t$(FC) -c $(subst O2,O0,$(FCFLAGSOPT)) $<\n"
+            self.log.info("Optimization level of mp2_optimize_ri_basis.F was decreased to '-O0'")
+
+        # RHEL8 intel/2020a lots of CPASSERT failed (due to high optimization in cholesky decomposition)
+        if ifortver >= LooseVersion("2019"):
+            self.make_instructions += "cp_fm_cholesky.o: cp_fm_cholesky.F\n\t$(FC) -c $(FCFLAGS2) $<\n"
+            self.log.info("Optimization flags for cp_fm_cholesky.F is set to '%s'", options['FCFLAGSOPT2'])
+
         # -i-static has been deprecated prior to 2013, but was still usable. From 2015 it is not.
         if ifortver < LooseVersion("2013"):
             options['LDFLAGS'] += ' -i-static '
@@ -528,6 +539,15 @@ def configure_GCC_based(self):
         options['FCFLAGSOPT'] += ' $(DFLAGS) $(CFLAGS) -fmax-stack-var-size=32768'
         options['FCFLAGSOPT2'] += ' $(DFLAGS) $(CFLAGS)'
 
+        gcc_version = get_software_version('GCCcore') or get_software_version('GCC')
+        if LooseVersion(gcc_version) >= LooseVersion('10.0') and LooseVersion(self.version) <= LooseVersion('7.1'):
+            # -fallow-argument-mismatch is required for CP2K 7.1 (and older) when compiling with GCC 10.x & more recent,
+            # see https://github.com/cp2k/cp2k/issues/1157, https://github.com/cp2k/dbcsr/issues/351,
+            # https://github.com/cp2k/dbcsr/commit/58ee9709545deda8524cab804bf1f88a61a864ac and
+            # https://gcc.gnu.org/legacy-ml/gcc-patches/2019-10/msg01861.html
+            options['FCFLAGSOPT'] += ' -fallow-argument-mismatch'
+            options['FCFLAGSOPT2'] += ' -fallow-argument-mismatch'
+
         return options
 
     def configure_ACML(self, options):