Merge pull request #2740 from easybuilders/4.5.x

release EasyBuild v4.5.5

Author: SebastianAchilles

RELEASE_NOTES

@@ -3,7 +3,37 @@ For more detailed information, please see the git log.
 
 These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.
 
-The latest version of easybuild-easyblocks provides 244 software-specific easyblocks and 37 generic easyblocks.
+The latest version of easybuild-easyblocks provides 245 software-specific easyblocks and 37 generic easyblocks.
+
+
+v4.5.5 (June 8th 2022)
+----------------------
+
+update/bugfix release
+
+- new software-specific easyblock for FFTW.MPI (#2724)
+- minor enhancements and updates, including:
+  - update NEURON easyblock to use CMakeMake for recent versions (#2304)
+  - enhance Clang easyblock to add support for building with AMDGPU offload (#2684, #2729)
+  - update sanity check in OpenMPI easyblock to support OpenMPI v5.0.0 (#2709)
+  - don't use gold linker by default for GCC >= 11.3 (#2711)
+  - update sanity check in R easyblock for versions >= 4.2.0, since S.h is not included anymore (#2713)
+  - update ABAQUS easyblock for ABAQUS 2022 (#2716)
+  - update LLVM easyblock for LLVM v14.0.x (#2718)
+  - update Mesa easyblock to remove swr driver configopts for versions 22+ (#2719)
+  - enhance Clang easyblock to support also installing Python bindings (#2721, #2725)
+  - enhance SuperLU easyblock to support building on top of FlexiBLAS and be compatible with SuperLU v5.3 (#2722)
+  - update TensorFlow easyblock for version 2.8.0 (#2723)
+  - modify FFTW's sanity check step to allow checking only for MPI parts of FFTW installation (#2724)
+  - add support to ConfigureMake for tweaking (first part of) test command via 'test_cmd' (#2726, #2737)
+  - enhance MrBayes easyblock with custom sanity check command (#2727)
+  - update cudnnarch string templates used to compose source tarball names from cuDNN 8.3.3 onwards (#2728)
+  - add sanity check command to OpenSSL wrapper easyblock to verify that system certificates are available to OpenSSL (#2735)
+  - ignore exit code of pkg-config command in OpenSSL wrapper easyblock, since with pkgconf they exit with a non-zero exit code if the OS package is not installed (#2736)
+- various bug fixes, including:
+  - remove system-compiled binutils dirs from $LDFLAGS in binutils easyblock (#2712)
+  - fix for FreeSurfer easyblock: define $FREESURFER needed by recon_all (#2717)
+  - also symlink cert.pem in from-source OpenSSL installation (if it exists) (#2735)
 
 
 v4.5.4 (March 31st 2022)

easybuild/easyblocks/__init__.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.5.4')
+VERSION = LooseVersion('4.5.5')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/a/abaqus.py

@@ -41,7 +41,8 @@
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.environment import setvar
-from easybuild.tools.filetools import change_dir, write_file
+from easybuild.tools.filetools import change_dir, symlink, write_file
+from easybuild.tools.modules import get_software_root
 from easybuild.tools.run import run_cmd_qa
 from easybuild.tools.py2vs3 import OrderedDict
 
@@ -109,6 +110,7 @@ def install_step(self):
             no_qa = [
                 '___',
                 r"\(\d+\s*[KM]B\)",
+                r'\.\.\.$',
             ]
 
             # Match string for continuing on with the selected items
@@ -148,6 +150,17 @@ def install_step(self):
             else:
                 std_qa[selectionstr % (r"\[\*\]", "Tosca")] = "%(nr)s"
 
+            # disable CloudView
+            std_qa[r"(?P<cloudview>[0-9]+) \[X\] Search using CloudView\nEnter selection:"] = '%(cloudview)s\n\n'
+            # accept default port for documentation server
+            std_qa[r"Check that the port is free.\nDefault \[[0-9]+\]:"] = '\n'
+
+            # disable feedback by users
+            std_qa[r"(?P<feedback>[0-9]+) \[X\] Allow users to send feedback.\nEnter selection:"] = '%(feedback)s\n\n'
+
+            # disable reverse proxy
+            std_qa[r"(?P<proxy>[0-9]+) \[X\] Use a reverse proxy.\nEnter selection:"] = '%(proxy)s\n\n'
+
             # Disable Isight
             std_qa[selectionstr % (r"\[\*\]", "Isight")] = "%(nr)s"
             # Disable Search using EXALEAD
@@ -184,10 +197,18 @@ def install_step(self):
 
             std_qa[r"Please choose an action:"] = '1'
 
+            if LooseVersion(self.version) >= LooseVersion('2022'):
+                java_root = get_software_root('Java')
+                if java_root:
+                    std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[\]:"] = java_root
+                    std_qa[r"Please enter .*Java Runtime Environment.* path.(\n.*)+Default \[.+\]:"] = ''
+                else:
+                    raise EasyBuildError("Java is a required dependency for ABAQUS versions >= 2022, but it is missing")
+
             # Continue
             std_qa[nextstr] = ''
 
-            run_cmd_qa('./StartTUI.sh', qa, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True, maxhits=100)
+            run_cmd_qa('./StartTUI.sh', qa, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True, maxhits=1000)
         else:
             change_dir(self.builddir)
             if self.cfg['install_cmd'] is None:
@@ -284,6 +305,22 @@ def install_step(self):
                     run_cmd_qa('./StartTUI.sh', {}, no_qa=no_qa, std_qa=std_qa, log_all=True, simple=True, maxhits=100)
                     change_dir(cwd)
 
+        # create 'abaqus' symlink for main command, which is not there anymore starting with ABAQUS 2022
+        if LooseVersion(self.version) >= LooseVersion('2022'):
+            commands_dir = os.path.join(self.installdir, 'Commands')
+            abaqus_cmd = os.path.join(commands_dir, 'abaqus')
+            if os.path.exists(abaqus_cmd):
+                self.log.info("Main 'abaqus' command already found at %s, no need to create symbolic link", abaqus_cmd)
+            else:
+                abq_ver_cmd = os.path.join(commands_dir, 'abq%s' % self.version)
+                self.log.info("Creating symbolic link 'abaqus' for main command %s", abq_ver_cmd)
+                if os.path.exists(abq_ver_cmd):
+                    cwd = change_dir(commands_dir)
+                    symlink(os.path.basename(abq_ver_cmd), os.path.basename(abaqus_cmd))
+                    change_dir(cwd)
+                else:
+                    raise EasyBuildError("Path to main command %s does not exist!", abq_ver_cmd)
+
     def sanity_check_step(self):
         """Custom sanity check for ABAQUS."""
         custom_paths = {

easybuild/easyblocks/b/binutils.py

@@ -108,6 +108,15 @@ def configure_step(self):
         else:
             libs = []
 
+        binutilsroot = get_software_root('binutils')
+        if binutilsroot:
+            # Remove LDFLAGS that start with '-L' + binutilsroot, since we don't
+            # want to link libraries from binutils compiled with the system toolchain
+            # into binutils binaries compiled with a compiler toolchain.
+            ldflags = os.getenv('LDFLAGS').split(' ')
+            ldflags = [p for p in ldflags if not p.startswith('-L' + binutilsroot)]
+            env.setvar('LDFLAGS', ' '.join(ldflags))
+
         # configure using `--with-system-zlib` if zlib is a (build) dependency
         zlibroot = get_software_root('zlib')
         if zlibroot:

easybuild/easyblocks/c/clang.py

@@ -52,7 +52,7 @@
 from easybuild.tools.environment import setvar
 
 # List of all possible build targets for Clang
-CLANG_TARGETS = ["all", "AArch64", "ARM", "CppBackend", "Hexagon", "Mips",
+CLANG_TARGETS = ["all", "AArch64", "AMDGPU", "ARM", "CppBackend", "Hexagon", "Mips",
                  "MBlaze", "MSP430", "NVPTX", "PowerPC", "R600", "Sparc",
                  "SystemZ", "X86", "XCore"]
 
@@ -64,6 +64,14 @@
     X86_64: ['X86'],
 }
 
+# List of all possible AMDGPU gfx targets supported by LLVM
+AMDGPU_GFX_SUPPORT = ['gfx700', 'gfx701', 'gfx801', 'gfx803', 'gfx900',
+                      'gfx902', 'gfx906', 'gfx908', 'gfx90a', 'gfx90c',
+                      'gfx1010', 'gfx1030', 'gfx1031']
+
+# List of all supported CUDA toolkit versions supported by LLVM
+CUDA_TOOLKIT_SUPPORT = ['80', '90', '91', '92', '100', '101', '102', '110', '111', '112']
+
 
 class EB_Clang(CMakeMake):
     """Support for bootstrapping Clang."""
@@ -72,21 +80,24 @@ class EB_Clang(CMakeMake):
     def extra_options():
         extra_vars = CMakeMake.extra_options()
         extra_vars.update({
+            'amd_gfx_list': [None, "List of AMDGPU targets to build for. Possible values: " +
+                             ', '.join(AMDGPU_GFX_SUPPORT), CUSTOM],
             'assertions': [True, "Enable assertions.  Helps to catch bugs in Clang.", CUSTOM],
-            'build_targets': [None, "Build targets for LLVM (host architecture if None). Possible values: " +
-                                    ', '.join(CLANG_TARGETS), CUSTOM],
             'bootstrap': [True, "Bootstrap Clang using GCC", CUSTOM],
-            'usepolly': [False, "Build Clang with polly", CUSTOM],
+            'build_extra_clang_tools': [False, "Build extra Clang tools", CUSTOM],
             'build_lld': [False, "Build the LLVM lld linker", CUSTOM],
+            'build_targets': [None, "Build targets for LLVM (host architecture if None). Possible values: " +
+                                    ', '.join(CLANG_TARGETS), CUSTOM],
+            'default_cuda_capability': [None, "Default CUDA capability specified for clang, e.g. '7.5'", CUSTOM],
             'default_openmp_runtime': [None, "Default OpenMP runtime for clang (for example, 'libomp')", CUSTOM],
             'enable_rtti': [False, "Enable Clang RTTI", CUSTOM],
+            'python_bindings': [False, "Install python bindings", CUSTOM],
             'libcxx': [False, "Build the LLVM C++ standard library", CUSTOM],
-            'static_analyzer': [True, "Install the static analyser of Clang", CUSTOM],
             'skip_all_tests': [False, "Skip running of tests", CUSTOM],
+            'static_analyzer': [True, "Install the static analyser of Clang", CUSTOM],
             # The sanitizer tests often fail on HPC systems due to the 'weird' environment.
             'skip_sanitizer_tests': [True, "Do not run the sanitizer tests", CUSTOM],
-            'default_cuda_capability': [None, "Default CUDA capability specified for clang, e.g. '7.5'", CUSTOM],
-            'build_extra_clang_tools': [False, "Build extra Clang tools", CUSTOM],
+            'usepolly': [False, "Build Clang with polly", CUSTOM],
         })
         # disable regular out-of-source build, too simplistic for Clang to work
         extra_vars['separate_build_dir'][0] = False
@@ -195,9 +206,15 @@ def prepare_step(self, *args, **kwargs):
             arch = get_cpu_architecture()
             try:
                 default_targets = DEFAULT_TARGETS_MAP[arch][:]
-                # If CUDA is included as a dep, add NVPTX as a target (could also support AMDGPU if we knew how)
+                # If CUDA is included as a dep, add NVPTX as a target
                 if get_software_root("CUDA"):
                     default_targets += ["NVPTX"]
+                # For AMDGPU support we need ROCR-Runtime and
+                # ROCT-Thunk-Interface, however, since ROCT is a dependency of
+                # ROCR we only check for the ROCR-Runtime here
+                # https://openmp.llvm.org/SupportAndFAQ.html#q-how-to-build-an-openmp-amdgpu-offload-capable-compiler
+                if get_software_root("ROCR-Runtime"):
+                    default_targets += ["AMDGPU"]
                 self.cfg['build_targets'] = build_targets = default_targets
                 self.log.debug("Using %s as default build targets for CPU/GPU architecture %s.", default_targets, arch)
             except KeyError:
@@ -339,6 +356,16 @@ def configure_step(self):
         # whether CUDA was found
         if not get_software_root('CUDA'):
             setvar('CUDA_NVCC_EXECUTABLE', 'IGNORE')
+        # If 'AMDGPU' is in the build targets we assume the user would like OpenMP offload support for AMD
+        if 'AMDGPU' in build_targets:
+            if not get_software_root('ROCR-Runtime'):
+                raise EasyBuildError("Can't build Clang with AMDGPU support "
+                                     "without dependency 'ROCR-Runtime'")
+            ec_amdgfx = self.cfg['amd_gfx_list']
+            if not ec_amdgfx:
+                raise EasyBuildError("Can't build Clang with AMDGPU support "
+                                     "without specifying 'amd_gfx_list'")
+            self.cfg.update('configopts', '-DLIBOMPTARGET_AMDGCN_GFXLIST=%s' % ' '.join(ec_amdgfx))
 
         self.log.info("Configuring")
         super(EB_Clang, self).configure_step(srcdir=self.llvm_src_dir)
@@ -460,6 +487,17 @@ def install_step(self):
             except OSError as err:
                 raise EasyBuildError("Failed to copy static analyzer dirs to install dir: %s", err)
 
+    def post_install_step(self):
+        """Install python bindings."""
+        super(EB_Clang, self).post_install_step()
+
+        # copy Python bindings here in post-install step so that it is not done more than once in multi_deps context
+        if self.cfg['python_bindings']:
+            python_bindings_source_dir = os.path.join(self.llvm_src_dir, "tools", "clang", "bindings", "python")
+            python_bindins_target_dir = os.path.join(self.installdir, 'lib', 'python')
+
+            shutil.copytree(python_bindings_source_dir, python_bindins_target_dir)
+
     def sanity_check_step(self):
         """Custom sanity check for Clang."""
         shlib_ext = get_shared_lib_ext()
@@ -491,24 +529,64 @@ def sanity_check_step(self):
         if LooseVersion(self.version) >= LooseVersion('3.8'):
             custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % self.version])
 
+        # Detect OpenMP support for CPU architecture
+        arch = get_cpu_architecture()
+        # Check architecture explicitly since Clang uses potentially
+        # different names
+        if arch == X86_64:
+            arch = 'x86_64'
+        elif arch == POWER:
+            arch = 'ppc64'
+        elif arch == AARCH64:
+            arch = 'aarch64'
+        else:
+            print_warning("Unknown CPU architecture (%s) for OpenMP library check!" % arch)
+        custom_paths['files'].extend(["lib/libomptarget.%s" % shlib_ext,
+                                      "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)])
+        # If building for CUDA check that OpenMP target library was created
         if 'NVPTX' in self.cfg['build_targets']:
-            arch = get_cpu_architecture()
-            # Check architecture explicitly since Clang uses potentially
-            # different names
-            if arch == X86_64:
-                arch = 'x86_64'
-            elif arch == POWER:
-                arch = 'ppc64'
-            elif arch == AARCH64:
-                arch = 'aarch64'
+            custom_paths['files'].append("lib/libomptarget.rtl.cuda.%s" % shlib_ext)
+            # The static 'nvptx.a' library is not built from version 12 onwards
+            if LooseVersion(self.version) < LooseVersion('12.0'):
+                custom_paths['files'].append("lib/libomptarget-nvptx.a")
+            ec_cuda_cc = self.cfg['cuda_compute_capabilities']
+            cfg_cuda_cc = build_option('cuda_compute_capabilities')
+            cuda_cc = cfg_cuda_cc or ec_cuda_cc or []
+            # We need the CUDA capability in the form of '75' and not '7.5'
+            cuda_cc = [cc.replace('.', '') for cc in cuda_cc]
+            if LooseVersion('11.0') < LooseVersion(self.version) < LooseVersion('13.0'):
+                custom_paths['files'].extend(["lib/libomptarget-nvptx-cuda_%s-sm_%s.bc" % (x, y)
+                                             for x in CUDA_TOOLKIT_SUPPORT for y in cuda_cc])
             else:
-                print_warning("Unknown CPU architecture (%s) for OpenMP offloading!" % arch)
-            custom_paths['files'].extend(["lib/libomptarget.%s" % shlib_ext,
-                                          "lib/libomptarget-nvptx.a",
-                                          "lib/libomptarget.rtl.cuda.%s" % shlib_ext,
-                                          "lib/libomptarget.rtl.%s.%s" % (arch, shlib_ext)])
+                custom_paths['files'].extend(["lib/libomptarget-nvptx-sm_%s.bc" % cc
+                                             for cc in cuda_cc])
+            # From version 13, and hopefully onwards, the naming of the CUDA
+            # '.bc' files became a bit simpler and now we don't need to take
+            # into account the CUDA version Clang was compiled with, making it
+            # easier to check for the bitcode files we expect
+            if LooseVersion(self.version) >= LooseVersion('13.0'):
+                custom_paths['files'].extend(["lib/libomptarget-new-nvptx-sm_%s.bc" % cc
+                                              for cc in cuda_cc])
+        # If building for AMDGPU check that OpenMP target library was created
+        if 'AMDGPU' in self.cfg['build_targets']:
+            custom_paths['files'].append("lib/libLLVMAMDGPUCodeGen.a")
+            # OpenMP offloading support to AMDGPU was not added until version
+            # 13, however, building for the AMDGPU target predates this and so
+            # doesn't necessarily mean that the AMDGPU target failed
+            if LooseVersion(self.version) >= LooseVersion('13.0'):
+                custom_paths['files'].append("lib/libomptarget.rtl.amdgpu.%s" % shlib_ext)
+                custom_paths['files'].extend(["lib/libomptarget-amdgcn-%s.bc" % gfx
+                                              for gfx in self.cfg['amd_gfx_list']])
+                custom_paths['files'].append("bin/amdgpu-arch")
+            if LooseVersion(self.version) >= LooseVersion('14.0'):
+                custom_paths['files'].extend(["lib/libomptarget-new-amdgpu-%s.bc" % gfx
+                                              for gfx in self.cfg['amd_gfx_list']])
 
         custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags']
+        if self.cfg['python_bindings']:
+            custom_paths['files'].extend([os.path.join("lib", "python", "clang", "cindex.py")])
+            custom_commands.extend(["python -c 'import clang'"])
+
         super(EB_Clang, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)
 
     def make_module_extra(self):
@@ -517,4 +595,6 @@ def make_module_extra(self):
         # we set the symbolizer path so that asan/tsan give meanfull output by default
         asan_symbolizer_path = os.path.join(self.installdir, 'bin', 'llvm-symbolizer')
         txt += self.module_generator.set_environment('ASAN_SYMBOLIZER_PATH', asan_symbolizer_path)
+        if self.cfg['python_bindings']:
+            txt += self.module_generator.prepend_paths('PYTHONPATH', os.path.join("lib", "python"))
         return txt