Skip to content

Commit

Permalink
Merge pull request #2214 from easybuilders/4.3.x
Browse files Browse the repository at this point in the history
release EasyBuild v4.3.1
  • Loading branch information
migueldiascosta authored Oct 29, 2020
2 parents 8b976f7 + 4e249f5 commit ff6e0f2
Show file tree
Hide file tree
Showing 24 changed files with 791 additions and 287 deletions.
38 changes: 37 additions & 1 deletion RELEASE_NOTES
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,43 @@ For more detailed information, please see the git log.

These release notes can also be consulted at http://easybuild.readthedocs.org/en/latest/Release_notes.html.

The latest version of easybuild-easyblocks provides 224 software-specific easyblocks and 37 generic easyblocks.
The latest version of easybuild-easyblocks provides 225 software-specific easyblocks and 37 generic easyblocks.

v4.3.1 (October 29th 2020)
--------------------------

update/bugfix release

- 2 new software-specific easyblocks:
- CFDEMcoupling (#1439), NVHPC (#2190)
- minor enhancements, including:
- support having PyQt5 installed as part of Qt5 in QScintilla easyblock (#2040)
- update TensorFlow easyblock to put Bazel build files in build directory + avoid unnecessary runtime patching (#2166)
- update CBLAS easyblock for toolchains that include imkl (#2175)
- add workaround for duplicate prefix path in Eigen CMake config (#2176)
- enable CTEST_OUTPUT_ON_FAILURE for CMakeMake test step (#2181)
- add XLA build support to TensorFlow easyblock (#2182)
- also consider libfabric dep (--with-ofi) when setting default OpenMPI configure options (#2184)
- make easyblock for installing CMake aware of --sysroot (#2187)
- make OpenBLAS respect the parallelism set by easybuild (#2191)
- handle option of building Michigan State University CCT3 & CCSD3A methods in GAMESS_US easyblock (#2194)
- add support to RubyGem easyblock for installing zipped gems (#2203)
- update ScaLAPACK easyblock to support installation with CMake for recent versions (>= v2.1.0) (#2205)
- update Score-P easyblock to add support for NVHPC toolchain (#2206)
- various bug fixes, including:
- add torch/lib subdirectory in Python lib dir to $LD_LIBRARY_PATH for PyTorch installations (#2183)
- update MUMmer easyblock to use apply_regex_substitutions and copy_file functions (#2185)
- configure OpenMPI 4.x with --without-verbs when using UCX (#2188)
- take into account that zlib may be listed in --filter-deps in custom easyblock for XML R package (#2189)
- add sanity check commands to Clang easyblock and print warning for missing ncurses (#2193)
- enhance OpenFOAM easyblock to add symlinks for libraries to ensure 'mpi' versions have preference over 'dummy' versions (#2196)
- leverage installed OpenSSL/BoringSSL when cURL is used as a dependency for TensorFlow (#2197)
- add -ffree-line-length-none to gfortran flags in Siesta easyblock (#2204)
- respect --disable-mpi-tests in Siesta easyblock (#2207)
- other changes, including:
- set $EB_INSTALLPYTHON in module generated for EasyBuild rather than setting $EB_PYTHON, to allow overriding Python command to be used for running EasyBuild with $EB_PYTHON (#2109)
- remove unused custom easyblock for DIRAC (#2192, #2198)


v4.3.0 (September 13th 2020)
----------------------------
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
# recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
# UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
# This causes problems further up the dependency chain...
VERSION = LooseVersion('4.3.0')
VERSION = LooseVersion('4.3.1')
UNKNOWN = 'UNKNOWN'


Expand Down
13 changes: 11 additions & 2 deletions easybuild/easyblocks/c/cblas.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@
import os

from easybuild.easyblocks.generic.configuremake import ConfigureMake
from easybuild.tools.filetools import copy_file
from easybuild.tools.filetools import copy_dir, copy_file
from easybuild.tools.modules import get_software_root
from easybuild.tools.systemtools import get_shared_lib_ext


Expand All @@ -55,12 +56,18 @@ def configure_step(self):
self.cfg.update('buildopts', 'FC="%s"' % os.getenv('F77'))
self.cfg.update('buildopts', 'CFLAGS="%s -DADD_"' % os.getenv('CFLAGS'))
self.cfg.update('buildopts', 'FFLAGS="%s -DADD_"' % os.getenv('FFLAGS'))

blas_lib_dir = os.getenv('BLAS_LIB_DIR')
blas_libs = []
for blas_lib in os.getenv('BLAS_STATIC_LIBS').split(','):
blas_lib = os.path.join(blas_lib_dir, blas_lib)
if os.path.exists(blas_lib):
blas_libs.append(blas_lib)

if get_software_root("imkl"):
extra_blas_libs = ['-lmkl_intel_thread', '-lmkl_lapack95_lp64', '-lmkl_intel_lp64', '-lmkl_core']
blas_libs += extra_blas_libs

self.cfg.update('buildopts', 'BLLIB="%s %s"' % (' '.join(blas_libs), os.getenv('LIBS', '')))

# default build procedure should do
Expand All @@ -78,12 +85,14 @@ def install_step(self):
for solib in glob.glob(os.path.join(srcdir, 'libcblas.so*')):
copy_file(solib, os.path.join(targetdir, os.path.basename(solib)))

copy_dir(os.path.join(self.cfg['start_dir'], 'include'), os.path.join(self.installdir, 'include'))

def sanity_check_step(self):
"""
Custom sanity check for CBLAS.
"""
custom_paths = {
'files': ['lib/libcblas.a', 'lib/libcblas.%s' % get_shared_lib_ext()],
'dirs': [],
'dirs': ['include'],
}
super(EB_CBLAS, self).sanity_check_step(custom_paths=custom_paths)
169 changes: 169 additions & 0 deletions easybuild/easyblocks/c/cfdemcoupling.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
##
# Copyright 2018-2020 Ghent University
#
# This file is part of EasyBuild,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# https://github.com/easybuilders/easybuild
#
# EasyBuild is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# EasyBuild is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with EasyBuild. If not, see <http://www.gnu.org/licenses/>.
##
"""
EasyBuild support for building and installing CFDEMcoupling, implemented as an easyblock
@author: Kenneth Hoste (Ghent University)
"""
import glob
import os

import easybuild.tools.environment as env
import easybuild.tools.toolchain as toolchain
from easybuild.framework.easyblock import EasyBlock
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.filetools import copy_file, mkdir, move_file
from easybuild.tools.modules import get_software_root, get_software_version
from easybuild.tools.run import run_cmd


class EB_CFDEMcoupling(EasyBlock):
"""Support for building/installing CFDEMcoupling."""

def __init__(self, *args, **kwargs):
"""Initialisation of custom class variables for CFDEMcoupling."""
super(EB_CFDEMcoupling, self).__init__(*args, **kwargs)

self.build_in_installdir = True

self.cfdem_project_dir = os.path.join(self.installdir, '%s-%s' % (self.name, self.version))
self.liggghts_dir = os.path.join(self.installdir, 'LIGGGHTS-%s' % self.version)

def configure_step(self):
"""Set up environment for building/installing CFDEMcoupling."""

# rename top-level directory to CFDEMcoupling-<version>
top_dirs = os.listdir(self.builddir)

for (pkgname, target_dir) in [('CFDEMcoupling', self.cfdem_project_dir), ('LIGGGHTS', self.liggghts_dir)]:
pkg_topdirs = [d for d in top_dirs if d.startswith(pkgname)]
if len(pkg_topdirs) == 1:
orig_dir = os.path.join(self.builddir, pkg_topdirs[0])
move_file(orig_dir, target_dir)
else:
error_msg = "Failed to find subdirectory for %s in %s %s (missing sources for %s?)",
raise EasyBuildError(error_msg, pkgname, self.builddir, top_dirs, pkgname)

env.setvar('CFDEM_VERSION', self.version)
env.setvar('CFDEM_PROJECT_DIR', self.cfdem_project_dir)

# define $CFDEM_PROJECT_USER_DIR to an empty existing directory
project_user_dir = os.path.join(self.builddir, 'project_user_dir')
env.setvar('CFDEM_PROJECT_USER_DIR', project_user_dir)
mkdir(project_user_dir, parents=True)

cfdem_bashrc = os.path.join(self.cfdem_project_dir, 'src', 'lagrangian', 'cfdemParticle', 'etc', 'bashrc')
env.setvar('CFDEM_bashrc', cfdem_bashrc)

env.setvar('CFDEM_LIGGGHTS_SRC_DIR', os.path.join(self.liggghts_dir, 'src'))
env.setvar('CFDEM_LIGGGHTS_MAKEFILE_NAME', 'auto')

lpp_dirs = glob.glob(os.path.join(self.builddir, 'LPP-*'))
if len(lpp_dirs) == 1:
env.setvar('CFDEM_LPP_DIR', lpp_dirs[0])
else:
raise EasyBuildError("Failed to isolate LPP-* directory in %s", self.builddir)

# build in parallel
env.setvar("WM_NCOMPPROCS", str(self.cfg['parallel']))

vtk_root = get_software_root('VTK')
if vtk_root:
vtk_ver_maj_min = '.'.join(get_software_version('VTK').split('.')[:2])
vtk_inc = os.path.join(vtk_root, 'include', 'vtk-%s' % vtk_ver_maj_min)
if os.path.exists(vtk_inc):
env.setvar('VTK_INC_USR', '-I%s' % vtk_inc)
else:
raise EasyBuildError("Expected directory %s does not exist!", vtk_inc)

vtk_lib = os.path.join(vtk_root, 'lib')
if os.path.exists(vtk_lib):
env.setvar('VTK_LIB_USR', '-L%s' % vtk_lib)
else:
raise EasyBuildError("Expected directory %s does not exist!", vtk_lib)
else:
raise EasyBuildError("VTK not included as dependency")

# can't seem to use defined 'cfdemSysTest' alias, so call cfdemSystemTest.sh script directly...
cmd = "source $CFDEM_bashrc && $CFDEM_SRC_DIR/lagrangian/cfdemParticle/etc/cfdemSystemTest.sh"
run_cmd(cmd, log_all=True, simple=True, log_ok=True)

def build_step(self):
"""Custom build procedure for CFDEMcoupling."""

if get_software_root('OpenFOAM'):
openfoam_ver = get_software_version('OpenFOAM')
openfoam_maj_ver = openfoam_ver.split('.')[0]
else:
raise EasyBuildError("OpenFOAM not included as dependency")

# make sure expected additionalLibs_* file is available
addlibs_subdir = os.path.join('src', 'lagrangian', 'cfdemParticle', 'etc', 'addLibs_universal')
src_addlibs = os.path.join(self.cfdem_project_dir, addlibs_subdir, 'additionalLibs_%s.x' % openfoam_maj_ver)
target_addlibs = os.path.join(self.cfdem_project_dir, addlibs_subdir, 'additionalLibs_%s' % openfoam_ver)
copy_file(src_addlibs, target_addlibs)

# can't seem to use defined 'cfdemCompCFDEMall' alias...
cmd = "$CFDEM_SRC_DIR/lagrangian/cfdemParticle/etc/compileCFDEMcoupling_all.sh"
run_cmd("source $FOAM_BASH && source $CFDEM_bashrc && %s" % cmd, log_all=True, simple=True, log_ok=True)

def install_step(self):
"""No custom install procedure for CFDEMcoupling."""
pass

def sanity_check_step(self):
"""Custom sanity check for CFDEMcoupling."""
comp_fam = self.toolchain.comp_family()
if comp_fam == toolchain.GCC:
wm_compiler = 'Gcc'
elif comp_fam == toolchain.INTELCOMP:
wm_compiler = 'Icc'
else:
raise EasyBuildError("Unknown compiler family, don't know how to set WM_COMPILER")

psubdir = "linux64%sDPInt32Opt" % wm_compiler

cfdem_base_dir = os.path.basename(self.cfdem_project_dir)
bins = ['cfdemPostproc', 'cfdemSolverIB', 'cfdemSolverPiso', 'cfdemSolverPisoScalar', 'cfdemSolverPisoSTM']
custom_paths = {
'files': [os.path.join(cfdem_base_dir, 'platforms', psubdir, 'bin', b) for b in bins],
'dirs': [os.path.join(cfdem_base_dir, 'platforms', psubdir, 'lib')],
}
super(EB_CFDEMcoupling, self).sanity_check_step(custom_paths=custom_paths)

def make_module_extra(self):
"""Custom extra module file entries for CFDEMcoupling."""

txt = super(EB_CFDEMcoupling, self).make_module_extra()

txt += self.module_generator.set_environment('CFDEM_VERSION', self.version)
txt += self.module_generator.set_environment('CFDEM_PROJECT_DIR', self.cfdem_project_dir)
txt += self.module_generator.set_environment('CFDEM_LIGGGHTS_SRC_DIR', os.path.join(self.liggghts_dir, 'src'))
txt += self.module_generator.set_environment('CFDEM_LIGGGHTS_MAKEFILE_NAME', 'auto')

cfdem_bashrc = os.path.join(self.cfdem_project_dir, 'src', 'lagrangian', 'cfdemParticle', 'etc', 'bashrc')
txt += self.module_generator.set_environment('CFDEM_bashrc', cfdem_bashrc)

return txt
8 changes: 6 additions & 2 deletions easybuild/easyblocks/c/clang.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
from easybuild.easyblocks.generic.cmakemake import CMakeMake
from easybuild.framework.easyconfig import CUSTOM
from easybuild.tools import run
from easybuild.tools.build_log import EasyBuildError
from easybuild.tools.build_log import EasyBuildError, print_warning
from easybuild.tools.config import build_option
from easybuild.tools.filetools import apply_regex_substitutions, change_dir, mkdir
from easybuild.tools.modules import get_software_root
Expand Down Expand Up @@ -179,6 +179,9 @@ def find_source_dir(globpatterns, targetdir):
def configure_step(self):
"""Run CMake for stage 1 Clang."""

if all(dep['name'] != 'ncurses' for dep in self.cfg['dependencies']):
print_warning('Clang requires ncurses to run, did you forgot to add it to dependencies?')

self.llvm_obj_dir_stage1 = os.path.join(self.builddir, 'llvm.obj.1')
if self.cfg['bootstrap']:
self.llvm_obj_dir_stage2 = os.path.join(self.builddir, 'llvm.obj.2')
Expand Down Expand Up @@ -433,7 +436,8 @@ def sanity_check_step(self):
if LooseVersion(self.version) >= LooseVersion('3.8'):
custom_paths['files'].extend(["lib/libomp.%s" % shlib_ext, "lib/clang/%s/include/omp.h" % self.version])

super(EB_Clang, self).sanity_check_step(custom_paths=custom_paths)
custom_commands = ['clang --help', 'clang++ --help', 'llvm-config --cxxflags']
super(EB_Clang, self).sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands)

def make_module_extra(self):
"""Custom variables for Clang module."""
Expand Down
15 changes: 14 additions & 1 deletion easybuild/easyblocks/c/cmake.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ def configure_step(self):

cmake_prefix_path = os.environ.get('CMAKE_PREFIX_PATH', '').split(':')
cmake_library_path = os.environ.get('CMAKE_LIBRARY_PATH', '').split(':')
cmake_include_path = []

available_system_options = ['BZIP2', 'CURL', 'EXPAT', 'LIBARCHIVE', 'ZLIB']
for dep in self.cfg.dependencies():
Expand All @@ -89,7 +90,19 @@ def configure_step(self):
if dep_name_upper in available_system_options and '-system-' + dep_name.lower() not in configure_opts:
add_cmake_opts['CMAKE_USE_SYSTEM_' + dep_name_upper] = 'ON'

for var, values in {'CMAKE_PREFIX_PATH': cmake_prefix_path, 'CMAKE_LIBRARY_PATH': cmake_library_path}.items():
sysroot = build_option('sysroot')
if sysroot:
self.log.info("Found sysroot '%s', adding it to $CMAKE_PREFIX_PATH and $CMAKE_LIBRARY_PATH", sysroot)
cmake_prefix_path.append(sysroot)
cmake_library_path.append(os.path.join(sysroot, 'usr', 'lib'))
cmake_include_path.append(os.path.join(sysroot, 'usr', 'include'))

cmake_path_env_vars = {
'CMAKE_PREFIX_PATH': cmake_prefix_path,
'CMAKE_LIBRARY_PATH': cmake_library_path,
'CMAKE_INCLUDE_PATH': cmake_include_path,
}
for var, values in cmake_path_env_vars.items():
value = ':'.join(values)
if os.environ.get(var, '') != value:
env.setvar(var, value)
Expand Down
Loading

0 comments on commit ff6e0f2

Please sign in to comment.