Skip to content

Commit

Permalink
Update easyblocks to use EasyConfig.parallel property
Browse files Browse the repository at this point in the history
Don't use the deprecated `ec['parallel']` value anymore.
Introduce a property to `ConfigureMake` to get the commonly used `-j <n>` string.
Verify each use converting them to fStrings.
  • Loading branch information
Flamefire committed Feb 11, 2025
1 parent e0de69f commit 6f0d056
Show file tree
Hide file tree
Showing 43 changed files with 88 additions and 128 deletions.
2 changes: 1 addition & 1 deletion easybuild/easyblocks/a/adf.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def install_step(self):
except OSError as err:
raise EasyBuildError("Failed to copy %s to %s: %s", src_init_path, target_init_path, err)

cmd = "./bin/foray -j %d" % self.cfg['parallel']
cmd = f"./bin/foray -j {self.cfg.parallel}"
run_shell_cmd(cmd)

def sanity_check_step(self):
Expand Down
6 changes: 1 addition & 5 deletions easybuild/easyblocks/a/aomp.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,11 +85,7 @@ def configure_step(self):
'AOMP_APPLY_ROCM_PATCHES=0',
'AOMP_STANDALONE_BUILD=1',
]
if self.cfg['parallel']:
install_options.append(
'NUM_THREADS={!s}'.format(self.cfg['parallel']))
else:
install_options.append('NUM_THREADS=1')
install_options.append(f'NUM_THREADS={self.cfg.parallel}')
# Check if CUDA is loaded and alternatively build CUDA backend
if get_software_root('CUDA') or get_software_root('CUDAcore'):
cuda_root = get_software_root('CUDA') or get_software_root('CUDAcore')
Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/b/bazel.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def configure_step(self):
])

# enable building in parallel
bazel_args = '--jobs=%d' % self.cfg['parallel']
bazel_args = f'--jobs={self.cfg.parallel}'

# Bazel provides a JDK by itself for some architectures
# We want to enforce it using the JDK we provided via modules
Expand Down Expand Up @@ -206,7 +206,7 @@ def test_step(self):
# Avoid bazel using $HOME
'--output_user_root=%s' % self.output_user_root,
runtest,
'--jobs=%d' % self.cfg['parallel'],
f'--jobs={self.cfg.parallel}',
'--host_javabase=@local_jdk//:jdk',
# Be more verbose
'--subcommands', '--verbose_failures',
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/b/berkeleygw.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def configure_step(self):
def build_step(self):
"""Custom build step for BerkeleyGW."""

self.cfg['parallel'] = 1
self.cfg.parallel = 1

self.cfg['buildopts'] = 'all-flavors'

Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/b/boost.py
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,8 @@ def build_step(self):
self.bjamoptions += " -s%s_INCLUDE=%s/include" % (lib.upper(), libroot)
self.bjamoptions += " -s%s_LIBPATH=%s/lib" % (lib.upper(), libroot)

if self.cfg['parallel']:
self.paracmd = "-j %s" % self.cfg['parallel']
if self.cfg.parallel > 1:
self.paracmd = f"-j {self.cfg.parallel}"
else:
self.paracmd = ''

Expand Down
8 changes: 2 additions & 6 deletions easybuild/easyblocks/c/clang.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,6 @@ def __init__(self, *args, **kwargs):
self.llvm_obj_dir_stage1 = None
self.llvm_obj_dir_stage2 = None
self.llvm_obj_dir_stage3 = None
self.make_parallel_opts = ""
self.runtime_lib_path = "lib"

# Bypass the .mod file check for GCCcore installs
Expand Down Expand Up @@ -404,9 +403,6 @@ def configure_step(self):

self.cfg.update('configopts', '-DLLVM_TARGETS_TO_BUILD="%s"' % ';'.join(build_targets))

if self.cfg['parallel']:
self.make_parallel_opts = "-j %s" % self.cfg['parallel']

# If hwloc is included as a dep, use it in OpenMP runtime for affinity
hwloc_root = get_software_root('hwloc')
if hwloc_root:
Expand Down Expand Up @@ -554,7 +550,7 @@ def build_with_prev_stage(self, prev_obj, next_obj):
run_shell_cmd("cmake %s %s" % (' '.join(options), self.llvm_src_dir))

self.log.info("Building")
run_shell_cmd("make %s VERBOSE=1" % self.make_parallel_opts)
run_shell_cmd(f"make {self.parallel_flag} VERBOSE=1")

# restore $PATH
setvar('PATH', orig_path)
Expand All @@ -581,7 +577,7 @@ def test_step(self):
change_dir(self.llvm_obj_dir_stage3)
else:
change_dir(self.llvm_obj_dir_stage1)
run_shell_cmd("make %s check-all" % self.make_parallel_opts)
run_shell_cmd(f"make {self.parallel_flag} check-all")

def install_step(self):
"""Install stage 3 binaries."""
Expand Down
11 changes: 5 additions & 6 deletions easybuild/easyblocks/c/cp2k.py
Original file line number Diff line number Diff line change
Expand Up @@ -669,18 +669,17 @@ def build_step(self):
change_dir(makefiles)

# modify makefile for parallel build
parallel = self.cfg['parallel']
if parallel:

parallel = self.cfg.parallel
if parallel > 1:
try:
for line in fileinput.input('Makefile', inplace=1, backup='.orig.patchictce'):
line = re.sub(r"^PMAKE\s*=.*$", "PMAKE\t= $(SMAKE) -j %s" % parallel, line)
line = re.sub(r"^PMAKE\s*=.*$", f"PMAKE\t= $(SMAKE) -j {parallel}", line)
sys.stdout.write(line)
except IOError as err:
raise EasyBuildError("Can't modify/write Makefile in %s: %s", makefiles, err)

# update make options with MAKE
self.cfg.update('buildopts', 'MAKE="make -j %s"' % self.cfg['parallel'])
self.cfg.update('buildopts', f'MAKE="make -j {self.cfg.parallel}"')

# update make options with ARCH and VERSION
self.cfg.update('buildopts', 'ARCH=%s VERSION=%s' % (self.typearch, self.cfg['type']))
Expand Down Expand Up @@ -757,7 +756,7 @@ def test_step(self):
self.log.info("No reference output found for regression test, just continuing without it...")

# prefer using 4 cores, since some tests require/prefer square (n^2) numbers or powers of 2 (2^n)
test_core_cnt = min(self.cfg['parallel'], 4)
test_core_cnt = min(self.cfg.parallel, 4)
if get_avail_core_count() < test_core_cnt:
raise EasyBuildError("Cannot run MPI tests as not enough cores (< %s) are available", test_core_cnt)
else:
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/d/dm_reverb.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ def build_step(self, *args, **kwargs):
# use JDK from EB
bazel_build_opts += " --host_javabase=@local_jdk//:jdk"
# explicitly set the number of processes
bazel_build_opts += " --jobs=%d" % self.cfg['parallel']
bazel_build_opts += f" --jobs={self.cfg.parallel}"
# print full compilation commands
bazel_build_opts += " --subcommands"

Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/f/flook.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def __init__(self, *args, **kwargs):
else:
local_comp_flags = 'FFLAGS="$FFLAGS" CFLAGS="$CFLAGS"'
self.cfg.update('buildopts', 'liball %s' % local_comp_flags)
self.cfg['parallel'] = 1
self.cfg.parallel = 1

def configure_step(self):
# flook has no configure step
Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/g/gamess_us.py
Original file line number Diff line number Diff line change
Expand Up @@ -427,11 +427,11 @@ def test_step(self):
return

# MPI builds can only run tests that support parallel execution
if int(self.cfg['parallel']) < 2:
if self.cfg.parallel < 2:
self.log.info("Skipping testing of GAMESS-US as MPI tests need at least 2 CPU cores to run")
return

test_procs = str(self.cfg['parallel'])
test_procs = str(self.cfg.parallel)
target_tests = [exam for exam in target_tests if exam[0] not in GAMESS_SERIAL_TESTS]

if self.toolchain.mpi_family() == toolchain.INTELMPI:
Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/g/gcc.py
Original file line number Diff line number Diff line change
Expand Up @@ -733,8 +733,8 @@ def build_step(self):

# make and install stage 1 build of GCC
paracmd = ''
if self.cfg['parallel']:
paracmd = "-j %s" % self.cfg['parallel']
if self.cfg.parallel > 1:
paracmd = f"-j {self.cfg.parallel}"

cmd = "%s make %s %s" % (self.cfg['prebuildopts'], paracmd, self.cfg['buildopts'])
run_shell_cmd(cmd)
Expand Down
12 changes: 6 additions & 6 deletions easybuild/easyblocks/g/gromacs.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,12 +293,12 @@ def configure_step(self):
mpi_numprocs = self.cfg.get('mpi_numprocs', 0)
if mpi_numprocs == 0:
self.log.info("No number of test MPI tasks specified -- using default: %s",
self.cfg['parallel'])
mpi_numprocs = self.cfg['parallel']
self.cfg.parallel)
mpi_numprocs = self.cfg.parallel

elif mpi_numprocs > self.cfg['parallel']:
elif mpi_numprocs > self.cfg.parallel:
self.log.warning("Number of test MPI tasks (%s) is greater than value for 'parallel': %s",
mpi_numprocs, self.cfg['parallel'])
mpi_numprocs, self.cfg.parallel)

mpiexec = self.cfg.get('mpiexec')
if mpiexec:
Expand Down Expand Up @@ -505,7 +505,7 @@ def test_step(self):

# run 'make check' or whatever the easyconfig specifies
# in parallel since it involves more compilation
self.cfg.update('runtest', "-j %s" % self.cfg['parallel'])
self.cfg.update('runtest', f"-j {self.cfg.parallel}")
super(EB_GROMACS, self).test_step()

if build_option('rpath'):
Expand All @@ -529,7 +529,7 @@ def install_step(self):
self.log.info("skipping install step")
else:
# run 'make install' in parallel since it involves more compilation
self.cfg.update('installopts', "-j %s" % self.cfg['parallel'])
self.cfg.update('installopts', f"-j {self.cfg.parallel}")
super(EB_GROMACS, self).install_step()

def extensions_step(self, fetch=False):
Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/generic/cargo.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,8 +356,8 @@ def profile(self):
def build_step(self):
"""Build with cargo"""
parallel = ''
if self.cfg['parallel']:
parallel = "-j %s" % self.cfg['parallel']
if self.cfg.parallel > 1:
parallel = f"-j {self.cfg.parallel}"

tests = ''
if self.cfg['enable_tests']:
Expand Down
11 changes: 6 additions & 5 deletions easybuild/easyblocks/generic/configuremake.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,11 @@ def __init__(self, *args, **kwargs):

self.config_guess = None

@property
def parallel_flag(self):
"""Return the flag to enable parallelism or empty for serial"""
return f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else ''

def obtain_config_guess(self, download_source_path=None, search_source_paths=None):
"""
Locate or download an up-to-date config.guess for use with ConfigureMake
Expand Down Expand Up @@ -360,10 +365,6 @@ def build_step(self, verbose=None, path=None):
if verbose is not None:
self.log.deprecated("The 'verbose' parameter to build_step is deprecated and unneeded.", '6.0')

paracmd = ''
if self.cfg['parallel']:
paracmd = "-j %s" % self.cfg['parallel']

targets = self.cfg.get('build_cmd_targets') or DEFAULT_BUILD_TARGET
# ensure strings are converted to list
targets = [targets] if isinstance(targets, str) else targets
Expand All @@ -373,7 +374,7 @@ def build_step(self, verbose=None, path=None):
self.cfg['prebuildopts'],
self.cfg.get('build_cmd') or DEFAULT_BUILD_CMD,
target,
paracmd,
self.parallel_flag,
self.cfg['buildopts'],
])
self.log.info("Building target '%s'", target)
Expand Down
8 changes: 2 additions & 6 deletions easybuild/easyblocks/generic/mesonninja.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,9 +136,7 @@ def build_step(self, verbose=False, path=None):
"""
build_cmd = self.cfg.get('build_cmd', DEFAULT_BUILD_CMD)

parallel = ''
if self.cfg['parallel']:
parallel = "-j %s" % self.cfg['parallel']
parallel = f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else ''

cmd = "%(prebuildopts)s %(build_cmd)s -v %(parallel)s %(buildopts)s" % {
'buildopts': self.cfg['buildopts'],
Expand All @@ -164,9 +162,7 @@ def install_step(self):
"""
install_cmd = self.cfg.get('install_cmd', DEFAULT_INSTALL_CMD)

parallel = ''
if self.cfg['parallel']:
parallel = "-j %s" % self.cfg['parallel']
parallel = f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else ''

cmd = "%(preinstallopts)s %(install_cmd)s %(parallel)s %(installopts)s install" % {
'installopts': self.cfg['installopts'],
Expand Down
4 changes: 1 addition & 3 deletions easybuild/easyblocks/generic/scons.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,7 @@ def build_step(self, verbose=False):
Build with SCons
"""

par = ''
if self.cfg['parallel']:
par = "-j %s" % self.cfg['parallel']
par = f'-j {self.cfg.parallel}' if self.cfg.parallel > 1 else ''

cmd = "%(prebuildopts)s scons %(par)s %(buildopts)s %(prefix)s" % {
'buildopts': self.cfg['buildopts'],
Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/h/hadoop.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,8 +64,8 @@ def build_step(self):
raise EasyBuildError("%s not found. Failing install" % native_lib)
cmd += ' -Drequire.%s=true -D%s.prefix=%s' % (native_lib, native_lib, lib_root)

if self.cfg['parallel'] > 1:
cmd += " -T%d" % self.cfg['parallel']
if self.cfg.parallel > 1:
cmd += f" -T{self.cfg.parallel}"
run_shell_cmd(cmd)

def install_step(self):
Expand Down
6 changes: 3 additions & 3 deletions easybuild/easyblocks/i/impi.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,8 +106,8 @@ def install_step(self):
libfabric_installpath = os.path.join(self.installdir, 'intel64', 'libfabric')

make = 'make'
if self.cfg['parallel']:
make += ' -j %d' % self.cfg['parallel']
if self.cfg.parallel > 1:
make += f' -j {self.cfg.parallel}'

cmds = [
f"./configure --prefix={libfabric_installpath} {self.cfg['libfabric_configopts']}",
Expand Down Expand Up @@ -217,7 +217,7 @@ def sanity_check_step(self):
build_cmd = "mpicc -cc=%s %s -o %s" % (os.getenv('CC'), impi_testsrc, impi_testexe)

# Execute test program with appropriate MPI executable for target toolchain
params = {'nr_ranks': self.cfg['parallel'], 'cmd': impi_testexe}
params = {'nr_ranks': self.cfg.parallel, 'cmd': impi_testexe}
mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.INTELMPI, params, mpi_version=self.version)

custom_commands.extend([
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/j/jaxlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ def configure_step(self):

# Passed to the build command of bazel
bazel_options = [
'--jobs=%s' % self.cfg['parallel'],
f'--jobs={self.cfg.parallel}',
'--subcommands',
'--action_env=PYTHONPATH',
'--action_env=EBPYTHONPREFIXES',
Expand Down
6 changes: 3 additions & 3 deletions easybuild/easyblocks/m/molpro.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,8 +148,8 @@ def configure_step(self):

# determine MPI launcher command that can be used during build/test
# obtain command with specific number of cores (required by mpi_cmd_for), then replace that number with '%n'
launcher = self.toolchain.mpi_cmd_for('%x', self.cfg['parallel'])
launcher = launcher.replace(' %s' % self.cfg['parallel'], ' %n')
launcher = self.toolchain.mpi_cmd_for('%x', self.cfg.parallel)
launcher = launcher.replace(f' {self.cfg.parallel}', ' %n')

# patch CONFIG file to change LAUNCHER definition, in order to avoid having to start mpd
apply_regex_substitutions(cfgfile, [(r"^(LAUNCHER\s*=\s*).*$", r"\1 %s" % launcher)])
Expand Down Expand Up @@ -177,7 +177,7 @@ def test_step(self):

if build_option('mpi_tests'):
# extensive test
run_shell_cmd("make MOLPRO_OPTIONS='-n%s' test" % self.cfg['parallel'])
run_shell_cmd(f"make MOLPRO_OPTIONS='-n{self.cfg.parallel}")
else:
self.log.info("Skipping extensive testing of Molpro since MPI testing is disabled")

Expand Down
3 changes: 1 addition & 2 deletions easybuild/easyblocks/m/mrtrix.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,7 @@ def configure_step(self):

def build_step(self):
"""Custom build procedure for MRtrix."""
parallel = self.cfg['parallel']
env.setvar('NUMBER_OF_PROCESSORS', str(parallel))
env.setvar('NUMBER_OF_PROCESSORS', str(self.cfg.parallel))

cmd = "python build -verbose"
run_shell_cmd(cmd)
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/n/namd.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def configure_step(self):
'arch': self.cfg['charm_arch'],
'cxxflags': os.environ['CXXFLAGS'] + ' -DMPICH_IGNORE_CXX_SEEK ' + self.cfg['charm_extra_cxxflags'],
'opts': self.cfg['charm_opts'],
'parallel': self.cfg['parallel'],
'parallel': self.cfg.parallel,
}
charm_subdir = '.'.join(os.path.basename(self.charm_tarballs[0]).split('.')[:-1])
self.log.debug("Building Charm++ using cmd '%s' in '%s'" % (cmd, charm_subdir))
Expand Down
2 changes: 1 addition & 1 deletion easybuild/easyblocks/n/neuron.py
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ def sanity_check_step(self):
self.log.info("Validation of NEURON demo OK!")

if build_option('mpi_tests'):
nproc = self.cfg['parallel']
nproc = self.cfg.parallel
try:
cwd = os.getcwd()
os.chdir(os.path.join(self.cfg['start_dir'], 'src', 'parallel'))
Expand Down
4 changes: 2 additions & 2 deletions easybuild/easyblocks/n/nwchem.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,8 +276,8 @@ def build_step(self):

# check whether 64-bit integers should be used, and act on it
if not self.toolchain.options['i8']:
if self.cfg['parallel']:
self.cfg.update('buildopts', '-j %s' % self.cfg['parallel'])
if self.parallel_flag:
self.cfg.update('buildopts', self.parallel_flag)
run_shell_cmd("make %s 64_to_32" % self.cfg['buildopts'])

self.setvar_env_makeopt('USE_64TO32', "y")
Expand Down
4 changes: 1 addition & 3 deletions easybuild/easyblocks/o/openblas.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,7 @@ def build_step(self):
del os.environ[cflags]
self.log.info("Environment variable %s unset and passed through command line" % cflags)

makecmd = 'make'
if self.cfg['parallel']:
makecmd += ' -j %s' % self.cfg['parallel']
makecmd = 'make {self.parallel_flag}'

cmd = ' '.join([self.cfg['prebuildopts'], makecmd, ' '.join(build_parts), self.cfg['buildopts']])
run_shell_cmd(cmd)
Expand Down
Loading

0 comments on commit 6f0d056

Please sign in to comment.