Merge pull request #10 from Edgar-21/add_apptainer_files

Add example CHTC apptainer files

apptainer_files/dagmc_openmc_uw_hpc/build_image.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --partition=pre
+#SBATCH --time=0-04:00:00
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=32
+#SBATCH --mem-per-cpu=4000
+#SBATCH --error=job.%J.err
+#SBATCH --output=job.%J.out
+
+# this is correct for UW HPC, it may vary on different clusters
+export JOB_TMP_PATH=/local/$USER/${SLURM_JOB_ID}
+export TMPDIR=$JOB_TMP_PATH/tmp
+export APPTAINER_TMPDIR=$JOB_TMP_PATH/apptainer
+mkdir -p $TMPDIR
+mkdir -p $APPTAINER_TMPDIR
+
+definition_file=dagmc_openmc.def
+container_name=dagmc_openmc.sif
+
+apptainer build \
+        --bind $TMPDIR:/tmp \
+        ${container_name} ${definition_file}
+
+rm -rf $JOB_TMP_PATH

apptainer_files/dagmc_openmc_uw_hpc/dagmc_openmc.def

@@ -0,0 +1,153 @@
+Bootstrap: localimage
+# This is an image provided by chtc that has a version of openmpi that will
+# play nicely with SLURM. On another cluster, a base image with a version
+# of openmpi appropriate for that cluster should be sufficient for this 
+# image to work there.
+From: /software/chtc/containers/ubuntu/22.04/openmpi-4.1.6_gcc-11.3.0.sif
+
+%arguments
+    HDF5_URL=https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_3/downloads/hdf5-1.14.3.tar.gz
+    HDF5_TAR=hdf5-1.14.3.tar.gz
+    MOAB_VERSION=5.5.1
+    DOUBLE_DOWN_VERSION=v1.1.0
+    EMBREE_VERSION=v4.3.3
+    DAGMC_VERSION=v3.2.4
+    BUILD_DIR=/root/build
+    EMBREE_INSTALL_DIR=/opt/Embree
+    HDF5_INSTALL_DIR=/opt/hdf5
+    MOAB_INSTALL_DIR=/opt/moab
+    DOUBLE_DOWN_INSTALL_DIR=/opt/double-down
+    DAGMC_INSTALL_DIR=/opt/dagmc
+    OPENMC_INSTALL_DIR=/opt/openmc
+    CROSS_SECTIONS_DIR=/opt/cross_sections
+
+%environment
+    export HDF5_ROOT={{HDF5_INSTALL_DIR}}
+    export HDF5_LIBDIR={{HDF5_INSTALL_DIR}}/lib
+    export MOAB_DIR={{MOAB_INSTALL_DIR}}
+    export PATH={{OPENMC_INSTALL_DIR}}/bin:$PATH
+    export PATH=/opt/bin:$PATH
+
+%post
+    set -e
+    chmod 777 /tmp
+    # Update and install dependencies
+    apt-get update && apt-get -y upgrade
+    apt-get install -y \
+        git \
+        ca-certificates \
+        autoconf \
+        make \
+        cmake \
+        g++ \
+        clang \
+        gfortran \
+        python3 \
+        libpython3-dev \
+        zlib1g-dev \
+        libeigen3-dev \
+        libpng-dev \
+        libnetcdf-dev \
+        curl
+
+    # Build HDF5
+    mkdir -p {{BUILD_DIR}}/hdf5 && cd {{BUILD_DIR}}/hdf5
+    wget {{HDF5_URL}}
+    tar -xvf {{HDF5_TAR}}
+    cd hdfsrc
+    ./configure --prefix={{HDF5_INSTALL_DIR}} \
+                --enable-optimization=high --enable-shared \
+                --enable-hl \
+                --enable-build-mode=production \
+                --enable-parallel
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/hdf5
+
+    # Build Embree
+    mkdir -p {{BUILD_DIR}}/embree && cd {{BUILD_DIR}}/embree
+    git clone -b {{EMBREE_VERSION}} --depth 1 https://github.com/embree/embree.git
+    mkdir build && cd build
+    cmake ../embree \
+        -DCMAKE_INSTALL_PREFIX={{EMBREE_INSTALL_DIR}} \
+        -DEMBREE_TASKING_SYSTEM=INTERNAL \
+        -DEMBREE_ISPC_SUPPORT=OFF \
+        -DEMBREE_TUTORIALS=OFF \
+        -DEMBREE_ROOT=/usr && \
+    make -j$(nproc) && \
+    make install
+    rm -rf {{BUILD_DIR}}/embree
+
+    # Build MOAB
+    mkdir -p {{BUILD_DIR}}/moab && cd {{BUILD_DIR}}/moab
+    git clone -b {{MOAB_VERSION}} --depth 1 https://bitbucket.org/fathomteam/moab.git
+    mkdir build && cd build
+    cmake ../moab \
+        -DCMAKE_INSTALL_RPATH={{HDF5_INSTALL_DIR}}/lib:{{MOAB_INSTALL_DIR}}/lib \
+        -DENABLE_HDF5=ON \
+        -DHDF5_ROOT={{HDF5_INSTALL_DIR}} \
+        -DCMAKE_INSTALL_PREFIX={{MOAB_INSTALL_DIR}} \
+        -DENABLE_BLASLAPACK=OFF \
+        -DBUILD_SHARED_LIBS=ON \
+        -DENABLE_FORTRAN=OFF \
+        -DCMAKE_C_COMPILER=mpicc \
+        -DCMAKE_CXX_COMPILER=mpicxx
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/moab
+
+    # Build Double-Down
+    mkdir -p {{BUILD_DIR}}/double-down && cd {{BUILD_DIR}}/double-down
+    git clone -b {{DOUBLE_DOWN_VERSION}} --depth 1 https://github.com/pshriwise/double-down.git
+    mkdir build && cd build
+    cmake ../double-down \
+        -DMOAB_DIR={{MOAB_INSTALL_DIR}} \
+        -DCMAKE_INSTALL_PREFIX={{DOUBLE_DOWN_INSTALL_DIR}} \
+        -DEMBREE_DIR={{EMBREE_INSTALL_DIR}}
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/double-down
+
+    # Build DAGMC
+    mkdir -p {{BUILD_DIR}}/dagmc && cd {{BUILD_DIR}}/dagmc
+    git clone -b {{DAGMC_VERSION}} --depth 1 https://github.com/svalinn/DAGMC.git
+    mkdir build && cd build
+    cmake ../DAGMC \
+        -DBUILD_TALLY=ON \
+        -DMOAB_DIR={{MOAB_INSTALL_DIR}} \
+        -DBUILD_STATIC_LIBS=OFF \
+        -DCMAKE_INSTALL_PREFIX={{DAGMC_INSTALL_DIR}} \
+        -DCMAKE_C_COMPILER=mpicc \
+        -DCMAKE_CXX_COMPILER=mpicxx \
+        -DDOUBLE_DOWN=ON \
+        -DDOUBLE_DOWN_DIR={{DOUBLE_DOWN_INSTALL_DIR}}
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/dagmc
+
+    #environment variables for openmc
+    export HDF5_ROOT={{HDF5_INSTALL_DIR}}
+    export HDF5_INCLUDE_DIR={{HDF5_INSTALL_DIR}}/include
+    export HDF5_LIBDIR={{HDF5_INSTALL_DIR}}/lib
+    export METHOD=opt
+
+    # Build OpenMC
+    mkdir -p {{BUILD_DIR}}/openmc && cd {{BUILD_DIR}}/openmc
+    git clone --recurse-submodules https://github.com/openmc-dev/openmc.git
+    mkdir build && cd build
+    cmake ../openmc \
+        -DCMAKE_INSTALL_PREFIX={{OPENMC_INSTALL_DIR}} \
+        -DOPENMC_USE_MPI=ON \
+        -DOPENMC_USE_DAGMC=ON \
+        -DDAGMC_ROOT={{DAGMC_INSTALL_DIR}} \
+        -DCMAKE_BUILD_TYPE=Release .. && \
+    make install -j$(nproc)
+
+    echo 'export PATH=$PATH:{{OPENMC_INSTALL_DIR}}/bin' >> $APPTAINER_ENVIRONMENT
+
+    # Install Python bindings
+    cd {{BUILD_DIR}}/openmc/openmc
+    python3 -m pip install --no-cache-dir .
+
+    # Cleanup build directory
+    rm -rf {{BUILD_DIR}}

apptainer_files/dagmc_openmc_uw_hpc/readme.md

@@ -0,0 +1,12 @@
+# How to Use
+This def file is intended to be built on UW Madison's HPC cluster, creating an image that can be used to run DAGMC enabled OpenMC. It uses a base image provided on the cluster. It includes HDF5, EMBREE, MOAB, Double Down, DAGMC, and OpenMC.
+
+To build the image, run
+
+`sbatch build_image.sh`
+
+On the UW HPC cluster.
+
+This will build the image using `dagmc_openmc.def` and place the finished image `dagmc_openmc.sif` in this directory. That image can then be used to run future jobs. See the top level readme for how to run a job with an Apptainer image.
+
+For details on the UW HPC cluster, see their [webpage](https://chtc.cs.wisc.edu/uw-research-computing/hpc-overview).

apptainer_files/example_submit_script.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+#SBATCH --partition=shared
+#SBATCH --time=0-24:00:00            
+#SBATCH --nodes=10               
+#SBATCH --ntasks-per-node=32   
+#SBATCH --mem=30000
+#SBATCH --error=job.%J.err
+#SBATCH --output=job.%J.out
+
+# load required modules here
+module load openmpi
+# image_path is the path to the Apptainer .sif file you wish to use
+export image_path=/path/to/apptainer/image
+
+# mounts the `bind_path` location on the system into the image 
+# input files can be stored here, and output files are written here, unless
+# otherwise specified.
+bind_path=/path/to/bind
+
+# Launches the command MY_COMMAND in parallel with SLURM_NNODES MPI processes
+# inside the above apptainer image while mounting the bind_path into the image
+# Note that MY_COMMAND may need a flag for thread parallelization to take advantage of the 
+# number of tasks per node specified
+srun --mpi=pmix -n ${SLURM_NNODES} apptainer exec --bind ${bind_path}:${bind_path} ${image_path} MY_COMMAND
+
+# alternatively, you can replace MY_COMMAND with a bash script (e.g. my_commands.sh) if you need to run multiple commands in sequence

apptainer_files/readme.md

@@ -0,0 +1,6 @@
+# apptainer_files
+Houses Apptainer def files which can be used to build Apptainer images for various uses.
+
+Each def file should be placed in its own subdirectory, named to indicate what software is included in the def file. In each subdirectory, a readme describing what the image is, what it is intended to be used for, and how to build it should be included.
+
+An example of how to run a job using an apptainer on the UW HPC is included in `example_submit_script.sh`. CHTC also has a short guide [here](https://chtc.cs.wisc.edu/uw-research-computing/apptainer-hpc).