organize files, add apptainer def

apptainer_files/dagmc_openmc/build_image.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH --partition=pre
+#SBATCH --time=0-04:00:00
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=8
+#SBATCH --mem-per-cpu=4000
+#SBATCH --error=job.%J.err
+#SBATCH --output=job.%J.out
+
+export JOB_TMP_PATH=/local/$USER/${SLURM_JOB_ID}
+export TMPDIR=$JOB_TMP_PATH/tmp
+export APPTAINER_TMPDIR=$JOB_TMP_PATH/apptainer
+mkdir -p $TMPDIR
+mkdir -p $APPTAINER_TMPDIR
+
+container_name=dagmc_openmc
+
+apptainer build \
+        --bind $TMPDIR:/tmp \
+        "${@:2}" \
+        ${container_name}.sif ${container_name}.def
+
+rm -rf $JOB_TMP_PATH

apptainer_files/dagmc_openmc/dagmc_openmc.def

@@ -0,0 +1,148 @@
+Bootstrap: localimage
+From: /software/chtc/containers/ubuntu/22.04/openmpi-4.1.6_gcc-11.3.0.sif
+
+%arguments
+    HDF5_VERSION=hdf5_1_14_3
+    MOAB_VERSION=5.3.0
+    DOUBLE_DOWN_VERSION=v1.0.0
+    EMBREE_VERSION=v3.12.2
+    DAGMC_VERSION=v3.2.1
+    BUILD_DIR=/root/build
+    EMBREE_INSTALL_DIR=/opt/Embree
+    HDF5_INSTALL_DIR=/opt/hdf5
+    MOAB_INSTALL_DIR=/opt/moab
+    DOUBLE_DOWN_INSTALL_DIR=/opt/double-down
+    DAGMC_INSTALL_DIR=/opt/dagmc
+    OPENMC_INSTALL_DIR=/opt/openmc
+    CROSS_SECTIONS_DIR=/opt/cross_sections
+
+%environment
+    export HDF5_ROOT={{HDF5_INSTALL_DIR}}
+    export HDF5_LIBDIR={{HDF5_INSTALL_DIR}}/lib
+    export MOAB_DIR={{MOAB_INSTALL_DIR}}
+    export PATH={{OPENMC_INSTALL_DIR}}/bin:$PATH
+    export PATH=/opt/bin:$PATH
+
+%post
+    set -e
+    chmod 777 /tmp
+    # Update and install dependencies
+    apt-get update && apt-get -y upgrade
+    apt-get install -y \
+        git \
+        ca-certificates \
+        autoconf \
+        make \
+        cmake \
+        g++ \
+        clang \
+        gfortran \
+        python3 \
+        libpython3-dev \
+        zlib1g-dev \
+        libeigen3-dev \
+		libpng-dev \
+		libnetcdf-dev \
+        curl
+
+    # Build HDF5
+    mkdir -p {{BUILD_DIR}}/hdf5 && cd {{BUILD_DIR}}/hdf5
+    wget https://support.hdfgroup.org/releases/hdf5/v1_14/v1_14_3/downloads/hdf5-1.14.3.tar.gz
+    tar -xvf hdf5-1.14.3.tar.gz
+    cd hdfsrc
+    ./configure --prefix={{HDF5_INSTALL_DIR}} \
+                --enable-optimization=high --enable-shared \
+                --enable-hl \
+                --enable-build-mode=production \
+                --enable-parallel
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/hdf5
+
+    # Build Embree
+    mkdir -p {{BUILD_DIR}}/embree && cd {{BUILD_DIR}}/embree
+    git clone -b {{EMBREE_VERSION}} --depth 1 https://github.com/embree/embree.git
+    mkdir build && cd build
+    cmake ../embree \
+        -DCMAKE_INSTALL_PREFIX={{EMBREE_INSTALL_DIR}} \
+        -DEMBREE_TASKING_SYSTEM=INTERNAL \
+        -DEMBREE_ISPC_SUPPORT=OFF \
+        -DEMBREE_TUTORIALS=OFF \
+        -DEMBREE_ROOT=/usr && \
+    make -j$(nproc) && \
+    make install
+    rm -rf {{BUILD_DIR}}/embree
+
+    # Build MOAB
+    mkdir -p {{BUILD_DIR}}/moab && cd {{BUILD_DIR}}/moab
+    git clone -b {{MOAB_VERSION}} --depth 1 https://bitbucket.org/fathomteam/moab.git
+    mkdir build && cd build
+    cmake ../moab \
+        -DCMAKE_INSTALL_RPATH={{HDF5_INSTALL_DIR}}/lib:{{MOAB_INSTALL_DIR}}/lib \
+        -DENABLE_HDF5=ON \
+        -DHDF5_ROOT={{HDF5_INSTALL_DIR}} \
+        -DCMAKE_INSTALL_PREFIX={{MOAB_INSTALL_DIR}} \
+        -DENABLE_BLASLAPACK=OFF \
+        -DBUILD_SHARED_LIBS=ON \
+        -DENABLE_FORTRAN=OFF \
+        -DCMAKE_C_COMPILER=mpicc \
+        -DCMAKE_CXX_COMPILER=mpicxx
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/moab
+
+    # Build Double-Down
+    mkdir -p {{BUILD_DIR}}/double-down && cd {{BUILD_DIR}}/double-down
+    git clone -b {{DOUBLE_DOWN_VERSION}} --depth 1 https://github.com/pshriwise/double-down.git
+    mkdir build && cd build
+    cmake ../double-down \
+        -DMOAB_DIR={{MOAB_INSTALL_DIR}} \
+        -DCMAKE_INSTALL_PREFIX={{DOUBLE_DOWN_INSTALL_DIR}} \
+        -DEMBREE_DIR={{EMBREE_INSTALL_DIR}}
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/double-down
+
+    # Build DAGMC
+    mkdir -p {{BUILD_DIR}}/dagmc && cd {{BUILD_DIR}}/dagmc
+    git clone -b {{DAGMC_VERSION}} --depth 1 https://github.com/svalinn/DAGMC.git
+    mkdir build && cd build
+    cmake ../DAGMC \
+        -DBUILD_TALLY=ON \
+        -DMOAB_DIR={{MOAB_INSTALL_DIR}} \
+        -DBUILD_STATIC_LIBS=OFF \
+        -DCMAKE_INSTALL_PREFIX={{DAGMC_INSTALL_DIR}} \
+        -DCMAKE_C_COMPILER=mpicc \
+        -DCMAKE_CXX_COMPILER=mpicxx \
+        -DDOUBLE_DOWN=ON \
+        -DDOUBLE_DOWN_DIR={{DOUBLE_DOWN_INSTALL_DIR}}
+    make -j$(nproc)
+    make install
+    rm -rf {{BUILD_DIR}}/dagmc
+
+    #environment variables for openmc
+    export HDF5_ROOT={{HDF5_INSTALL_DIR}}
+    export HDF5_INCLUDE_DIR={{HDF5_INSTALL_DIR}}/include
+    export HDF5_LIBDIR={{HDF5_INSTALL_DIR}}/lib
+    export METHOD=opt
+
+    # Build OpenMC
+    mkdir -p {{BUILD_DIR}}/openmc && cd {{BUILD_DIR}}/openmc
+    git clone --recurse-submodules https://github.com/openmc-dev/openmc.git
+    mkdir build && cd build
+    cmake ../openmc \
+        -DCMAKE_INSTALL_PREFIX={{OPENMC_INSTALL_DIR}} \
+        -DOPENMC_USE_MPI=ON \
+        -DOPENMC_USE_DAGMC=ON \
+        -DDAGMC_ROOT={{DAGMC_INSTALL_DIR}} \
+        -DCMAKE_BUILD_TYPE=Release .. && \
+    make install -j$(nproc)
+
+    echo 'export PATH=$PATH:{{OPENMC_INSTALL_DIR}}/bin' >> $APPTAINER_ENVIRONMENT
+
+    # Install Python bindings
+    cd {{BUILD_DIR}}/openmc/openmc
+    python3 -m pip install --no-cache-dir .
+
+    # Cleanup build directory
+    rm -rf {{BUILD_DIR}}

apptainer_files/dagmc_openmc/readme.md

@@ -0,0 +1,8 @@
+# How to Use
+This def file is intended to be built on UW Madison's HPC cluster, creating an image that can be used to run DAGMC enabled OpenMC. It uses a base image provided on the cluster. It includes HDF5, EMBREE, MOAB, Double Down, DAGMC, and OpenMC.
+
+To build the image, run
+
+`sbatch build_image.sh`
+
+This will build the image using `dagmc_openmc.def` and place the finished image `dagmc_openmc.sif` in this directory. That image can then be used to run future jobs. See the top level readme for how to run a job with an Apptainer image.

apptainer_files/example_submit_script.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#SBATCH --partition=pre       # default "shared", if not specified
+#SBATCH --time=0-24:00:00       # run time in days-hh:mm:ss
+#SBATCH --nodes=10               # require 1 nodes
+#SBATCH --ntasks-per-node=32   # cpus per node (by default, "ntasks"="cpus")
+#SBATCH --mem=30000
+#SBATCH --error=job.%J.err
+#SBATCH --output=job.%J.out
+
+module load openmpi
+# image_path is the path to the Apptainer .sif file you wish to use
+export image_path=/path/to/apptainer/image
+
+# the bind path is effectively the working directory used by the image.
+# input files can be stored here, and output files are written here, unless
+# otherwise specified.
+bind_path=/path/to/bind
+
+# This tells slurm to use the apptainer image specified above, along with the
+# the bind path to execute the contents of my_executable.sh. You can also
+# just type bash commands rather than keeping them in a script, e.g.
+# replace my_executable.sh with openmc model.xml
+srun --mpi=pmix -n $SLURM_NNODES apptainer exec --bind ${bind_path}:${bind_path} ${image_path} my_executable.sh

apptainer_files/readme.md

@@ -0,0 +1,6 @@
+# hpc_apptainers
+Houses Apptainer def files which can be used to build Apptainer images for various uses.
+
+Each def file should be placed in its own subdirectory, named to indicate what software is included in the def file. In each subdirectory, a readme describing what the image is, what it is intended to be used for, and how to build it should be included.
+
+An example of how to run a job using an apptainer is included in `example_submit_script.sh`. CHTC also has a short guide [here](https://chtc.cs.wisc.edu/uw-research-computing/apptainer-hpc).

dockerfiles/moab

@@ -0,0 +1,32 @@
+FROM ubuntu:16.04
+
+RUN apt-get -y --force-yes update
+RUN apt-get install -y --force-yes \
+    software-properties-common python-software-properties wget
+
+# moab specific dependencies
+RUN apt-get install -y build-essential git cmake vim autoconf automake libtool libhdf5-dev libeigen3-dev \
+    libblas-dev liblapack-dev gfortran
+
+
+RUN \
+    cd && \
+    git clone --branch fortran_free --single-branch https://bitbucket.org/gonuke/moab
+
+RUN \
+    cd && \
+    cd moab && \
+    mkdir bld_cmake && \
+    cd bld_cmake && \
+    cmake .. -DCMAKE_C_FLAGS="-fPIC -DPIC" -DCMAKE_CXX_FLAGS="-fPIC -DPIC" -DBUILD_SHARED_LIBS=ON \
+       -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--no-undefined" -DENABLE_MPI=OFF  \
+       -DENABLE_HDF5=ON -DHDF5_ROOT=/usr/lib/x86_64-linux-gnu/hdf5/serial \
+       -DENABLE_NETCDF=OFF -DENABLE_METIS=OFF -DENABLE_IMESH=OFF -DENABLE_FBIGEOM=OFF && \
+    make all VERBOSE=1 -j 8
+
+RUN \
+    cd && cd moab/bld_cmake && \
+    make test VERBOSE=1 -j 8
+
+
+

dockerfiles/moab-fortran-free

@@ -0,0 +1,32 @@
+FROM ubuntu:16.04
+
+RUN apt-get -y --force-yes update
+RUN apt-get install -y --force-yes \
+    software-properties-common python-software-properties wget
+
+# moab specific dependencies
+RUN apt-get install -y build-essential git cmake vim autoconf automake libtool libhdf5-dev libeigen3-dev
+
+
+RUN \
+    cd && \
+    git clone --branch fortran_free --single-branch https://bitbucket.org/gonuke/moab
+
+RUN \
+    cd && \
+    cd moab && \
+    mkdir bld_cmake && \
+    cd bld_cmake && \
+    cmake .. -DCMAKE_C_FLAGS="-fPIC -DPIC" -DCMAKE_CXX_FLAGS="-fPIC -DPIC" -DBUILD_SHARED_LIBS=ON \
+       -DCMAKE_SHARED_LINKER_FLAGS="-Wl,--no-undefined" -DENABLE_MPI=OFF  \
+       -DENABLE_HDF5=ON -DHDF5_ROOT=/usr/lib/x86_64-linux-gnu/hdf5/serial \
+       -DENABLE_BLASLAPACK=OFF  -DENABLE_FORTRAN=OFF \
+       -DENABLE_NETCDF=OFF -DENABLE_METIS=OFF -DENABLE_IMESH=OFF -DENABLE_FBIGEOM=OFF && \
+    make all VERBOSE=1 -j 8
+
+RUN \
+    cd && cd moab/bld_cmake && \
+    make test VERBOSE=1 -j 8
+
+
+

dockerfiles/pyne-apt-get

@@ -0,0 +1,11 @@
+FROM ubuntu:16.04
+
+RUN apt-get -y --force-yes update
+RUN apt-get install -y --force-yes \
+    software-properties-common python-software-properties wget
+
+# pyne specific dependencies
+RUN apt-get install -y build-essential python-numpy python-scipy cython \
+                       python-nose git cmake vim emacs gfortran libblas-dev \
+                       liblapack-dev libhdf5-dev gfortran python-tables \
+                       python-matplotlib python-jinja2 autoconf libtool

dockerfiles/pyne-deps

@@ -0,0 +1,40 @@
+FROM gonuke/pyne-apt-get:latest
+
+# need to put libhdf5.so on LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu
+ENV LIBRARY_PATH /usr/lib/x86_64-linux-gnu
+
+# make starting directory
+RUN cd $HOME \
+  && mkdir opt
+
+# build MOAB
+RUN cd $HOME/opt \
+  && mkdir moab \
+  && cd moab \
+  && git clone https://bitbucket.org/fathomteam/moab \ 
+  && cd moab \ 
+  && git checkout -b Version4.9.1 origin/Version4.9.1 \ 
+  && autoreconf -fi \ 
+  && cd .. \ 
+  && mkdir build \
+  && cd build \
+  && ../moab/configure --enable-shared --enable-dagmc --with-hdf5=/usr/lib/x86_64-linux-gnu/hdf5/serial --prefix=$HOME/opt/moab \
+  && make \
+  && make install \
+  && cd .. \
+  && rm -rf build moab
+
+# put MOAB on the path
+ENV LD_LIBRARY_PATH $HOME/opt/moab/lib:$LD_LIBRARY_PATH
+ENV LIBRARY_PATH $HOME/opt/moab/lib:$LIBRARY_PATH
+
+# build PyTAPS
+RUN cd $HOME/opt \
+  && wget https://pypi.python.org/packages/source/P/PyTAPS/PyTAPS-1.4.tar.gz \
+  && tar zxvf PyTAPS-1.4.tar.gz \ 
+  && rm PyTAPS-1.4.tar.gz \
+  && cd PyTAPS-1.4/ \
+  && python setup.py --iMesh-path=$HOME/opt/moab --without-iRel --without-iGeom install --user \ 
+  && cd .. \
+  && rm -rf PyTAPS-1.4

dockerfiles/pyne-install

@@ -0,0 +1,26 @@
+FROM gonuke/pyne-deps:latest
+
+# need to put libhdf5.so on LD_LIBRARY_PATH
+ENV LD_LIBRARY_PATH /usr/lib/x86_64-linux-gnu
+ENV LIBRARY_PATH /usr/lib/x86_64-linux-gnu
+
+# put MOAB on the path
+ENV LD_LIBRARY_PATH $HOME/opt/moab/lib:$LD_LIBRARY_PATH
+ENV LIBRARY_PATH $HOME/opt/moab/lib:$LIBRARY_PATH
+
+# Install PyNE
+RUN cd $HOME/opt \
+    && git clone https://github.com/pyne/pyne.git \
+    && cd pyne \
+    && python setup.py install --user -- -DMOAB_LIBRARY=$HOME/opt/moab/lib -DMOAB_INCLUDE_DIR=$HOME/opt/moab/include
+
+RUN echo "export PATH=$HOME/.local/bin:\$PATH" >> ~/.bashrc \
+    && echo "export LD_LIBRARY_PATH=$HOME/.local/lib:\$LD_LIBRARY_PATH" >> ~/.bashrc \
+    && echo "alias build_pyne='python setup.py install --user -- -DMOAB_LIBRARY=\$HOME/opt/moab/lib -DMOAB_INCLUDE_DIR=\$HOME/opt/moab/include'" >> ~/.bashrc
+
+ENV LD_LIBRARY_PATH $HOME/.local/lib:$LD_LIBRARY_PATH
+
+RUN cd $HOME/opt/pyne && ./scripts/nuc_data_make \
+    && cd tests \ 
+    && . ./travis-run-tests.sh \
+    && echo "PyNE build complete. PyNE can be rebuilt with the alias 'build_pyne' executed from $HOME/opt/pyne"

dockerfiles/snippets

@@ -0,0 +1,11 @@
+FROM ubuntu:20.04
+
+RUN apt update \
+    # Dependencies.
+    && apt -y install python3 python3-pip \
+    && pip3 install -U \
+        numpy \
+        scipy \
+        matplotlib \
+    # Install pytest.
+    && pip3 install -U pytest

dockerfiles/u16.04.clean

@@ -0,0 +1,8 @@
+FROM ubuntu:16.04
+
+RUN apt-get -y --force-yes update
+RUN apt-get install -y --force-yes \
+    software-properties-common python-software-properties sudo wget
+
+RUN apt-get -y --force-yes update
+