aws-pcluster: install gcc in the image and upload the build cache as an artifact (#30)

Install GCC in the same location where spack pipelines will build. This
enables the prefix location to relocate the compiler paths in the apps
correctly. Also, postinstall.sh will install gcc from hash, so there is no risk
that gcc will build from source in the pipeline.

`config.yaml` should stay constant, but is part of another repo `spack/spack`.
We will keep a copy around and issue a warning when running cloud_pipelines if
the file has changed.

Co-authored-by: Massimiliano Culpo <[email protected]>

Author: Stephen Sachs

.github/workflows/containers.yml

@@ -40,7 +40,7 @@ jobs:
                      [e4s-amazonlinux-2, e4s-amazonlinux-2.dockerfile, 'linux/amd64,linux/arm64'],
                      [e4s-centos-7, e4s-centos-7.dockerfile, 'linux/amd64'],
                      [e4s-fedora-36, e4s-fedora-36.dockerfile, 'linux/amd64,linux/ppc64le,linux/arm64'],
-                     [pcluster-amazonlinux-2, pcluster-amazonlinux-2.dockerfile, 'linux/amd64,linux/arm64']]
+                     [pcluster-amazonlinux-2, pcluster-amazonlinux-2/Dockerfile, 'linux/amd64,linux/arm64']]
     name: Build ${{ matrix.dockerfile[0] }}
     steps:
       - name: Checkout
@@ -79,3 +79,42 @@ jobs:
             ghcr.io/spack/${{ matrix.dockerfile[0] }}:latest
             ${{ steps.meta.outputs.tags }}
           cache-to: type=inline
+
+  upload-gcc-buildcache:
+    timeout-minutes: 60
+    runs-on: self-hosted
+    permissions:
+      packages: write
+    needs: [ deploy-images ]
+    env:
+      CONTAINER_NAME: 'ghcr.io/spack/pcluster-amazonlinux-2'
+    steps:
+      - name: Get Container Tag
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ${{ env.CONTAINER_NAME }}
+      - uses: actions/checkout@v3
+      - env:
+          CONTAINER_TAG: ${{ steps.meta.outputs.tags }}
+        run: |
+          id=$(docker create --platform linux/amd64 ${CONTAINER_TAG})
+          mkdir bootstrap-gcc-cache-x86_64
+          docker cp $id:/bootstrap-gcc-cache bootstrap-gcc-cache-x86_64
+          docker rm -v $id
+
+          id=$(docker create --platform linux/arm64 ${CONTAINER_TAG})
+          mkdir bootstrap-gcc-cache-aarch64
+          docker cp $id:/bootstrap-gcc-cache bootstrap-gcc-cache-aarch64
+          docker rm -v $id
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: bootstrap-gcc-cache-x86_64
+          path: |
+            bootstrap-gcc-cache-x86_64
+      - uses: actions/upload-artifact@v3
+        with:
+          name: bootstrap-gcc-cache-aarch64
+          path: |
+            bootstrap-gcc-cache-aarch64

Dockerfiles/pcluster-amazonlinux-2.dockerfile renamed to Dockerfiles/pcluster-amazonlinux-2/Dockerfile

@@ -109,18 +109,25 @@ RUN curl -sOL https://efa-installer.amazonaws.com/aws-efa-installer-${EFA_INSTAL
     && cd .. \
     && rm -rf aws-efa-installer-${EFA_INSTALLER_VERSION}.tar.gz aws-efa-installer
 
-# Bootstrap spack compiler installation
-RUN mkdir -p /bootstrap && \
-    cd /bootstrap && \
-    git clone https://github.com/spack/spack spack \
-    && export SPACK_ROOT=/bootstrap/spack \
-    && . spack/share/spack/setup-env.sh \
-    && curl -sOL https://raw.githubusercontent.com/spack/spack-configs/main/AWS/parallelcluster/postinstall.sh \
-    && /bin/bash postinstall.sh -fg -nointel \
-    && spack clean -a \
-    && cd /bootstrap/spack \
-    && find . -type f -maxdepth 1 -delete \
-    && rm -rf bin lib share var /root/.spack
+# Bootstrap spack compiler installation into the eventual installation tree
+# Defined in spack/share/spack/gitlab/cloud_pipelines/configs/config.yaml
+ARG SPACK_ROOT="/bootstrap-compilers/spack"
+#  commit after https://github.com/spack/spack/pull/34821
+ARG SPACK_COMMIT=f27d012
+ARG TARGETARCH
+COPY Dockerfiles/pcluster-amazonlinux-2/${TARGETARCH}/packages.yaml /root/.spack/packages.yaml
+RUN mkdir -p $(dirname "${SPACK_ROOT}") \
+    && git clone https://github.com/spack/spack "${SPACK_ROOT}" \
+    && pushd "${SPACK_ROOT}" && git checkout ${SPACK_COMMIT} && popd \
+    && cp "${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs/config.yaml" "${SPACK_ROOT}/etc/spack/config.yaml" \
+    && cp "${SPACK_ROOT}/share/spack/gitlab/cloud_pipelines/configs/config.yaml" "/bootstrap/cloud_pipelines-config.yaml" \
+    && . "${SPACK_ROOT}/share/spack/setup-env.sh" \
+    && spack compiler add \
+    && spack external find \
+    && spack tags build-tools | xargs -I {} spack config rm packages:{} \
+    && spack install gcc \
+    && spack buildcache create -a -u /bootstrap-gcc-cache $(spack find --format '/{hash}') \
+    && rm -rf $(dirname "${SPACK_ROOT}") /root/.spack
 
 ENV PATH=/bootstrap/runner/view/bin:$PATH \
     NVIDIA_VISIBLE_DEVICES=all \

Dockerfiles/pcluster-amazonlinux-2/amd64/packages.yaml

@@ -0,0 +1,5 @@
+---  # GCC setup for all AMD64 based targets
+packages:
+  gcc:
+    compiler: [gcc]
+    require: "gcc@12 +binutils ^[email protected] target=x86_64_v3"

Dockerfiles/pcluster-amazonlinux-2/arm64/packages.yaml

@@ -0,0 +1,5 @@
+---  # GCC setup for all ARM64 based targets
+packages:
+  gcc:
+    compiler: [gcc]
+    require: "gcc@12 +binutils ^[email protected] target=aarch64"