Support multiarch driver builds

Author: iamjoemccormick

.github/workflows/build-test-publish.yaml

@@ -19,6 +19,14 @@ env:
   REGISTRY: ghcr.io
   # Where to push an image of the CSI driver that will be retained (for master builds or releases) without a specific tag:
   IMAGE_NAME: ghcr.io/thinkparq/beegfs-csi-driver
+  # Equivalent of BUILD_PLATFORMS in the Makefile and release-tools build.make. We cannot just set
+  # this as a default inside the project Makefile because it will be overridden by the release-tools
+  # build.make. We can't update release-tools because it there is a check to prevent modifying
+  # release-tools. Note release-tools specifies the buildx_platform without the os (i.e., arm64
+  # instead of linux/arm64).
+  RELEASE_TOOLS_BUILD_PLATFORMS: "linux amd64 amd64 amd64;linux arm64 arm64 arm64"
+  # Used as the list of platforms for Docker buildx when it is not called through release-tools.
+  DOCKER_BUILDX_BUILD_PLATFORMS: "linux/amd64,linux/arm64"
   # Where to push an image of the CSI driver for testing (including the operator) without a specific tag:
   TEST_IMAGE_NAME: ghcr.io/thinkparq/test-beegfs-csi-driver
   # Where to push an image of the operator that will be retained (for master builds or releases) without a specific tag:
@@ -50,12 +58,12 @@ jobs:
       # Dependencies are cached by default: https://github.com/actions/setup-go#v4
       # This can be explicitly disabled if it ever causes problems.
 
-      - name: Build the container image
+      - name: Build the BeeGFS CSI driver binaries and assemble chwrap tar files for each architecture
         run: |
           export SHELL=/bin/bash
-          make container
-          echo -n "verifying images:"
-          docker images
+          make BUILD_PLATFORMS="${{ env.RELEASE_TOOLS_BUILD_PLATFORMS }}" all
+          echo -n "built artifacts:"
+          ls -alh bin/
 
       - name: Install test dependencies
         run: |
@@ -75,6 +83,9 @@ jobs:
       # TODO: Can we cache anything here? test-vendor downloads a lot of stuff.
       # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go#caching-dependencies
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Log into the GitHub Container Registry
         uses: docker/login-action@v2
         with:
@@ -84,10 +95,58 @@ jobs:
 
       # Push the image for reuse in subsequent steps, jobs, and workflows.
       # For now just tag with the commit ID to ensure subsequent jobs in this workflow run use the correct image.
-      - name: Tag and push the CSI driver as a test package
+      # This uses the Git sha: https://github.com/docker/metadata-action?tab=readme-ov-file#typesha
+      - name: Extract metadata for test CSI driver container image
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ${{ env.TEST_IMAGE_NAME }}
+          tags: |
+            type=sha,enable=true,priority=100,prefix=,suffix=,format=long
+
+      - name: Build and push test container images for each supported platform
+        uses: docker/[email protected]
+        id: build-and-push
+        with:
+          context: .
+          platforms: "${{ env.DOCKER_BUILDX_BUILD_PLATFORMS }}"
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          # If provenance is not set to false then the manifest list will contain unknown platform
+          # entries that are also displayed in GitHub. Some detail on why this is needed in:
+          # https://github.com/docker/buildx/issues/1509 and
+          # https://github.com/docker/build-push-action/issues/755#issuecomment-1607792956.
+          provenance: false
+          # Reference: https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#adding-a-description-to-multi-arch-images
+          outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=The BeeGFS Container Storage Interface (CSI) driver provides high performing and scalable storage for workloads running in Kubernetes,org.opencontainers.image.source=https://github.com/ThinkParQ/beegfs-csi-driver,org.opencontainers.image.licenses=Apache-2.0
+
+
+      - name: Install Cosign
+        uses: sigstore/[email protected]
+        with:
+          cosign-release: "v2.1.1"
+
+      # Adapted from:
+      # https://github.blog/2021-12-06-safeguard-container-signing-capability-actions/
+      # https://github.com/sigstore/cosign-installer#usage
+      - name: Sign CSI driver images for each platform with Cosign
         run: |
-          docker tag beegfs-csi-driver:latest ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }}
-          docker push ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }}
+          images=""
+          for tag in ${TAGS}; do
+            images+="${tag}@${DIGEST} "
+          done
+          cosign sign --yes --key env://COSIGN_PRIVATE_KEY \
+          -a "repo=${{ github.repository }}" \
+          -a "run=${{ github.run_id }}" \
+          -a "ref=${{ github.sha }}" \
+           ${images}
+        env:
+          TAGS: ${{ steps.meta.outputs.tags }}
+          COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
+          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
 
       # TODO: Cache this dependency for reuse here and in e2e tests.
       # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go#caching-dependencies
@@ -128,6 +187,12 @@ jobs:
           cd operator
           make -e IMG=${{ env.OPERATOR_TEST_IMAGE_NAME }}:${{ github.sha }} -e BUNDLE_IMG=${{ env.OPERATOR_TEST_BUNDLE_NAME }}:${{ github.sha }} bundle bundle-build bundle-push
 
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: beegfs-csi-driver-artifacts
+          path: bin/
+
   e2e-tests:
     runs-on: ubuntu-22.04
     timeout-minutes: 10

Dockerfile

@@ -1,23 +1,44 @@
 # Modifications Copyright 2021 NetApp, Inc. All Rights Reserved.
+# Modifications Copyright 2024 ThinkParQ, GmbH. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0.
 
 # Use distroless as minimal base image to package the driver binary. Refer to
 # https://github.com/GoogleContainerTools/distroless for more details.
-FROM gcr.io/distroless/static:latest
-
+FROM --platform=$BUILDPLATFORM gcr.io/distroless/static:latest
 LABEL maintainers="ThinkParQ"
 LABEL description="BeeGFS CSI Driver"
 LABEL org.opencontainers.image.description="BeeGFS CSI Driver"
 LABEL org.opencontainers.image.source="https://github.com/ThinkParQ/beegfs-csi-driver"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 
-# Copy all built binaries to netapp/ directory.
-COPY bin/beegfs-csi-driver bin/chwrap netapp/
+# Multi-arch images can be built from this Dockerfile. When the container image is built it is
+# expected binaries and a chwrap tar file were already created under bin/ using Make. By default
+# calling Make with no arguments builds these files for the current architecture with no suffix
+# allowing the container image to be built without multiarch support by default.
+#
+# If Make is called with the `BUILD_PLATFORMS` build argument, then binaries and chwrap tar files
+# will be generared for each platform with an architecture suffix. These can then be used to build a
+# multiarch container image using `docker buildx build` by specifying the same list of platforms
+# using the `--platform` flag. Note the buildx flag and BUILD_PLATFORMS argument accept slightly
+# different values, for example to build for both amd64 and arm64:
+#
+# `make BUILD_PLATFORMS="linux amd64 amd64 amd64;linux arm64 arm64 arm64" all`
+# `docker buildx build --platform=linux/amd64,linux/arm64`
+ARG TARGETARCH
+# Work around the fact TARGETARCH is not set consistently when building multiarch images using
+# release-tools versus docker buildx. While release-tools isn't currently used by GitHub Actions to
+# publish multiarch images, this is the only thing preventing use of release-tools, which may be
+# useful for local testing.
+ARG ARCH=$TARGETARCH
+WORKDIR /
+
+# Copy architecture specific BeeGFS CSI driver to the image.
+COPY bin/beegfs-csi-driver$ARCH /beegfs-csi-driver
 
-# Add chwrap symbolic links to netapp/ directory.
-ADD bin/chwrap.tar /
+# Unpack architecture specific chwrap symbolic links into osutils directory.
+ADD bin/chwrap$ARCH.tar /
 
 # Call chwrap linked binaries before container installed binaries.
-ENV PATH "/netapp:/$PATH"
+ENV PATH "/osutils:$PATH"
 
-ENTRYPOINT ["beegfs-csi-driver"]
+ENTRYPOINT ["/beegfs-csi-driver"]

Makefile

@@ -15,13 +15,14 @@
 # limitations under the License.
 
 # Modifications Copyright 2021 NetApp, Inc. All Rights Reserved.
+# Modifications Copyright 2024 ThinkParQ, GmbH. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0.
 
 CMDS ?= beegfs-csi-driver
 # Speed up unit testing by explicitly NOT building anything in the e2e folder.
 # Do not run any operator tests during normal testing.
 TEST_GO_FILTER_CMD = -e '/test/e2e' -e '/operator'
-all: build
+all: build build-chwrap bin/chwrap.tar
 
 check-go-version:
 	./hack/check-go-version.sh
@@ -36,7 +37,7 @@ generate-notices:
 build-%: check-go-version-go
 	# Commands are taken directly from build.make build-%.
 	mkdir -p bin
-	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \
 		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags \
 		'$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
 			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
@@ -46,14 +47,22 @@ build-%: check-go-version-go
 
 # Put symbolic links between various commands (e.g. beegfs-ctl, mount, and umount) and cmd/chwrap into a .tar file to
 # be unpacked in the container. chwrap.tar is obviously not a binary file, but bin/ is where release-tools/build.make
-# outputs files and it is cleaned out on "make clean".
+# outputs files and it is cleaned out on "make clean". If we BUILD_PLATFORMS is set then we will create multiple tar
+# files each suffixed with the appropriate architecture. Otherwise we will create a single tar file with no suffix
+# for the current architecture.
 bin/chwrap.tar: build-chwrap cmd/chwrap/chwrap.sh
-	cmd/chwrap/chwrap.sh bin/chwrap bin/chwrap.tar
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \
+		if ! (set -x; cmd/chwrap/chwrap.sh bin/chwrap$$arch bin/chwrap$$arch.tar osutils); then \
+			echo "Building $* for $$arch failed, see error(s) above."; \
+			exit 1; \
+		fi; \
+	done	
 
-# The beegfs-csi-driver container requires chwrap to be built and included, so we build it anytime container or push
-# are made. Additional prerequisites and the recipes for container and push are defined in release-tools/build.make. A
-# different workaround will likely be required for multiarch builds.
+# The beegfs-csi-driver container requires chwrap to be built and included, so we build it anytime
+# container, push, or push-multiarch are made. Additional prerequisites and the recipes for
+# container and push are defined in release-tools/build.make.
 container: build-chwrap bin/chwrap.tar
+push-multiarch: build-chwrap bin/chwrap.tar
 push: container  # not explicitly executed in release-tools/build.make
 
 # For details on what licenses are disallowed see

cmd/chwrap/chwrap.sh

@@ -4,15 +4,16 @@
 
 # Copyright 2020 NetApp, Inc. All Rights Reserved.
 # Modifications Copyright 2021 NetApp, Inc. All Rights Reserved.
+# Modifications Copyright 2024 ThinkParQ, GmbH. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0.
 
-[ -n "$1" ] && [ -n "$2" ] || exit 1
+[ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] || exit 1
 
 PREFIX=/tmp/$(uuidgen)
-mkdir -p $PREFIX/netapp
-cp "$1" $PREFIX/netapp/chwrap
+mkdir -p $PREFIX/$3
+cp "$1" $PREFIX/$3/chwrap
 for BIN in beegfs-ctl lsmod modprobe mount touch umount; do
-  ln -s chwrap $PREFIX/netapp/$BIN
+  ln -s chwrap $PREFIX/$3/$BIN
 done
-tar --owner=0 --group=0 -C $PREFIX -cf "$2" netapp
+tar --owner=0 --group=0 -C $PREFIX -cf "$2" $3
 rm -rf $PREFIX