Merge pull request #24 from ThinkParQ/iamjoemccormick/add-multi-architecture-builds

Add support for multiarch builds (amd64 and arm64)

Author: iamjoemccormick

.github/workflows/build-test-publish.yaml

@@ -1,23 +1,44 @@
 # Modifications Copyright 2021 NetApp, Inc. All Rights Reserved.
+# Modifications Copyright 2024 ThinkParQ, GmbH. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0.
 
 # Use distroless as minimal base image to package the driver binary. Refer to
 # https://github.com/GoogleContainerTools/distroless for more details.
-FROM gcr.io/distroless/static:latest
-
+FROM --platform=$TARGETPLATFORM gcr.io/distroless/static:latest
 LABEL maintainers="ThinkParQ"
 LABEL description="BeeGFS CSI Driver"
 LABEL org.opencontainers.image.description="BeeGFS CSI Driver"
 LABEL org.opencontainers.image.source="https://github.com/ThinkParQ/beegfs-csi-driver"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
 
-# Copy all built binaries to netapp/ directory.
-COPY bin/beegfs-csi-driver bin/chwrap netapp/
+# Multi-arch images can be built from this Dockerfile. When the container image is built it is
+# expected binaries and a chwrap tar file were already created under bin/ using Make. By default
+# calling Make with no arguments builds these files for the current architecture with no suffix
+# allowing the container image to be built without multiarch support by default.
+#
+# If Make is called with the `BUILD_PLATFORMS` build argument, then binaries and chwrap tar files
+# will be generated for each platform with an architecture suffix. These can then be used to build a
+# multiarch container image using `docker buildx build` by specifying the same list of platforms
+# using the `--platform` flag. Note the buildx flag and BUILD_PLATFORMS argument accept slightly
+# different values, for example to build for both amd64 and arm64:
+#
+# `make BUILD_PLATFORMS="linux amd64 amd64 amd64;linux arm64 arm64 arm64" all`
+# `docker buildx build --platform=linux/amd64,linux/arm64`
+ARG TARGETARCH
+# Work around the fact TARGETARCH is not set consistently when building multiarch images using
+# release-tools versus docker buildx. While release-tools isn't currently used by GitHub Actions to
+# publish multiarch images, this is the only thing preventing use of release-tools, which may be
+# useful for local testing.
+ARG ARCH=$TARGETARCH
+WORKDIR /
+
+# Copy architecture specific BeeGFS CSI driver to the image.
+COPY bin/beegfs-csi-driver$ARCH /beegfs-csi-driver
 
-# Add chwrap symbolic links to netapp/ directory.
-ADD bin/chwrap.tar /
+# Unpack architecture specific chwrap symbolic links into osutils directory.
+ADD bin/chwrap$ARCH.tar /
 
 # Call chwrap linked binaries before container installed binaries.
-ENV PATH "/netapp:/$PATH"
+ENV PATH "/osutils:$PATH"
 
-ENTRYPOINT ["beegfs-csi-driver"]
+ENTRYPOINT ["/beegfs-csi-driver"]

Dockerfile

@@ -15,13 +15,14 @@
 # limitations under the License.
 
 # Modifications Copyright 2021 NetApp, Inc. All Rights Reserved.
+# Modifications Copyright 2024 ThinkParQ, GmbH. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0.
 
 CMDS ?= beegfs-csi-driver
 # Speed up unit testing by explicitly NOT building anything in the e2e folder.
 # Do not run any operator tests during normal testing.
 TEST_GO_FILTER_CMD = -e '/test/e2e' -e '/operator'
-all: build
+all: build build-chwrap bin/chwrap.tar
 
 check-go-version:
 	./hack/check-go-version.sh
@@ -36,7 +37,7 @@ generate-notices:
 build-%: check-go-version-go
 	# Commands are taken directly from build.make build-%.
 	mkdir -p bin
-	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch suffix; do \
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \
 		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags \
 		'$(FULL_LDFLAGS)' -o "./bin/$*$$suffix" ./cmd/$*); then \
 			echo "Building $* for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
@@ -46,14 +47,22 @@ build-%: check-go-version-go
 
 # Put symbolic links between various commands (e.g. beegfs-ctl, mount, and umount) and cmd/chwrap into a .tar file to
 # be unpacked in the container. chwrap.tar is obviously not a binary file, but bin/ is where release-tools/build.make
-# outputs files and it is cleaned out on "make clean".
+# outputs files and it is cleaned out on "make clean". If we BUILD_PLATFORMS is set then we will create multiple tar
+# files each suffixed with the appropriate architecture. Otherwise we will create a single tar file with no suffix
+# for the current architecture.
 bin/chwrap.tar: build-chwrap cmd/chwrap/chwrap.sh
-	cmd/chwrap/chwrap.sh bin/chwrap bin/chwrap.tar
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \
+		if ! (set -x; cmd/chwrap/chwrap.sh bin/chwrap$$arch bin/chwrap$$arch.tar osutils); then \
+			echo "Building $* for $$arch failed, see error(s) above."; \
+			exit 1; \
+		fi; \
+	done	
 
-# The beegfs-csi-driver container requires chwrap to be built and included, so we build it anytime container or push
-# are made. Additional prerequisites and the recipes for container and push are defined in release-tools/build.make. A
-# different workaround will likely be required for multiarch builds.
+# The beegfs-csi-driver container requires chwrap to be built and included, so we build it anytime
+# container, push, or push-multiarch are made. Additional prerequisites and the recipes for
+# container and push are defined in release-tools/build.make.
 container: build-chwrap bin/chwrap.tar
+push-multiarch: build-chwrap bin/chwrap.tar
 push: container  # not explicitly executed in release-tools/build.make
 
 # For details on what licenses are disallowed see

Makefile

@@ -4,15 +4,16 @@
 
 # Copyright 2020 NetApp, Inc. All Rights Reserved.
 # Modifications Copyright 2021 NetApp, Inc. All Rights Reserved.
+# Modifications Copyright 2024 ThinkParQ, GmbH. All Rights Reserved.
 # Licensed under the Apache License, Version 2.0.
 
-[ -n "$1" ] && [ -n "$2" ] || exit 1
+[ -n "$1" ] && [ -n "$2" ] && [ -n "$3" ] || exit 1
 
 PREFIX=/tmp/$(uuidgen)
-mkdir -p $PREFIX/netapp
-cp "$1" $PREFIX/netapp/chwrap
+mkdir -p $PREFIX/$3
+cp "$1" $PREFIX/$3/chwrap
 for BIN in beegfs-ctl lsmod modprobe mount touch umount; do
-  ln -s chwrap $PREFIX/netapp/$BIN
+  ln -s chwrap $PREFIX/$3/$BIN
 done
-tar --owner=0 --group=0 -C $PREFIX -cf "$2" netapp
+tar --owner=0 --group=0 -C $PREFIX -cf "$2" $3
 rm -rf $PREFIX

cmd/chwrap/chwrap.sh

@@ -6,12 +6,31 @@
 
 # Use distroless as minimal base image to package the manager binary. Refer to
 # https://github.com/GoogleContainerTools/distroless for more details.
-FROM gcr.io/distroless/static:nonroot
+FROM --platform=$TARGETPLATFORM gcr.io/distroless/static:nonroot
+LABEL maintainers="ThinkParQ"
+LABEL description="BeeGFS CSI Driver Operator"
 LABEL org.opencontainers.image.description="BeeGFS CSI Driver Operator"
 LABEL org.opencontainers.image.source="https://github.com/ThinkParQ/beegfs-csi-driver/operator"
 LABEL org.opencontainers.image.licenses="Apache-2.0"
+
+# Multi-arch images can be built from this Dockerfile. When the container image is built it is
+# expected the controller binary was already created and exists bin/ using Make. By default calling
+# Make with no arguments builds these files for the current architecture with no suffix allowing the
+# container image to be built without multiarch support by default.
+#
+# If Make is called with the `BUILD_PLATFORMS` build argument, a controller binary will be
+# compiled for each platform with an architecture suffix. These can then be used to build a
+# multiarch container image using `docker buildx build` by specifying the same list of platforms
+# using the `--platform` flag. Note the buildx flag and BUILD_PLATFORMS argument accept slightly
+# different values, for example to build for both amd64 and arm64:
+#
+# `make BUILD_PLATFORMS="linux amd64 amd64 amd64;linux arm64 arm64 arm64" all` 
+# `docker buildx build --platform=linux/amd64,linux/arm64`
+ARG TARGETARCH
 WORKDIR /
-COPY bin/manager .
+
+# Copy architecture specific manager to the image.
+COPY bin/manager$TARGETARCH /manager
 USER 65532:65532
 
 ENTRYPOINT ["/manager"]

operator/Dockerfile

@@ -5,6 +5,12 @@
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
 VERSION ?= 1.5.0
 
+# BUILD_PLATFORMS contains a set of tuples [os arch buildx_platform suffix base_image addon_image]
+# separated by semicolon. An empty variable or empty entry (= just a
+# semicolon) builds for the default platform of the current Go
+# toolchain. This approach was adapted from the CSI driver release-tools.1
+BUILD_PLATFORMS =
+
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable")
 # To re-generate a bundle for other specific channels without changing the standard setup, you can:
@@ -110,15 +116,24 @@ test: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v ./... -coverprofile cover.out
 
 ##@ Build
-
 .PHONY: build
 build: generate fmt vet ## Build manager binary.
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/manager main.go
+	mkdir -p bin
+	echo '$(BUILD_PLATFORMS)' | tr ';' '\n' | while read -r os arch buildx_platform suffix base_image addon_image; do \
+		if ! (set -x; CGO_ENABLED=0 GOOS="$$os" GOARCH="$$arch" go build $(GOFLAGS_VENDOR) -a -ldflags \
+		'$(FULL_LDFLAGS)' -o "./bin/manager$$suffix" main.go); then \
+			echo "Building manager for GOOS=$$os GOARCH=$$arch failed, see error(s) above."; \
+			exit 1; \
+		fi; \
+	done
 
 .PHONY: run
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go --zap-devel=true --zap-log-level=5
 
+# Note the Makefile doesn't build multiarch images (only the current architecture).
+# Multiarch images are currently built/published using GitHub actions or manually.
+
 .PHONY: docker-build
 docker-build: test ## Build docker image with the manager.
 	docker build -t ${IMG} .
@@ -249,9 +264,14 @@ PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le
 .PHONY: docker-buildx
 docker-buildx: test ## Build and push docker image for the manager for cross-platform support
 	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
-	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
-	- docker buildx create --name project-v3-builder
-	docker buildx use project-v3-builder
-	- docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross
-	- docker buildx rm project-v3-builder
-	rm Dockerfile.cross
+	# sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+	# - docker buildx create --name project-v3-builder
+	# docker buildx use project-v3-builder
+	# - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross
+	# - docker buildx rm project-v3-builder
+	# rm Dockerfile.cross
+	# This has not been updated to work with how we build multiarch images using GitHub actions.
+	# Fail the target to prevent accidental usage.
+	@echo "Using the docker-buildx target is not currently supported"
+	@exit 1
+

operator/Makefile

@@ -333,6 +333,9 @@ chmod +x install.sh
 * All prerequisites for the BeeGFS CSI driver must be installed on your
   Kubernetes nodes. If you are using Minikube there is a script to do this at
   `hack/minikube_install_driver_prerequisites.sh`.
+   * You must also provide Minikube its own base client configuration file. For example you might
+     bind mount /etc/beegfs from the host OS into the Minikube container using `minikube mount
+     /etc/beegfs:/etc/beegfs` (note the command must stay running for the mount to stay active).
 
 Steps:
 

operator/docs/developer-docs.md

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-export BEEGFS_VERSION=7.3.4
+export BEEGFS_VERSION=7.4.2
 
 # Install the BeeGFS beegfs-ctl tool into the Minikube container:
 minikube ssh "sudo apt-get update"

operator/hack/minikube_install_driver_prerequisites.sh

@@ -18,20 +18,24 @@ aliases:
   # when they are temporarily unable to review PRs.
   kubernetes-csi-reviewers:
   - andyzhangx
+  - carlory
   - chrishenzie
   - ggriffiths
   - gnufied
   - humblec
+  - mauriciopoppe
   - j-griffith
-  - Jiawei0227
   - jingxu97
   - jsafrane
   - pohly
+  - RaunakShah
+  - sunnylovestiramisu
   - xing-yang
 
 # This documents who previously contributed to Kubernetes-CSI
 # as approver.
 emeritus_approvers:
+- Jiawei0227
 - lpabon
 - sbezverk
 - vladimirvivien

release-tools/KUBERNETES_CSI_OWNERS_ALIASES

@@ -17,7 +17,7 @@ The release manager must:
 Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs
 must be updated.
 
-[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the
+[Our CI jobs](https://testgrid.k8s.io/sig-storage-csi-ci) have the
 naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 
 1. Jobs should be actively monitored to find and fix failures in sidecars and
@@ -46,63 +46,59 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 ## Release Process
 1. Identify all issues and ongoing PRs that should go into the release, and
   drive them to resolution.
-1. Download the latest version of the
-   [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes)
-1. Create a
-   [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
-   with `repo:public_repo` access
-1. Generate release notes for the release. Replace arguments with the relevant
-   information.
-    * Clean up old cached information (also needed if you are generating release
-      notes for multiple repos)
-      ```bash
-      rm -rf /tmp/k8s-repo
-      ```
-    * For new minor releases on master:
-        ```bash
-        GITHUB_TOKEN=<token> release-notes \
-          --discover=mergebase-to-latest \
-          --org=kubernetes-csi \
-          --repo=external-provisioner \
-          --required-author="" \
-          --markdown-links \
-          --output out.md
-        ```
-    * For new patch releases on a release branch:
-        ```bash
-        GITHUB_TOKEN=<token> release-notes \
-          --discover=patch-to-latest \
-          --branch=release-1.1 \
-          --org=kubernetes-csi \
-          --repo=external-provisioner \
-          --required-author="" \
-          --markdown-links \
-          --output out.md
-        ```
-1. Compare the generated output to the new commits for the release to check if
-   any notable change missed a release note.
-1. Reword release notes as needed. Make sure to check notes for breaking
-   changes and deprecations.
-1. If release is a new major/minor version, create a new `CHANGELOG-<major>.<minor>.md`
-   file. Otherwise, add the release notes to the top of the existing CHANGELOG
-   file for that minor version.
-1. Submit a PR for the CHANGELOG changes.
-1. Submit a PR for README changes, in particular, Compatibility, Feature status,
-   and any other sections that may need updating.
+1. Update dependencies for sidecars via
+   [go-modules-update.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/HEAD/release-tools/go-modules-update.sh),
+  and get PRs approved and merged.
 1. Check that all [canary CI
-  jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing,
+  jobs](https://testgrid.k8s.io/sig-storage-csi-ci) are passing,
   and that test coverage is adequate for the changes that are going into the release.
+1. Check that the post-\<sidecar\>-push-images builds are succeeding.
+   [Example](https://testgrid.k8s.io/sig-storage-image-build#post-external-snapshotter-push-images)
+1. Generate release notes.
+    1.  Download the latest version of the [K8s release notes generator](https://github.com/kubernetes/release/tree/HEAD/cmd/release-notes)
+    1. Create a
+       [Github personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)
+       with `repo:public_repo` access
+    1. For patch release, use the script generate_patch_release_notes.sh. Read the instructions at the top of the
+       script. The script also creates PRs for each branch.
+    1. For new minor releases, follow these steps and replace arguments with the relevant
+       information.
+        * Clean up old cached information (also needed if you are generating release
+          notes for multiple repos)
+          ```bash
+          rm -rf /tmp/k8s-repo
+          ```
+        * For new minor releases on master:
+            ```bash
+            GITHUB_TOKEN=<token> release-notes \
+              --discover=mergebase-to-latest \
+              --org=kubernetes-csi \
+              --repo=external-provisioner \
+              --required-author="" \
+              --markdown-links \
+              --output out.md
+            ```
+    1. Compare the generated output to the new commits for the release to check if
+       any notable change missed a release note.
+    1. Reword release notes as needed, ideally in the original PRs so that the
+       release notes can be regnerated. Make sure to check notes for breaking
+       changes and deprecations.
+    1. If release is a new major/minor version, create a new `CHANGELOG-<major>.<minor>.md`
+       file.
+    1. Submit a PR for the CHANGELOG changes.
+1. Submit a PR for README changes, in particular, Compatibility, Feature status,
+   and any other sections that may need updating.
 1. Make sure that no new PRs have merged in the meantime, and no PRs are in
    flight and soon to be merged.
 1. Create a new release following a previous release as a template. Be sure to select the correct
    branch. This requires Github release permissions as required by the prerequisites.
    [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new)
 1. If release was a new major/minor version, create a new `release-<minor>`
    branch at that commit.
-1. Check [image build status](https://k8s-testgrid.appspot.com/sig-storage-image-build).
-1. Promote images from k8s-staging-sig-storage to k8s.gcr.io/sig-storage. From
+1. Check [image build status](https://testgrid.k8s.io/sig-storage-image-build).
+1. Promote images from k8s-staging-sig-storage to registry.k8s.io/sig-storage. From
    the [k8s image
-   repo](https://github.com/kubernetes/k8s.io/tree/HEAD/k8s.gcr.io/images/k8s-staging-sig-storage),
+   repo](https://github.com/kubernetes/k8s.io/tree/HEAD/registry.k8s.io/images/k8s-staging-sig-storage),
    run `./generate.sh > images.yaml`, and send a PR with the updated images.
    Once merged, the image promoter will copy the images from staging to prod.
 1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar
@@ -118,7 +114,7 @@ naming convention `<hostpath-deployment-version>-on-<kubernetes-version>`.
 
 The following jobs are triggered after tagging to produce the corresponding
 image(s):
-https://k8s-testgrid.appspot.com/sig-storage-image-build
+https://testgrid.k8s.io/sig-storage-image-build
 
 Clicking on a failed build job opens that job in https://prow.k8s.io. Next to
 the job title is a rerun icon (circle with arrow). Clicking it opens a popup