Add custom CA support for buildkit/pro-builder

These are breaking changes to allow easier management of the images, and root/rootless settings. Signed-off-by: Alex Ellis (OpenFaaS Ltd) <[email protected]>
openfaas · Nov 20, 2024 · e3afd14 · e3afd14
1 parent d76026c
commit e3afd14
Show file tree

Hide file tree

Showing 3 changed files with 119 additions and 23 deletions.
diff --git a/chart/pro-builder/templates/buildkit-toml-cfg.yml b/chart/pro-builder/templates/buildkit-toml-cfg.yml
@@ -0,0 +1,12 @@
+{{- if .Values.buildkit.config}}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: buildkit-config
+  namespace: {{ .Release.Namespace }}
+data:
+  buildkitd.toml: |
+{{ .Values.buildkit.config | indent 4 }}
+
+{{- end}}
diff --git a/chart/pro-builder/templates/deployment.yml b/chart/pro-builder/templates/deployment.yml
@@ -1,7 +1,7 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-{{- if .Values.buildkit.rootless }}
+{{- if eq .Values.buildkit.mode "rootless" }}
   annotations:
     container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined
 {{- end }}
@@ -36,6 +36,20 @@ spec:
         component: pro-builder
     spec:
       volumes:
+
+{{- if .Values.buildkit.config }}
+        - name: buildkit-config
+          configMap:
+           name: buildkit-config
+{{- end }}
+
+# If given, mount buildkit caSecret as /var/var/run/registry-tls/
+{{- if .Values.buildkit.caSecret }}
+        - name: registry-tls
+          secret:
+            secretName: {{ .Values.buildkit.caSecret }}
+{{- end }}
+
         - name: client-certs
           secret:
             secretName: buildkit-client-certs
@@ -146,27 +160,67 @@ spec:
           - "--tlscert=/var/secrets/certs/server.crt"
           - "--tlskey=/var/secrets/certs/server.key"
           - "--tlscacert=/var/secrets/certs/ca.crt"
-{{- if .Values.buildkit.rootless }}
+{{- if eq .Values.buildkit.mode "rootless" }}
           - "--oci-worker-no-process-sandbox"
+          - "--config=/home/user/.config/buildkit/buildkitd.toml"
 {{- end }}
-        image: {{ .Values.buildkit.image }}
+
+{{- if eq .Values.buildkit.mode "rootless" }}
+        image: {{ .Values.buildkit.rootless.image }}
+{{- else }}
+        image: {{ .Values.buildkit.root.image }}
+{{- end }}
+
         imagePullPolicy: {{ .Values.imagePullPolicy | quote }}
         ports:
         - containerPort: 1234
           protocol: TCP
         resources:
           {{- .Values.buildkit.resources | toYaml | nindent 12 }}
-        {{- with .Values.buildkit.securityContext }}
+
+{{- if eq .Values.buildkit.mode "rootless" }}
+        {{- with .Values.buildkit.rootless.securityContext }}
         securityContext:
           {{- . | toYaml | nindent 12 }}
         {{- end }}
+{{- else }}
+        {{- with .Values.buildkit.root.securityContext }}
+        securityContext:
+          {{- . | toYaml | nindent 12 }}
+        {{- end }}
+{{- end }}
+
         volumeMounts:
         - name: daemon-certs
           readOnly: true
           mountPath: /var/secrets/certs
         - name: buildkit-workspace
           mountPath: /tmp/
           readOnly: false
+
+{{- if .Values.buildkit.config }}
+
+# If rootless, add buildkit-config ConfigMap to: ~/.config/buildkit/buildkitd
+{{- if eq .Values.buildkit.mode "rootless" }}
+        - name: buildkit-config
+          mountPath: /home/user/.config/buildkit
+{{- else }}
+# If rootful, add buildkit-config ConfigMap to: /etc/buildkit/buildkitd.toml
+        - name: buildkit-config
+          mountPath: /etc/buildkit
+{{- end }}
+{{- end }}
+
+# If rootless, mount CA to /home/user/.config/buildkit-tls
+{{- if eq .Values.buildkit.mode "rootless" }}
+        - name: registry-tls
+          mountPath: /home/user/.config/buildkit-tls
+{{- else }}
+# If rootful, mount CA to /var/run/registry-tls/
+        - name: registry-tls
+          mountPath: /var/run/registry-tls/
+{{- end }}
+
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

diff --git a/chart/pro-builder/values.yaml b/chart/pro-builder/values.yaml
@@ -27,32 +27,62 @@ proBuilder:
 # Both configurations are "rootless", however the rootless: true mode does not
 # require Buildkit to run as a privileged container and is preferred.
 buildkit:
+
+  # "mode" can be set to "rootless" or "root"
+  #
+  # "rootless" (preferred)
+  # If the Kubernetes node's OS/configuration, Kernel and Kubernetes
+  # support it, rootless mode runs without needing root or a privileged container.
+  #
+  # "root" (fallback)
   # A configuration which uses a privileged container for when 
   # your nodes have issues running in rootless mode
   #
   # Use rootless if possible, and if not, set up a dedicated 
   # nodepool for the function builder pods, which is recycled often
   # through the use of spot instances or preemptive VMs.
+
+  mode: rootless
+
+  rootless:
+    image: moby/buildkit:v0.15.1-rootless
+    securityContext:
+      seccompProfile:
+        type: Unconfined
+      runAsUser: 1000
+      runAsGroup: 1000
+      privileged: false
+
+  root:
+    image: moby/buildkit:v0.15.1
+    securityContext:
+      runAsUser: 0
+      runAsGroup: 0
+      privileged: true
+
+  # Custom CA certificates for the registry / and custom buildkit configuration
+
+  # Provide the name of the secret containing the CA certificate
+  # for a self-signed registry, when pushing or pulling images
+  #
+  # kubectl create secret generic -n openfaas \
+  #   registry-tls --from-file=ca.crt=ca.crt
+  # caSecret: "registry-tls"
+  caSecret: ""
+
+  # Provide a custom buildkit configuration, ideal for setting up
+  # a custom CA for a registry, or other advanced configuration.
   #
-  # image: moby/buildkit:v0.15.1
-  # rootless: false
-  # securityContext:
-  #   runAsUser: 0
-  #   runAsGroup: 0
-  #   privileged: true
-
-  # For a rootless configuration, preferred, if the configuration
-  # and Kernel version of your Kubernetes nodes supports it
-  # 
-  image: moby/buildkit:v0.15.1-rootless
-  rootless: true
-  securityContext:
-    # Needs Kubernetes >= 1.19
-    seccompProfile:
-      type: Unconfined
-    runAsUser: 1000
-    runAsGroup: 1000
-    privileged: false
+  # Reference: https://docs.docker.com/build/buildkit/toml-configuration/
+  # Config for rootless mode, reads the CA from the home directory
+  # config: |
+  #   [registry."registry-service:443"]
+  #     ca=["/home/user/.config/buildkit-tls/ca.crt"]
+
+  # The config for root mode, reads the CA from the /var/run/registry-tls directory
+  config: |
+    [registry."registry-service:443"]
+      ca=["/var/run/registry-tls/ca.crt"]
 
   resources:
     requests: