From e3afd14025419592690d146046dc71cd5349499f Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Wed, 20 Nov 2024 12:39:50 +0000 Subject: [PATCH] Add custom CA support for buildkit/pro-builder These are breaking changes to allow easier management of the images, and root/rootless settings. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- .../templates/buildkit-toml-cfg.yml | 12 ++++ chart/pro-builder/templates/deployment.yml | 62 +++++++++++++++-- chart/pro-builder/values.yaml | 68 +++++++++++++------ 3 files changed, 119 insertions(+), 23 deletions(-) create mode 100644 chart/pro-builder/templates/buildkit-toml-cfg.yml diff --git a/chart/pro-builder/templates/buildkit-toml-cfg.yml b/chart/pro-builder/templates/buildkit-toml-cfg.yml new file mode 100644 index 000000000..a960058ad --- /dev/null +++ b/chart/pro-builder/templates/buildkit-toml-cfg.yml @@ -0,0 +1,12 @@ +{{- if .Values.buildkit.config}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: buildkit-config + namespace: {{ .Release.Namespace }} +data: + buildkitd.toml: | +{{ .Values.buildkit.config | indent 4 }} + +{{- end}} diff --git a/chart/pro-builder/templates/deployment.yml b/chart/pro-builder/templates/deployment.yml index 28b16d3e8..ca3c50c4a 100644 --- a/chart/pro-builder/templates/deployment.yml +++ b/chart/pro-builder/templates/deployment.yml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: -{{- if .Values.buildkit.rootless }} +{{- if eq .Values.buildkit.mode "rootless" }} annotations: container.apparmor.security.beta.kubernetes.io/buildkitd: unconfined {{- end }} @@ -36,6 +36,20 @@ spec: component: pro-builder spec: volumes: + +{{- if .Values.buildkit.config }} + - name: buildkit-config + configMap: + name: buildkit-config +{{- end }} + +# If given, mount buildkit caSecret as /var/var/run/registry-tls/ +{{- if .Values.buildkit.caSecret }} + - name: registry-tls + secret: + secretName: {{ .Values.buildkit.caSecret }} +{{- end }} + - name: client-certs secret: secretName: buildkit-client-certs @@ -146,20 +160,36 @@ spec: - "--tlscert=/var/secrets/certs/server.crt" - "--tlskey=/var/secrets/certs/server.key" - "--tlscacert=/var/secrets/certs/ca.crt" -{{- if .Values.buildkit.rootless }} +{{- if eq .Values.buildkit.mode "rootless" }} - "--oci-worker-no-process-sandbox" + - "--config=/home/user/.config/buildkit/buildkitd.toml" {{- end }} - image: {{ .Values.buildkit.image }} + +{{- if eq .Values.buildkit.mode "rootless" }} + image: {{ .Values.buildkit.rootless.image }} +{{- else }} + image: {{ .Values.buildkit.root.image }} +{{- end }} + imagePullPolicy: {{ .Values.imagePullPolicy | quote }} ports: - containerPort: 1234 protocol: TCP resources: {{- .Values.buildkit.resources | toYaml | nindent 12 }} - {{- with .Values.buildkit.securityContext }} + +{{- if eq .Values.buildkit.mode "rootless" }} + {{- with .Values.buildkit.rootless.securityContext }} securityContext: {{- . | toYaml | nindent 12 }} {{- end }} +{{- else }} + {{- with .Values.buildkit.root.securityContext }} + securityContext: + {{- . | toYaml | nindent 12 }} + {{- end }} +{{- end }} + volumeMounts: - name: daemon-certs readOnly: true @@ -167,6 +197,30 @@ spec: - name: buildkit-workspace mountPath: /tmp/ readOnly: false + +{{- if .Values.buildkit.config }} + +# If rootless, add buildkit-config ConfigMap to: ~/.config/buildkit/buildkitd +{{- if eq .Values.buildkit.mode "rootless" }} + - name: buildkit-config + mountPath: /home/user/.config/buildkit +{{- else }} +# If rootful, add buildkit-config ConfigMap to: /etc/buildkit/buildkitd.toml + - name: buildkit-config + mountPath: /etc/buildkit +{{- end }} +{{- end }} + +# If rootless, mount CA to /home/user/.config/buildkit-tls +{{- if eq .Values.buildkit.mode "rootless" }} + - name: registry-tls + mountPath: /home/user/.config/buildkit-tls +{{- else }} +# If rootful, mount CA to /var/run/registry-tls/ + - name: registry-tls + mountPath: /var/run/registry-tls/ +{{- end }} + {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/chart/pro-builder/values.yaml b/chart/pro-builder/values.yaml index 02592e5e1..5b9ca1672 100644 --- a/chart/pro-builder/values.yaml +++ b/chart/pro-builder/values.yaml @@ -27,32 +27,62 @@ proBuilder: # Both configurations are "rootless", however the rootless: true mode does not # require Buildkit to run as a privileged container and is preferred. buildkit: + + # "mode" can be set to "rootless" or "root" + # + # "rootless" (preferred) + # If the Kubernetes node's OS/configuration, Kernel and Kubernetes + # support it, rootless mode runs without needing root or a privileged container. + # + # "root" (fallback) # A configuration which uses a privileged container for when # your nodes have issues running in rootless mode # # Use rootless if possible, and if not, set up a dedicated # nodepool for the function builder pods, which is recycled often # through the use of spot instances or preemptive VMs. + + mode: rootless + + rootless: + image: moby/buildkit:v0.15.1-rootless + securityContext: + seccompProfile: + type: Unconfined + runAsUser: 1000 + runAsGroup: 1000 + privileged: false + + root: + image: moby/buildkit:v0.15.1 + securityContext: + runAsUser: 0 + runAsGroup: 0 + privileged: true + + # Custom CA certificates for the registry / and custom buildkit configuration + + # Provide the name of the secret containing the CA certificate + # for a self-signed registry, when pushing or pulling images + # + # kubectl create secret generic -n openfaas \ + # registry-tls --from-file=ca.crt=ca.crt + # caSecret: "registry-tls" + caSecret: "" + + # Provide a custom buildkit configuration, ideal for setting up + # a custom CA for a registry, or other advanced configuration. # - # image: moby/buildkit:v0.15.1 - # rootless: false - # securityContext: - # runAsUser: 0 - # runAsGroup: 0 - # privileged: true - - # For a rootless configuration, preferred, if the configuration - # and Kernel version of your Kubernetes nodes supports it - # - image: moby/buildkit:v0.15.1-rootless - rootless: true - securityContext: - # Needs Kubernetes >= 1.19 - seccompProfile: - type: Unconfined - runAsUser: 1000 - runAsGroup: 1000 - privileged: false + # Reference: https://docs.docker.com/build/buildkit/toml-configuration/ + # Config for rootless mode, reads the CA from the home directory + # config: | + # [registry."registry-service:443"] + # ca=["/home/user/.config/buildkit-tls/ca.crt"] + + # The config for root mode, reads the CA from the /var/run/registry-tls directory + config: | + [registry."registry-service:443"] + ca=["/var/run/registry-tls/ca.crt"] resources: requests: