From 8a992f8b6a692db2b6cf34e120a929ce069b586c Mon Sep 17 00:00:00 2001 From: Michael Kruggel <108417058+Michael-Kruggel@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:40:53 -0700 Subject: [PATCH] feat: config to enable resilient registry (#2440) ## Description - Provided fields for affinity and toleration of the registry pod - Added s3 irsa configurability for the docker registry helm chart if using a compatible image ## Related Issue Relates to #375 ## Checklist before merging - [x] Test, docs, adr added or updated as needed - [x] [Contributor Guide Steps](https://github.com/defenseunicorns/zarf/blob/main/.github/CONTRIBUTING.md#developer-workflow) followed --------- Co-authored-by: Zack A Co-authored-by: corang Co-authored-by: Zack Annexstein Co-authored-by: Lucas Rodriguez Co-authored-by: Lucas Rodriguez Co-authored-by: razzle Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- .../chart/templates/_helpers.tpl | 42 +++++++++++++++++++ .../chart/templates/deployment.yaml | 11 +++++ .../zarf-registry/chart/templates/hpa.yaml | 5 +++ .../chart/templates/serviceaccount.yaml | 13 ++++++ packages/zarf-registry/chart/values.yaml | 12 ++++++ packages/zarf-registry/registry-values.yaml | 19 ++++++++- packages/zarf-registry/zarf.yaml | 35 ++++++++++++++++ site/src/content/docs/ref/init-package.mdx | 32 +++++++++++++- src/test/external/docker-registry-values.yaml | 2 +- 9 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 packages/zarf-registry/chart/templates/serviceaccount.yaml diff --git a/packages/zarf-registry/chart/templates/_helpers.tpl b/packages/zarf-registry/chart/templates/_helpers.tpl index 3e570d3078..76cd228c13 100644 --- a/packages/zarf-registry/chart/templates/_helpers.tpl +++ b/packages/zarf-registry/chart/templates/_helpers.tpl @@ -23,6 +23,37 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- end -}} {{- end -}} +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "docker-registry.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "docker-registry.selectorLabels" -}} +app.kubernetes.io/name: {{ include "docker-registry.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "docker-registry.labels" -}} +{{ include "docker-registry.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/part-of: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ include "docker-registry.chart" . }} +{{- with .Values.customLabels }} +{{ toYaml . }} +{{- end }} +{{- end -}} + {{/* Merge all configmaps */}} @@ -34,3 +65,14 @@ Merge all configmaps {{ .Values.caBundle | indent 6 }} {{- end }} {{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "docker-registry.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "docker-registry.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/packages/zarf-registry/chart/templates/deployment.yaml b/packages/zarf-registry/chart/templates/deployment.yaml index ce8126f60a..e0e878eb82 100644 --- a/packages/zarf-registry/chart/templates/deployment.yaml +++ b/packages/zarf-registry/chart/templates/deployment.yaml @@ -26,6 +26,7 @@ spec: annotations: checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} spec: + serviceAccountName: {{ include "docker-registry.serviceAccountName" . }} {{- if .Values.imagePullSecrets }} imagePullSecrets: {{ toYaml .Values.imagePullSecrets | indent 8 }} @@ -83,7 +84,11 @@ spec: subPath: ca-certificates.crt readOnly: true {{- end }} +{{- if .Values.affinity.enabled }} affinity: +{{- if .Values.affinity.custom }} +{{ toYaml .Values.affinity.custom | indent 8 }} +{{- else }} {{- if (eq "ReadWriteMany" .Values.persistence.accessMode) }} podAntiAffinity: {{- else }} @@ -99,6 +104,12 @@ spec: values: - {{ template "docker-registry.name" . }} topologyKey: kubernetes.io/hostname +{{- end }} +{{- end }} +{{- if .Values.tolerations}} + tolerations: +{{ toYaml .Values.tolerations | indent 8 }} +{{- end }} volumes: - name: config secret: diff --git a/packages/zarf-registry/chart/templates/hpa.yaml b/packages/zarf-registry/chart/templates/hpa.yaml index cdb4468872..7045a709d3 100644 --- a/packages/zarf-registry/chart/templates/hpa.yaml +++ b/packages/zarf-registry/chart/templates/hpa.yaml @@ -13,8 +13,13 @@ spec: apiVersion: apps/v1 kind: Deployment name: {{ template "docker-registry.fullname" . }} +{{- if .Values.autoscaling.mapReplicasToNodes }} + minReplicas: {{ len (lookup "v1" "Node" "" "") }} + maxReplicas: {{ add (len (lookup "v1" "Node" "" "")) 4 }} +{{- else }} minReplicas: {{ .Values.autoscaling.minReplicas }} maxReplicas: {{ .Values.autoscaling.maxReplicas }} +{{- end }} metrics: - type: Resource resource: diff --git a/packages/zarf-registry/chart/templates/serviceaccount.yaml b/packages/zarf-registry/chart/templates/serviceaccount.yaml new file mode 100644 index 0000000000..7103f59fd9 --- /dev/null +++ b/packages/zarf-registry/chart/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: {{ .Values.namespace | default .Release.Namespace }} + name: {{ template "docker-registry.serviceAccountName" . }} + labels: + {{- include "docker-registry.labels" . | nindent 4 }} +{{- if .Values.serviceAccount.annotations }} + annotations: +{{ toYaml .Values.serviceAccount.annotations | indent 4 }} +{{- end }} +{{- end -}} diff --git a/packages/zarf-registry/chart/values.yaml b/packages/zarf-registry/chart/values.yaml index 527578eca0..77a88650e1 100644 --- a/packages/zarf-registry/chart/values.yaml +++ b/packages/zarf-registry/chart/values.yaml @@ -48,8 +48,15 @@ secrets: podDisruptionBudget: minAvailable: 1 +affinity: + enabled: true + custom: {} + +tolerations: [] + autoscaling: enabled: true + mapReplicasToNodes: false minReplicas: 1 maxReplicas: 5 targetCPUUtilizationPercentage: 80 @@ -75,3 +82,8 @@ extraEnvVars: [] ## Additional ENV variables to set # - name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY # value: "/var/lib/example" + +serviceAccount: + create: false + name: "" + annotations: {} diff --git a/packages/zarf-registry/registry-values.yaml b/packages/zarf-registry/registry-values.yaml index 891cb9a891..98a02f38fc 100644 --- a/packages/zarf-registry/registry-values.yaml +++ b/packages/zarf-registry/registry-values.yaml @@ -34,14 +34,31 @@ fullnameOverride: "zarf-docker-registry" podLabels: zarf.dev/agent: "ignore" +affinity: + enabled: ###ZARF_VAR_REGISTRY_AFFINITY_ENABLE### + custom: + ###ZARF_VAR_REGISTRY_AFFINITY_CUSTOM### + +tolerations: + ###ZARF_VAR_REGISTRY_TOLERATIONS### + autoscaling: enabled: ###ZARF_VAR_REGISTRY_HPA_ENABLE### + mapReplicasToNodes: ###ZARF_VAR_REGISTRY_HPA_AUTO_SIZE### minReplicas: "###ZARF_VAR_REGISTRY_HPA_MIN###" maxReplicas: "###ZARF_VAR_REGISTRY_HPA_MAX###" - targetCPUUtilizationPercentage: 80 + targetCPUUtilizationPercentage: ###ZARF_VAR_REGISTRY_HPA_TARGET_CPU### caBundle: | ###ZARF_VAR_REGISTRY_CA_BUNDLE### extraEnvVars: ###ZARF_VAR_REGISTRY_EXTRA_ENVS### + +serviceAccount: + # Specifies whether a service account should be created + create: ###ZARF_VAR_REGISTRY_CREATE_SERVICE_ACCOUNT### + # The name of the service account to use. If name not set and create is true, a name is generated using fullname template + name: "###ZARF_VAR_REGISTRY_SERVICE_ACCOUNT_NAME###" + annotations: + ###ZARF_VAR_REGISTRY_SERVICE_ACCOUNT_ANNOTATIONS### diff --git a/packages/zarf-registry/zarf.yaml b/packages/zarf-registry/zarf.yaml index 38a13006c1..2da03ccda6 100644 --- a/packages/zarf-registry/zarf.yaml +++ b/packages/zarf-registry/zarf.yaml @@ -58,6 +58,41 @@ variables: default: "" autoIndent: true + - name: REGISTRY_CREATE_SERVICE_ACCOUNT + description: Toggle the creation of a new service account for the registry + default: "false" + + - name: REGISTRY_SERVICE_ACCOUNT_NAME + description: The name of the service account to use. If not set and create is true, a name is generated using fullname template + default: "" + + - name: REGISTRY_SERVICE_ACCOUNT_ANNOTATIONS + description: Map of annotations to add to the created service account + default: "" + autoIndent: true + + - name: REGISTRY_AFFINITY_ENABLE + description: Enable pod affinity for the registry + default: "true" + + - name: REGISTRY_AFFINITY_CUSTOM + description: Custom pod affinity yaml block for the registry + default: "" + autoIndent: true + + - name: REGISTRY_TOLERATIONS + description: Custom tolerations array for the registry + default: "" + autoIndent: true + + - name: REGISTRY_HPA_AUTO_SIZE + description: Enable to set min and max replicas based on amount of nodes + default: "false" + + - name: REGISTRY_HPA_TARGET_CPU + description: The target CPU utilization percentage for the registry + default: "80" + constants: - name: REGISTRY_IMAGE value: "###ZARF_PKG_TMPL_REGISTRY_IMAGE###" diff --git a/site/src/content/docs/ref/init-package.mdx b/site/src/content/docs/ref/init-package.mdx index 62954eb6f5..11e07ca97c 100644 --- a/site/src/content/docs/ref/init-package.mdx +++ b/site/src/content/docs/ref/init-package.mdx @@ -104,6 +104,36 @@ The `registry:2` image and the Zarf Agent image can be configured with a custom ::: +## Making the Registry Highly-Available + +By default, the registry included in the init package creates a ReadWriteOnce PVC and is only scheduled to run on one node at a time. This setup is usually enough for smaller and simpler deployments. However, for larger deployments or those where nodes are frequently restarted or updated, you may want to make the registry highly-available. + +This approach requires certain prerequisites, such as a storage class that supports ReadWriteMany, or being in an environment that allows you to configure the registry to use an S3-compatible backend. Additionally, you must provide custom configuration to the registry to ensure it is distributed across all nodes and has the appropriate number of replicas. Below is an example [configuration file](/ref/config-files/) using a ReadWriteMany storage class: + +```yaml +# zarf-config.yaml +package: + deploy: + set: + REGISTRY_PVC_ENABLED: "true" + REGISTRY_PVC_ACCESS_MODE: "ReadWriteMany" + REGISTRY_HPA_AUTO_SIZE: "true" + REGISTRY_AFFINITY_CUSTOM: | + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - docker-registry + topologyKey: kubernetes.io/hostname +``` + +Notably, the `REGISTRY_AFFINITY_CUSTOM` variable overrides the default pod anti-affinity, and `REGISTRY_HPA_AUTO_SIZE` automatically adjusts the minimum and maximum replicas for the registry based on the number of nodes in the cluster. If you prefer to manually set the minimum and maximum replicas, you can use `REGISTRY_HPA_MIN` and `REGISTRY_HPA_MAX` to specify the desired values. + ## The `zarf init` Lifecycle The `zarf init` lifecycle is _very similar_ to the [`zarf package deploy` lifecycle](/ref/deploy/) except that it sets up resources specific to Zarf such as the `zarf-state` and performs special actions such as the injection procedure. @@ -123,7 +153,7 @@ graph TD B12(prompt to confirm components)-->B13 B13(prompt to choose components in '.group')-->B14 - subgraph + subgraph B52 --> |Yes|B14(deploy each component)-->B14 B14 --> B15{Component is \n zarf-seed-registry} B15 --> |Yes|B51(initialize zarf-state secret):::action diff --git a/src/test/external/docker-registry-values.yaml b/src/test/external/docker-registry-values.yaml index 7de626eabc..65b8a1722c 100644 --- a/src/test/external/docker-registry-values.yaml +++ b/src/test/external/docker-registry-values.yaml @@ -3,7 +3,7 @@ image: tag: 2.8.3 pullPolicy: IfNotPresent imagePullSecrets: - - name: private-registry + - name: private-registry service: name: registry