feat: implement a registry proxy

.github/workflows/test-proxy-kind-nftables.yaml

@@ -0,0 +1,4 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  kubeProxyMode: "nftables"

.github/workflows/test-proxy.yml

@@ -0,0 +1,107 @@
+name: Test Cross-Cluster
+on:
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - "**.jpg"
+      - "**.png"
+      - "**.gif"
+      - "**.svg"
+      - "adr/**"
+      - "docs/**"
+      - "CODEOWNERS"
+  merge_group:
+    paths-ignore:
+      - "**.md"
+      - "**.jpg"
+      - "**.png"
+      - "**.gif"
+      - "**.svg"
+      - "adr/**"
+      - "docs/**"
+      - "CODEOWNERS"
+
+permissions:
+  contents: read
+
+# Abort prior jobs in the same workflow / PR
+concurrency:
+  group: e2e-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Build the binary and init package
+  build-zarf:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup golang
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version-file: go.mod
+
+      - name: Build binary and zarf packages
+        uses: ./.github/actions/packages
+        with:
+          build-examples: "false"
+
+      # Upload the contents of the build directory for later stages to use
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: build-artifacts
+          path: build/
+          retention-days: 1
+
+  # Run the tests on kind
+  validate-nftables:
+    runs-on: ubuntu-latest
+    needs: build-zarf
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: build-artifacts
+          path: build/
+
+      - name: Setup golang
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version-file: go.mod
+
+      - name: setup kind config
+        run: |
+          cat <<EOF > kind-nftables-config.yml
+          kind: Cluster
+          apiVersion: kind.x-k8s.io/v1alpha4
+          networking:
+            kubeProxyMode: "nftables"
+          EOF
+
+
+      - name: Setup Kind
+        run: |
+          kind delete cluster && kind create cluster --config=kind-nftables-config.yml
+
+      - name: Make Zarf executable
+        run: |
+          chmod +x build/zarf
+
+      - name: Run tests
+        run: |
+          zarf init --host-network --confirm
+
+      - name: get cluster info
+        uses: ./.github/actions/debug-cluster
+        if: always()
+
+      - name: Save logs
+        if: always()
+        uses: ./.github/actions/save-logs
+        with:
+          suffix: -validate-kind

Makefile

@@ -122,7 +122,7 @@ build-local-agent-image: ## Build the Zarf agent image to be used in a locally b
 
 init-package: ## Create the zarf init package (must `brew install coreutils` on macOS and have `docker` first)
 	@test -s $(ZARF_BIN) || $(MAKE)
-	$(ZARF_BIN) package create -o build -a $(ARCH) --confirm .
+	$(ZARF_BIN) package create -o build -a $(ARCH) --confirm --skip-sbom .
 
 # INTERNAL: used to build a release version of the init package with a specific agent image
 release-init-package:

kind-config.yaml

@@ -0,0 +1,32 @@
+# kind: Cluster
+# apiVersion: kind.x-k8s.io/v1alpha4
+# networking:
+#   ipFamily: ipv6
+
+
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  kubeProxyMode: "nftables"
+  ipFamily: ipv6
+
+# Calico setup
+# kind: Cluster
+# apiVersion: kind.x-k8s.io/v1alpha4
+# nodes:
+# - role: control-plane
+# - role: worker
+# - role: worker
+# networking:
+#   disableDefaultCNI: true
+#   podSubnet: 192.168.0.0/16
+
+## Get nodeip
+## pick the first node (kind is single-node unless you asked for more)
+# NODE=$(kubectl get nodes -o name | head -1)
+
+# # grab its InternalIP that looks like an IPv6 address
+# NODE_IP=$(kubectl get $NODE \
+#   -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+
+# echo "Node IP is: $NODE_IP"

packages/zarf-registry/chart/templates/daemonset.yaml

@@ -0,0 +1,47 @@
+{{ if eq .Values.service.registryProxy "true" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: zarf-registry-proxy
+spec:
+  selector:
+    matchLabels:
+      app: zarf-registry-proxy
+  template:
+    metadata:
+      labels:
+        app: zarf-registry-proxy
+        # Don't mutate this pod
+        zarf.dev/agent: ignore
+    spec:
+      containers:
+        - args:
+          {{ if eq .Values.service.ipv6Only "true" }}
+            - tcp6-listen:{{ .Values.service.nodePort }},bind=[::1],reuseaddr,fork
+            - tcp6:{{ template "docker-registry.fullname" . }}:5000
+          {{- else }}
+            - tcp4-listen:{{ .Values.service.nodePort }},reuseaddr,fork
+            - tcp4:{{ template "docker-registry.fullname" . }}:5000
+          {{- end }}
+          image: "{{ .Values.proxy.image.repository }}:{{ .Values.proxy.image.tag }}"
+          imagePullPolicy: IfNotPresent
+          name: zarf-registry-proxy
+          ports:
+            - containerPort: {{ .Values.service.nodePort }}
+            {{ if ne .Values.service.ipv6Only "true" }}
+              hostPort: {{ .Values.service.nodePort }}
+              hostIP: "127.0.0.1"
+            {{- end }}
+          resources:
+            limits:
+              cpu: 100m
+              memory: 200Mi
+            requests:
+              cpu: 100m
+              memory: 200Mi
+    # IPV6 does not support rewriting packets to localhost so it needs to use host network over hostport
+    {{ if eq .Values.service.ipv6Only "true" }}
+      dnsPolicy: ClusterFirstWithHostNet
+      hostNetwork: true
+    {{- end }}
+{{- end }}

packages/zarf-registry/chart/templates/service.yaml

@@ -9,13 +9,20 @@ metadata:
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
 spec:
-  type: {{ .Values.service.type }}
+  {{- if eq .Values.service.registryProxy "false" }}
+  type: NodePort
+  {{- else }}
+  type: ClusterIP
+  {{- end }}
+  ipFamilyPolicy: PreferDualStack
   ports:
     - port: {{ .Values.service.port }}
       protocol: TCP
       name: {{ if .Values.tlsSecretName }}https{{ else }}http{{ end }}-{{ .Values.service.port }}
       targetPort: 5000
+      {{- if eq .Values.service.registryProxy "false" }}
       nodePort: {{ .Values.service.nodePort }}
+      {{- end }}
   selector:
     app: {{ template "docker-registry.name" . }}
     release: {{ .Release.Name }}

packages/zarf-registry/chart/values.yaml

@@ -6,7 +6,14 @@ image:
   repository: registry
   tag: 2.8.3
 
+proxy:
+  image:
+    repository: alpine/socat
+    tag: 1.8.0.3
+
 service:
+  ipv6Only: "false"
+  registryProxy: "false"
   name: registry
   type: NodePort
   port: 5000

packages/zarf-registry/registry-values.yaml

@@ -12,14 +12,21 @@ image:
 imagePullSecrets:
   - name: private-registry
 
+proxy:
+  image:
+    repository: "###ZARF_SEED_REGISTRY###/###ZARF_CONST_REGISTRY_PROXY_IMAGE###"
+    tag: "###ZARF_CONST_REGISTRY_PROXY_IMAGE_TAG###"
+
 secrets:
   htpasswd: "###ZARF_HTPASSWD###"
   configData:
     http:
       secret: "###ZARF_REGISTRY_SECRET###"
 
 service:
+  ipv6Only: "###ZARF_IPV6_ONLY###"
   nodePort: "###ZARF_NODEPORT###"
+  registryProxy: "###ZARF_REGISTRY_PROXY###"
 
 resources:
   requests:

packages/zarf-registry/zarf.yaml

@@ -100,6 +100,12 @@ constants:
   - name: REGISTRY_IMAGE_TAG
     value: "###ZARF_PKG_TMPL_REGISTRY_IMAGE_TAG###"
 
+  - name: REGISTRY_PROXY_IMAGE
+    value: "###ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE###"
+
+  - name: REGISTRY_PROXY_IMAGE_TAG
+    value: "###ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE_TAG###"
+
 components:
   - name: zarf-injector
     description: |
@@ -111,9 +117,9 @@ components:
         architecture: amd64
     files:
       # Rust Injector Binary
-      - source: https://zarf-init-resources.s3.us-east-1.amazonaws.com/injector/###ZARF_PKG_TMPL_INJECTOR_VERSION###/zarf-injector-amd64
+      - source: ../../src/injector/target/x86_64-unknown-linux-musl/release/zarf-injector
         target: "###ZARF_TEMP###/zarf-injector"
-        shasum: "###ZARF_PKG_TMPL_INJECTOR_AMD64_SHASUM###"
+        # shasum: "###ZARF_PKG_TMPL_INJECTOR_AMD64_SHASUM###"
         executable: true
 
   - name: zarf-injector
@@ -146,6 +152,7 @@ components:
     images:
       # The seed image (or images) that will be injected (see zarf-config.toml)
       - "###ZARF_PKG_TMPL_REGISTRY_IMAGE_DOMAIN######ZARF_PKG_TMPL_REGISTRY_IMAGE###:###ZARF_PKG_TMPL_REGISTRY_IMAGE_TAG###"
+      - "###ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE_DOMAIN######ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE###:###ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE_TAG###"
 
   - name: zarf-registry
     description: |
@@ -171,3 +178,4 @@ components:
     images:
       # This image (or images) must match that used for injection (see zarf-config.toml)
       - "###ZARF_PKG_TMPL_REGISTRY_IMAGE_DOMAIN######ZARF_PKG_TMPL_REGISTRY_IMAGE###:###ZARF_PKG_TMPL_REGISTRY_IMAGE_TAG###"
+      - "###ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE_DOMAIN######ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE###:###ZARF_PKG_TMPL_REGISTRY_PROXY_IMAGE_TAG###"

src/cmd/initialize.go

@@ -54,6 +54,7 @@ func newInitCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&config.CommonOptions.Confirm, "confirm", false, lang.CmdInitFlagConfirm)
 	cmd.Flags().StringVar(&pkgConfig.PkgOpts.OptionalComponents, "components", v.GetString(VInitComponents), lang.CmdInitFlagComponents)
 	cmd.Flags().StringVar(&pkgConfig.InitOpts.StorageClass, "storage-class", v.GetString(VInitStorageClass), lang.CmdInitFlagStorageClass)
+	cmd.Flags().BoolVar(&pkgConfig.InitOpts.RegistryProxy, "registry-proxy", false, "uses the registry-proxy solution")
 
 	// Flags for using an external Git server
 	cmd.Flags().StringVar(&pkgConfig.InitOpts.GitServer.Address, "git-url", v.GetString(VInitGitURL), lang.CmdInitFlagGitURL)
@@ -129,6 +130,10 @@ func (o *initOptions) run(cmd *cobra.Command, _ []string) error {
 	defer func() {
 		err = errors.Join(err, pkgLayout.Cleanup())
 	}()
+	var registryProxy *bool
+	if cmd.Flag("registry-proxy").Changed {
+		registryProxy = &pkgConfig.InitOpts.RegistryProxy
+	}
 
 	opts := packager.DeployOptions{
 		GitServer:              pkgConfig.InitOpts.GitServer,
@@ -140,6 +145,7 @@ func (o *initOptions) run(cmd *cobra.Command, _ []string) error {
 		OCIConcurrency:         config.CommonOptions.OCIConcurrency,
 		SetVariables:           pkgConfig.PkgOpts.SetVariables,
 		StorageClass:           pkgConfig.InitOpts.StorageClass,
+		RegistryProxy:          registryProxy,
 		RemoteOptions:          defaultRemoteOptions(),
 	}
 	_, err = deploy(ctx, pkgLayout, opts)

src/cmd/internal.go

@@ -18,6 +18,7 @@ import (
 	"github.com/zarf-dev/zarf/src/config/lang"
 	"github.com/zarf-dev/zarf/src/internal/agent"
 	"github.com/zarf-dev/zarf/src/internal/gitea"
+	injectorcontroller "github.com/zarf-dev/zarf/src/internal/injector-controller"
 	"github.com/zarf-dev/zarf/src/pkg/cluster"
 	"github.com/zarf-dev/zarf/src/pkg/logger"
 	"github.com/zarf-dev/zarf/src/pkg/state"
@@ -32,6 +33,7 @@ func newInternalCommand(rootCmd *cobra.Command) *cobra.Command {
 
 	cmd.AddCommand(newInternalAgentCommand())
 	cmd.AddCommand(newInternalHTTPProxyCommand())
+	cmd.AddCommand(newInternalRunControllerCommand())
 	cmd.AddCommand(newInternalGenCliDocsCommand(rootCmd))
 	cmd.AddCommand(newInternalCreateReadOnlyGiteaUserCommand())
 	cmd.AddCommand(newInternalCreateArtifactRegistryTokenCommand())
@@ -410,3 +412,30 @@ func (o *internalCrc32Options) run(_ *cobra.Command, args []string) {
 	hash := helpers.GetCRCHash(text)
 	fmt.Printf("%d\n", hash)
 }
+
+type internalRunControllerOptions struct{}
+
+func newInternalRunControllerCommand() *cobra.Command {
+	o := &internalRunControllerOptions{}
+
+	cmd := &cobra.Command{
+		Use:   "run-controller",
+		Short: lang.CmdInternalRunControllerShort,
+		Long:  lang.CmdInternalRunControllerLong,
+		RunE:  o.run,
+	}
+
+	return cmd
+}
+
+func (o *internalRunControllerOptions) run(cmd *cobra.Command, _ []string) error {
+	ctx := cmd.Context()
+
+	c, err := cluster.New(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create cluster client: %w", err)
+	}
+
+	controller := injectorcontroller.New(c)
+	return controller.Start(ctx)
+}

src/config/config.go

@@ -54,7 +54,7 @@ var (
 	CLIArch string
 
 	// ZarfSeedPort is the NodePort Zarf uses for the 'seed registry'
-	ZarfSeedPort string
+	ZarfSeedPort int
 
 	// Timestamp of when the CLI was started
 	operationStartTime  = time.Now().Unix()

src/config/lang/english.go

@@ -204,6 +204,10 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA
 
 	CmdInternalCrc32Short = "Generates a decimal CRC32 for the given text"
 
+	CmdInternalRunControllerShort = "Runs the injector controller to monitor registry proxy pods"
+	CmdInternalRunControllerLong  = "Runs the injector controller that watches for pods in the zarf-registry-proxy daemonset " +
+		"and logs when they encounter ErrImagePull status. This is used internally for monitoring registry proxy health."
+
 	// zarf package
 	CmdPackageShort                       = "Zarf package commands for creating, deploying, and inspecting packages"
 	CmdPackageFlagConcurrency             = "Number of concurrent layer operations when pulling or pushing images or packages to/from OCI registries."