fix: avoid injector pod name collisions (#2620)

## Description
This PR updates the injector pod naming by appending a UUID. This helps
avoid collisions with pods in a terminating state from previous zarf
init runs.

---------

Co-authored-by: Austin Abro <[email protected]>

go.mod

@@ -279,7 +279,7 @@ require (
  github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect
  github.com/google/s2a-go v0.1.7 // indirect
  github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
- github.com/google/uuid v1.6.0 // indirect
+ github.com/google/uuid v1.6.0
  github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
  github.com/googleapis/gax-go/v2 v2.12.4 // indirect
  github.com/gookit/color v1.5.4 // indirect

src/pkg/cluster/injector.go

@@ -7,26 +7,29 @@ package cluster
 import (
  "context"
  "fmt"
- "net/http"
  "os"
  "path/filepath"
  "regexp"
  "time"
 
  "github.com/defenseunicorns/pkg/helpers/v2"
+ pkgkubernetes "github.com/defenseunicorns/pkg/kubernetes"
  "github.com/defenseunicorns/zarf/src/config"
  "github.com/defenseunicorns/zarf/src/pkg/k8s"
  "github.com/defenseunicorns/zarf/src/pkg/layout"
  "github.com/defenseunicorns/zarf/src/pkg/message"
  "github.com/defenseunicorns/zarf/src/pkg/transform"
  "github.com/defenseunicorns/zarf/src/pkg/utils"
  "github.com/google/go-containerregistry/pkg/crane"
+ "github.com/google/uuid"
  "github.com/mholt/archiver/v3"
  corev1 "k8s.io/api/core/v1"
  kerrors "k8s.io/apimachinery/pkg/api/errors"
  "k8s.io/apimachinery/pkg/api/resource"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime/schema"
  "k8s.io/apimachinery/pkg/util/intstr"
+ "sigs.k8s.io/cli-utils/pkg/object"
 )
 
 // The chunk size for the tarball chunks.
@@ -43,7 +46,7 @@ var (
 type imageNodeMap map[string][]string
 
 // StartInjectionMadness initializes a Zarf injection into the cluster.
-func (c *Cluster) StartInjectionMadness(ctx context.Context, tmpDir string, imagesDir string, injectorSeedSrcs []string) {
+func (c *Cluster) StartInjectionMadness(ctx context.Context, tmpDir string, imagesDir string, injectorSeedSrcs []string) error {
  spinner := message.NewProgressSpinner("Attempting to bootstrap the seed image into the cluster")
  defer spinner.Stop()
 
@@ -56,118 +59,138 @@ func (c *Cluster) StartInjectionMadness(ctx context.Context, tmpDir string, imag
  }
 
  if err := helpers.CreateDirectory(tmp.SeedImagesDir, helpers.ReadWriteExecuteUser); err != nil {
- spinner.Fatalf(err, "Unable to create the seed images directory")
+ return fmt.Errorf("unable to create the seed images directory: %w", err)
  }
 
  var err error
  var images imageNodeMap
  var payloadConfigmaps []string
  var sha256sum string
- var seedImages []transform.Image
 
- // Get all the images from the cluster
- spinner.Updatef("Getting the list of existing cluster images")
  findImagesCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
  defer cancel()
- if images, err = c.getImagesAndNodesForInjection(findImagesCtx); err != nil {
- spinner.Fatalf(err, "Unable to generate a list of candidate images to perform the registry injection")
+ images, err = c.getImagesAndNodesForInjection(findImagesCtx)
+ if err != nil {
+ return err
  }
 
- spinner.Updatef("Creating the injector configmap")
  if err = c.createInjectorConfigMap(ctx, tmp.InjectionBinary); err != nil {
- spinner.Fatalf(err, "Unable to create the injector configmap")
+ return fmt.Errorf("unable to create the injector configmap: %w", err)
  }
 
- spinner.Updatef("Creating the injector service")
- if service, err := c.createService(ctx); err != nil {
- spinner.Fatalf(err, "Unable to create the injector service")
- } else {
- config.ZarfSeedPort = fmt.Sprintf("%d", service.Spec.Ports[0].NodePort)
+ service, err := c.createService(ctx)
+ if err != nil {
+ return fmt.Errorf("unable to create the injector service: %w", err)
  }
+ config.ZarfSeedPort = fmt.Sprintf("%d", service.Spec.Ports[0].NodePort)
 
- spinner.Updatef("Loading the seed image from the package")
- if seedImages, err = c.loadSeedImages(imagesDir, tmp.SeedImagesDir, injectorSeedSrcs, spinner); err != nil {
- spinner.Fatalf(err, "Unable to load the injector seed image from the package")
+ _, err = c.loadSeedImages(imagesDir, tmp.SeedImagesDir, injectorSeedSrcs)
+ if err != nil {
+ return fmt.Errorf("unable to load the injector seed image from the package: %w", err)
  }
 
- spinner.Updatef("Loading the seed registry configmaps")
  if payloadConfigmaps, sha256sum, err = c.createPayloadConfigMaps(ctx, tmp.SeedImagesDir, tmp.InjectorPayloadTarGz, spinner); err != nil {
- spinner.Fatalf(err, "Unable to generate the injector payload configmaps")
+ return fmt.Errorf("unable to generate the injector payload configmaps: %w", err)
  }
 
  // https://regex101.com/r/eLS3at/1
  zarfImageRegex := regexp.MustCompile(`(?m)^127\.0\.0\.1:`)
 
+ var injectorImage string
+ var injectorNode string
  // Try to create an injector pod using an existing image in the cluster
  for image, node := range images {
  // Don't try to run against the seed image if this is a secondary zarf init run
  if zarfImageRegex.MatchString(image) {
  continue
  }
-
  spinner.Updatef("Attempting to bootstrap with the %s/%s", node, image)
+ injectorImage = image
+ injectorNode = node[0]
+ }
+ // Make sure the pod is not there first
+ // TODO: Explain why no grace period is given.
+ deleteGracePeriod := int64(0)
+ deletePolicy := metav1.DeletePropagationForeground
+ deleteOpts := metav1.DeleteOptions{
+ GracePeriodSeconds: &deleteGracePeriod,
+ PropagationPolicy: &deletePolicy,
+ }
+ selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ "app": "zarf-injector",
+ },
+ })
+ if err != nil {
+ return err
+ }
+ listOpts := metav1.ListOptions{
+ LabelSelector: selector.String(),
+ }
+ err = c.Clientset.CoreV1().Pods(ZarfNamespaceName).DeleteCollection(ctx, deleteOpts, listOpts)
+ if err != nil {
+ return err
+ }
 
- // Make sure the pod is not there first
- // TODO: Explain why no grace period is given.
- deleteGracePeriod := int64(0)
- deletePolicy := metav1.DeletePropagationForeground
- deleteOpts := metav1.DeleteOptions{
- GracePeriodSeconds: &deleteGracePeriod,
- PropagationPolicy: &deletePolicy,
- }
- err := c.Clientset.CoreV1().Pods(ZarfNamespaceName).Delete(ctx, "injector", deleteOpts)
- if err != nil {
- message.Debug("could not delete pod injector:", err)
- }
-
- // Update the podspec image path and use the first node found
-
- pod, err := c.buildInjectionPod(node[0], image, payloadConfigmaps, sha256sum)
- if err != nil {
- // Just debug log the output because failures just result in trying the next image
- message.Debug("error making injection pod:", err)
- continue
- }
-
- // Create the pod in the cluster
- pod, err = c.Clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
- if err != nil {
- // Just debug log the output because failures just result in trying the next image
- message.Debug("error creating pod in cluster:", pod, err)
- continue
- }
+ pod, err := c.buildInjectionPod(injectorNode, injectorImage, payloadConfigmaps, sha256sum)
+ if err != nil {
+ return fmt.Errorf("error making injection pod: %w", err)
+ }
 
- // if no error, try and wait for a seed image to be present, return if successful
- if c.injectorIsReady(ctx, seedImages, spinner) {
- spinner.Success()
- return
- }
+ pod, err = c.Clientset.CoreV1().Pods(pod.Namespace).Create(ctx, pod, metav1.CreateOptions{})
+ if err != nil {
+ return fmt.Errorf("error creating pod in cluster: %w", err)
+ }
 
- // Otherwise just continue to try next image
+ objs := []object.ObjMetadata{
+ {
+ GroupKind: schema.GroupKind{
+ Kind: "Pod",
+ },
+ Namespace: ZarfNamespaceName,
+ Name: pod.Name,
+ },
+ }
+ waitCtx, waitCancel := context.WithTimeout(ctx, 60*time.Second)
+ defer waitCancel()
+ err = pkgkubernetes.WaitForReady(waitCtx, c.Watcher, objs)
+ if err != nil {
+ return err
  }
+ spinner.Success()
+ return nil
 
- // All images were exhausted and still no happiness
- spinner.Fatalf(nil, "Unable to perform the injection")
 }
 
 // StopInjectionMadness handles cleanup once the seed registry is up.
 func (c *Cluster) StopInjectionMadness(ctx context.Context) error {
  // Try to kill the injector pod now
- err := c.Clientset.CoreV1().Pods(ZarfNamespaceName).Delete(ctx, "injector", metav1.DeleteOptions{})
+ selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+ MatchLabels: map[string]string{
+ "app": "zarf-injector",
+ },
+ })
+ if err != nil {
+ return err
+ }
+ listOpts := metav1.ListOptions{
+ LabelSelector: selector.String(),
+ }
+ err = c.Clientset.CoreV1().Pods(ZarfNamespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts)
  if err != nil {
  return err
  }
 
  // Remove the configmaps
- selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
+ selector, err = metav1.LabelSelectorAsSelector(&metav1.LabelSelector{
  MatchLabels: map[string]string{
  "zarf-injector": "payload",
  },
  })
  if err != nil {
  return err
  }
- listOpts := metav1.ListOptions{
+ listOpts = metav1.ListOptions{
  LabelSelector: selector.String(),
  }
  err = c.Clientset.CoreV1().ConfigMaps(ZarfNamespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, listOpts)
@@ -183,13 +206,12 @@ func (c *Cluster) StopInjectionMadness(ctx context.Context) error {
  return nil
 }
 
-func (c *Cluster) loadSeedImages(imagesDir, seedImagesDir string, injectorSeedSrcs []string, spinner *message.Spinner) ([]transform.Image, error) {
+func (c *Cluster) loadSeedImages(imagesDir, seedImagesDir string, injectorSeedSrcs []string) ([]transform.Image, error) {
  seedImages := []transform.Image{}
  localReferenceToDigest := make(map[string]string)
 
  // Load the injector-specific images and save them as seed-images
  for _, src := range injectorSeedSrcs {
- spinner.Updatef("Loading the seed image '%s' from the package", src)
  ref, err := transform.ParseImageRef(src)
  if err != nil {
  return nil, fmt.Errorf("failed to create ref for image %s: %w", src, err)
@@ -230,7 +252,6 @@ func (c *Cluster) createPayloadConfigMaps(ctx context.Context, seedImagesDir, ta
  return configMaps, "", err
  }
 
- spinner.Updatef("Creating the seed registry archive to send to the cluster")
  // Create a tar archive of the injector payload
  if err := archiver.Archive(tarFileList, tarPath); err != nil {
  return configMaps, "", err
@@ -241,8 +262,6 @@ func (c *Cluster) createPayloadConfigMaps(ctx context.Context, seedImagesDir, ta
  return configMaps, "", err
  }
 
- spinner.Updatef("Splitting the archive into binary configmaps")
-
  chunkCount := len(chunks)
 
  // Loop over all chunks and generate configmaps
@@ -282,43 +301,6 @@ func (c *Cluster) createPayloadConfigMaps(ctx context.Context, seedImagesDir, ta
  return configMaps, sha256sum, nil
 }
 
-// Test for pod readiness and seed image presence.
-func (c *Cluster) injectorIsReady(ctx context.Context, seedImages []transform.Image, spinner *message.Spinner) bool {
- tunnel, err := c.NewTunnel(ZarfNamespaceName, SvcResource, ZarfInjectorName, "", 0, ZarfInjectorPort)
- if err != nil {
- return false
- }
-
- _, err = tunnel.Connect(ctx)
- if err != nil {
- return false
- }
- defer tunnel.Close()
-
- spinner.Updatef("Testing the injector for seed image availability")
-
- for _, seedImage := range seedImages {
- seedRegistry := fmt.Sprintf("%s/v2/%s/manifests/%s", tunnel.HTTPEndpoint(), seedImage.Path, seedImage.Tag)
-
- var resp *http.Response
- var err error
- err = tunnel.Wrap(func() error {
- message.Debug("getting seed registry %v", seedRegistry)
- resp, err = http.Get(seedRegistry)
- return err
- })
-
- if err != nil || resp.StatusCode != 200 {
- // Just debug log the output because failures just result in trying the next image
- message.Debug(resp, err)
- return false
- }
- }
-
- spinner.Updatef("Seed image found, injector is ready")
- return true
-}
-
 func (c *Cluster) createInjectorConfigMap(ctx context.Context, binaryPath string) error {
  name := "rust-binary"
  // TODO: Replace with a create or update.
@@ -384,13 +366,18 @@ func (c *Cluster) createService(ctx context.Context) (*corev1.Service, error) {
 // buildInjectionPod return a pod for injection with the appropriate containers to perform the injection.
 func (c *Cluster) buildInjectionPod(node, image string, payloadConfigmaps []string, payloadShasum string) (*corev1.Pod, error) {
  executeMode := int32(0777)
+
+ // Generate a UUID to append to the pod name.
+ // This prevents collisions where `zarf init` is ran back to back and a previous injector pod still exists.
+ uuid := uuid.New().String()[:16]
+
  pod := &corev1.Pod{
  TypeMeta: metav1.TypeMeta{
  APIVersion: corev1.SchemeGroupVersion.String(),
  Kind: "Pod",
  },
  ObjectMeta: metav1.ObjectMeta{
- Name: "injector",
+ Name: fmt.Sprintf("injector-%s", uuid),
  Namespace: ZarfNamespaceName,
  Labels: map[string]string{
  "app": "zarf-injector",

src/pkg/cluster/injector_test.go

@@ -77,6 +77,9 @@ func TestBuildInjectionPod(t *testing.T) {
  c := &Cluster{}
  pod, err := c.buildInjectionPod("injection-node", "docker.io/library/ubuntu:latest", []string{"foo", "bar"}, "shasum")
  require.NoError(t, err)
+ require.Contains(t, pod.Name, "injector-")
+ // Replace the random UUID in the pod name with a fixed placeholder for consistent comparison.
+ pod.ObjectMeta.Name = "injector-UUID"
  b, err := json.Marshal(pod)
  require.NoError(t, err)
  expected, err := os.ReadFile("./testdata/expected-injection-pod.json")

src/pkg/cluster/testdata/expected-injection-pod.json

@@ -1 +1 @@
-{"kind":"Pod","apiVersion":"v1","metadata":{"name":"injector","namespace":"zarf","creationTimestamp":null,"labels":{"app":"zarf-injector","zarf.dev/agent":"ignore"}},"spec":{"volumes":[{"name":"init","configMap":{"name":"rust-binary","defaultMode":511}},{"name":"seed","emptyDir":{}},{"name":"foo","configMap":{"name":"foo"}},{"name":"bar","configMap":{"name":"bar"}}],"containers":[{"name":"injector","image":"docker.io/library/ubuntu:latest","command":["/zarf-init/zarf-injector","shasum"],"workingDir":"/zarf-init","resources":{"limits":{"cpu":"1","memory":"256Mi"},"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"init","mountPath":"/zarf-init/zarf-injector","subPath":"zarf-injector"},{"name":"seed","mountPath":"/zarf-seed"},{"name":"foo","mountPath":"/zarf-init/foo","subPath":"foo"},{"name":"bar","mountPath":"/zarf-init/bar","subPath":"bar"}],"readinessProbe":{"httpGet":{"path":"/v2/","port":5000},"periodSeconds":2,"successThreshold":1,"failureThreshold":10},"imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Never","nodeName":"injection-node"},"status":{}}
+{"kind":"Pod","apiVersion":"v1","metadata":{"name":"injector-UUID","namespace":"zarf","creationTimestamp":null,"labels":{"app":"zarf-injector","zarf.dev/agent":"ignore"}},"spec":{"volumes":[{"name":"init","configMap":{"name":"rust-binary","defaultMode":511}},{"name":"seed","emptyDir":{}},{"name":"foo","configMap":{"name":"foo"}},{"name":"bar","configMap":{"name":"bar"}}],"containers":[{"name":"injector","image":"docker.io/library/ubuntu:latest","command":["/zarf-init/zarf-injector","shasum"],"workingDir":"/zarf-init","resources":{"limits":{"cpu":"1","memory":"256Mi"},"requests":{"cpu":"500m","memory":"64Mi"}},"volumeMounts":[{"name":"init","mountPath":"/zarf-init/zarf-injector","subPath":"zarf-injector"},{"name":"seed","mountPath":"/zarf-seed"},{"name":"foo","mountPath":"/zarf-init/foo","subPath":"foo"},{"name":"bar","mountPath":"/zarf-init/bar","subPath":"bar"}],"readinessProbe":{"httpGet":{"path":"/v2/","port":5000},"periodSeconds":2,"successThreshold":1,"failureThreshold":10},"imagePullPolicy":"IfNotPresent"}],"restartPolicy":"Never","nodeName":"injection-node"},"status":{}}

src/pkg/packager/deploy.go

@@ -237,13 +237,13 @@ func (p *Packager) deployInitComponent(ctx context.Context, component types.Zarf
  if component.RequiresCluster() && p.state == nil {
  err = p.cluster.InitZarfState(ctx, p.cfg.InitOpts)
  if err != nil {
- return charts, fmt.Errorf("unable to initialize Zarf state: %w", err)
+ return nil, fmt.Errorf("unable to initialize Zarf state: %w", err)
  }
  }
 
  if hasExternalRegistry && (isSeedRegistry || isInjector || isRegistry) {
  message.Notef("Not deploying the component (%s) since external registry information was provided during `zarf init`", component.Name)
- return charts, nil
+ return nil, nil
  }
 
  if isRegistry {
@@ -253,18 +253,21 @@ func (p *Packager) deployInitComponent(ctx context.Context, component types.Zarf
 
  // Before deploying the seed registry, start the injector
  if isSeedRegistry {
- p.cluster.StartInjectionMadness(ctx, p.layout.Base, p.layout.Images.Base, component.Images)
+ err := p.cluster.StartInjectionMadness(ctx, p.layout.Base, p.layout.Images.Base, component.Images)
+ if err != nil {
+ return nil, err
+ }
  }
 
  charts, err = p.deployComponent(ctx, component, isAgent /* skip img checksum if isAgent */, isSeedRegistry /* skip image push if isSeedRegistry */)
  if err != nil {
- return charts, err
+ return nil, err
  }
 
  // Do cleanup for when we inject the seed registry during initialization
  if isSeedRegistry {
  if err := p.cluster.StopInjectionMadness(ctx); err != nil {
- return charts, fmt.Errorf("unable to seed the Zarf Registry: %w", err)
+ return nil, fmt.Errorf("unable to seed the Zarf Registry: %w", err)
  }
  }