Add the ability to specify registry overrides on package create (#1683)

## Description

Allows registries to be overridden with registry mirrors on Zarf package
create.

## Related Issue

Fixes #1577 

- [x] Fix the faq for large local images - this can fix that use case.
#1679 (comment)

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [X] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [X] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed

---------

Co-authored-by: Jonathan Perry <[email protected]>

docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md

@@ -16,17 +16,18 @@ zarf package create [DIRECTORY] [flags]
 ## Options
 
 ```
- --confirm Confirm package creation without prompting
- --differential string Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package
- -h, --help help for create
- -k, --key string Path to private key file for signing packages
- --key-pass string Password to the private key file used for signing packages
- -m, --max-package-size int Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting.
- -o, --output-directory string Specify the output directory for the created Zarf package
- -s, --sbom View SBOM contents after creating the package
- --sbom-out string Specify an output directory for the SBOMs from the created Zarf package
- --set stringToString Specify package variables to set on the command line (KEY=value) (default [])
- --skip-sbom Skip generating SBOM for this package
+ --confirm Confirm package creation without prompting
+ --differential string Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package
+ -h, --help help for create
+ -k, --key string Path to private key file for signing packages
+ --key-pass string Password to the private key file used for signing packages
+ -m, --max-package-size int Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting.
+ -o, --output-directory string Specify the output directory for the created Zarf package
+ --registry-override stringToString Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default [])
+ -s, --sbom View SBOM contents after creating the package
+ --sbom-out string Specify an output directory for the SBOMs from the created Zarf package
+ --set stringToString Specify package variables to set on the command line (KEY=value) (default [])
+ --skip-sbom Skip generating SBOM for this package
 ```
 
 ## Options inherited from parent commands

docs/3-create-a-zarf-package/4-zarf-schema.md

@@ -424,6 +424,46 @@ Must be one of:
 </blockquote>
 </details>
 
+<details open>
+<summary>
+<strong> <a name="build_registryOverrides"></a>registryOverrides *</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+ ## build > registryOverrides
+![Required](https://img.shields.io/badge/Required-red)
+
+**Description:** Any registry domains that were overridden on package create when pulling images
+
+| | |
+| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
+| **Type** | `object` |
+| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") |
+
+<details>
+<summary>
+<strong> <a name="build_registryOverrides_pattern1"></a>Pattern Property .*</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+:::note
+All properties whose name matches the regular expression
+```.*``` ([Test](https://regex101.com/?regex=.%2A))
+must respect the following conditions
+:::
+
+| | |
+| -------- | -------- |
+| **Type** | `string` |
+
+</blockquote>
+</details>
+
+</blockquote>
+</details>
+
 </blockquote>
 </details>
 

docs/6-zarf-tutorials/0-creating-a-zarf-package.md

@@ -254,7 +254,7 @@ This will create a zarf package in the current directory with a package name tha
 
 :::tip
 
-You can learn more about what is going on behind the scenes of this process on the [package create lifecycle page](../3-create-a-zarf-package/5-package-create-lifecycle.md).
+You can learn more about what is going on behind the scenes of this process on the [package create lifecycle page](../3-create-a-zarf-package/5-package-create-lifecycle.md), and can view other useful command flags like `--differential` and `--registry-override` on the [package create command flags page](../2-the-zarf-cli/100-cli-commands/zarf_package_create.md).
 
 :::
 

docs/6-zarf-tutorials/8-custom-init-packages.md

@@ -74,6 +74,19 @@ As of v0.26.0 you can swap the `registry` and `agent` images by specifying diffe
 
 For other components, or older versions of Zarf, you can modify the manifests of the components you want to change in their individual packages under the `packages` folder of the Zarf repo.
 
+:::tip
+
+If your enterprise uses pull-through mirrors to host vetted images you can run the following command to create a Zarf 'init' package from those mirrors (where `<registry>.enterprise.corp` are your enterprise mirror(s)):
+
+```
+$ zarf package create . --set AGENT_IMAGE_TAG=vX.X.X \
+ --registry-override docker.io=dockerio.enterprise.corp \
+ --registry-override ghcr.io=ghcr.enterprise.corp \
+ --registry-override quay.io=quay.enterprise.corp
+```
+
+:::
+
 ### Removing Components
 
 You may not need or want all of the components in your 'init' package and may choose to slim down your package by removing them. Because the [Zarf Package is composed](../3-create-a-zarf-package/2-zarf-components.md#composing-package-components) all you need to do is remove the component that imports the component you wish to exclude.

docs/8-faq.md

@@ -46,14 +46,16 @@ Additionally, when adopting resources, you should ensure that the namespaces you
 
 ## How can I improve the speed of loading large images from Docker on `zarf package create`?
 
-Due to some limitations with how Docker provides access to local image layers, `zarf package create` has to rely on `docker save` under the hood which is [very slow overall](https://github.com/defenseunicorns/zarf/issues/1214) and also takes a long time to report progress. We experimented with many ways to improve this, but for now recommend leveraging a local docker registry to speed up the process. This can be done by running a local registry and pushing the images to it before running `zarf package create`. This will allow `zarf package create` to pull the images from the local registry instead of Docker. This can also be combined with [component actions](3-create-a-zarf-package/6-component-actions.md) to make the process automatic. Given an example image of `my-giant-image:###ZARF_PKG_TMPL_IMG###` you could do something like this:
+Due to some limitations with how Docker provides access to local image layers, `zarf package create` has to rely on `docker save` under the hood which is [very slow overall](https://github.com/defenseunicorns/zarf/issues/1214) and also takes a long time to report progress. We experimented with many ways to improve this, but for now recommend leveraging a local docker registry to speed up the process.
+
+This can be done by running a local registry and pushing the images to it before running `zarf package create`. This will allow `zarf package create` to pull the images from the local registry instead of Docker. This can also be combined with [component actions](3-create-a-zarf-package/6-component-actions.md) and [`--registry-override`](./2-the-zarf-cli/100-cli-commands/zarf_package_create.md) to make the process automatic. Given an example image of `registry.enterprise.corp/my-giant-image:v2` you could do something like this:
 
 ```sh
 # Create a local registry
 docker run -d -p 5000:5000 --restart=always --name registry registry:2
 
 # Run the package create with a tag variable
-zarf package create --set IMG=my-giant-image:v2
+zarf package create --registry-override registry.enterprise.corp=localhost:5000 --set IMG=my-giant-image:v2
 ```
 
 ```yaml
@@ -68,11 +70,11 @@ components:
  onCreate:
  # runs before the component is created
  before:
- - cmd: 'docker tag ###ZARF_PKG_TMPL_IMG### localhost:5000/###ZARF_PKG_TMPL_IMG###'
+ - cmd: 'docker tag registry.enterprise.corp/###ZARF_PKG_TMPL_IMG### localhost:5000/###ZARF_PKG_TMPL_IMG###'
  - cmd: 'docker push localhost:5000/###ZARF_PKG_TMPL_IMG###'
 
  images:
- - 'localhost:5000/###ZARF_PKG_TMPL_IMG###'
+ - 'registry.enterprise.corp/###ZARF_PKG_TMPL_IMG###'
 ```
 
 ## Can I pull in more than http(s) git repos on `zarf package create`?

examples/helm-git-chart/zarf.yaml

@@ -1,6 +1,6 @@
 kind: ZarfPackageConfig
 metadata:
- name: test-helm-git-chart
+ name: helm-git-chart
  description: "Deploys a helm chart from git"
 components:
  - name: demo-helm-git-chart

packages/distros/k3s/zarf.yaml

@@ -62,4 +62,4 @@ components:
  before:
  - cmd: if [ "$(arch)" != "arm64" ]; then echo "this package architecture is arm64, but the target system has a different architecture. These architectures must be the same" && exit 1; fi
  description: Check that the host architecture matches the package architecture
- maxRetries: 0
+ maxRetries: 0

src/cmd/package.go

@@ -315,6 +315,7 @@ func bindCreateFlags() {
  createFlags.IntVarP(&pkgConfig.CreateOpts.MaxPackageSizeMB, "max-package-size", "m", v.GetInt(V_PKG_CREATE_MAX_PACKAGE_SIZE), lang.CmdPackageCreateFlagMaxPackageSize)
  createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(V_PKG_CREATE_SIGNING_KEY), lang.CmdPackageCreateFlagSigningKey)
  createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(V_PKG_CREATE_SIGNING_KEY_PASSWORD), lang.CmdPackageCreateFlagSigningKeyPassword)
+ createFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(V_PKG_CREATE_REGISTRY_OVERRIDE), lang.CmdPackageCreateFlagRegistryOverride)
 }
 
 func bindDeployFlags() {

src/cmd/viper.go

@@ -59,6 +59,7 @@ const (
  V_PKG_CREATE_SIGNING_KEY = "package.create.signing_key"
  V_PKG_CREATE_SIGNING_KEY_PASSWORD = "package.create.signing_key_password"
  V_PKG_CREATE_DIFFERENTIAL = "package.create.differential"
+ V_PKG_CREATE_REGISTRY_OVERRIDE = "package.create.registry_override"
 
  // Package deploy config keys
  V_PKG_DEPLOY_SET = "package.deploy.set"

src/config/lang/english.go

@@ -217,6 +217,7 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur
  CmdPackageCreateFlagSigningKey = "Path to private key file for signing packages"
  CmdPackageCreateFlagSigningKeyPassword = "Password to the private key file used for signing packages"
  CmdPackageCreateFlagDifferential = "Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package"
+ CmdPackageCreateFlagRegistryOverride = "Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet)"
 
  CmdPackageDeployFlagConfirm = "Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes."
  CmdPackageDeployFlagAdoptExistingResources = "Adopts any pre-existing K8s resources into the Helm charts managed by Zarf. ONLY use when you have existing deployments you want Zarf to takeover."

src/internal/packager/git/clone.go

@@ -44,7 +44,7 @@ func (g *Git) clone(gitURL string, ref plumbing.ReferenceName) error {
  // Clone the given repo.
  repo, err := git.PlainClone(g.GitPath, false, cloneOptions)
  if err != nil {
- message.Debugf("Failed to clone repo %s: %s", gitURL, err.Error())
+ message.Warnf("Falling back to host 'git', failed to clone the repo with Zarf - %s: %s", gitURL, err.Error())
  return g.gitCloneFallback(gitURL, ref)
  }
 
@@ -66,8 +66,6 @@ func (g *Git) clone(gitURL string, ref plumbing.ReferenceName) error {
 
 // gitCloneFallback is a fallback if go-git fails to clone a repo.
 func (g *Git) gitCloneFallback(gitURL string, ref plumbing.ReferenceName) error {
- g.Spinner.Updatef("Falling back to host git for %s", gitURL)
-
  // If we can't clone with go-git, fallback to the host clone
  // Only support "all tags" due to the azure clone url format including a username
  cmdArgs := []string{"clone", "--origin", onlineRemoteName, gitURL, g.GitPath}

src/internal/packager/images/common.go

@@ -28,6 +28,8 @@ type ImgConfig struct {
  Insecure bool
 
  Architectures []string
+
+ RegistryOverrides map[string]string
 }
 
 // GetLegacyImgTarballPath returns the ImagesPath as if it were a path to a tarball instead of a directory.

src/internal/packager/images/pull.go

@@ -17,6 +17,7 @@ import (
 
  "github.com/defenseunicorns/zarf/src/config"
  "github.com/defenseunicorns/zarf/src/pkg/message"
+ "github.com/defenseunicorns/zarf/src/pkg/transform"
  "github.com/defenseunicorns/zarf/src/pkg/utils"
  "github.com/google/go-containerregistry/pkg/crane"
  "github.com/google/go-containerregistry/pkg/logs"
@@ -55,9 +56,22 @@ func (i *ImgConfig) PullAll() error {
  for idx, src := range i.ImgList {
  spinner.Updatef("Fetching image metadata (%d of %d): %s", idx+1, imgCount, src)
 
- img, err := i.PullImage(src, spinner)
+ srcParsed, err := transform.ParseImageRef(src)
  if err != nil {
- return fmt.Errorf("failed to pull image %s: %w", src, err)
+ return fmt.Errorf("failed to parse image ref %s: %w", src, err)
+ }
+
+ actualSrc := src
+ if overrideHost, present := i.RegistryOverrides[srcParsed.Host]; present {
+ actualSrc, err = transform.ImageTransformHostWithoutChecksum(overrideHost, src)
+ if err != nil {
+ return fmt.Errorf("failed to swap override host %s for %s: %w", overrideHost, src, err)
+ }
+ }
+
+ img, err := i.PullImage(actualSrc, spinner)
+ if err != nil {
+ return fmt.Errorf("failed to pull image %s: %w", actualSrc, err)
  }
  imageMap[src] = img
  }

src/pkg/packager/create.go

@@ -150,10 +150,11 @@ func (p *Packager) Create(baseDir string) error {
 
  doPull := func() error {
  imgConfig := images.ImgConfig{
- ImagesPath: p.tmp.Images,
- ImgList: imgList,
- Insecure: config.CommonOptions.Insecure,
- Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture},
+ ImagesPath: p.tmp.Images,
+ ImgList: imgList,
+ Insecure: config.CommonOptions.Insecure,
+ Architectures: []string{p.cfg.Pkg.Metadata.Architecture, p.cfg.Pkg.Build.Architecture},
+ RegistryOverrides: p.cfg.CreateOpts.RegistryOverrides,
  }
 
  return imgConfig.PullAll()

src/pkg/packager/yaml.go

@@ -77,5 +77,7 @@ func (p *Packager) writeYaml() error {
  deprecated.PluralizeSetVariable,
  }
 
+ p.cfg.Pkg.Build.RegistryOverrides = p.cfg.CreateOpts.RegistryOverrides
+
  return utils.WriteYaml(p.tmp.ZarfYaml, p.cfg.Pkg, 0400)
 }