1056 support (more) remote files in zarf.yaml manifests (#1633)

Support added for the following resources to be URLs:

- manifests + kustomizations
- data injectsions
- helm chart values files

files + helm charts already had support for URLs

additionally, URLs can be verified w/ SHA256 checksums, stored alongside
the URL:

```
https://k8s.io/examples/application/deployment.yaml@c57f73449b26eae02ca2a549c388807d49ef6d3f2dc040a9bbb1290128d97157

github.com/stefanprodan/podinfo//kustomize?ref=0647aea75b85755411b007a290b9321668370be5
```

This also supports image discovery during `zarf prepare find-images`

## Related Issue

Fixes #1056 

## Type of change

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [x] Test, docs, adr added or updated as needed
- [x] [Contributor Guide
Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow)
followed

---------

Signed-off-by: razzle <[email protected]>
Co-authored-by: Wayne Starr <[email protected]>

Makefile

@@ -122,6 +122,8 @@ build-examples: ## Build all of the example packages
 
  @test -s ./build/zarf-package-dos-games-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/dos-games -o build -a $(ARCH) --confirm
 
+ @test -s ./build/zarf-package-remote-manifests-$(ARCH)-0.0.1.tar.zst || $(ZARF_BIN) package create examples/remote-manifests -o build -a $(ARCH) --confirm
+
  @test -s ./build/zarf-package-component-actions-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/component-actions -o build -a $(ARCH) --confirm
 
  @test -s ./build/zarf-package-component-choice-$(ARCH).tar.zst || $(ZARF_BIN) package create examples/component-choice -o build -a $(ARCH) --confirm

docs/4-user-guide/3-zarf-schema.md

@@ -1840,7 +1840,7 @@ Must be one of:
  
 <blockquote>
 
-**Description:** List of values files to include in the package; these will be merged together
+**Description:** List of local values file paths or remote URLs to include in the package; these will be merged together
 
 | | |
 | -------- | ----------------- |
@@ -1985,7 +1985,7 @@ Must be one of:
 &nbsp;
 <blockquote>
 
-**Description:** List of individual K8s YAML files to deploy (in order)
+**Description:** List of local K8s YAML files or remote URLs to deploy (in order)
 
 | | |
 | -------- | ----------------- |
@@ -2028,7 +2028,7 @@ Must be one of:
 &nbsp;
 <blockquote>
 
-**Description:** List of kustomization paths to include in the package
+**Description:** List of local kustomization paths or remote URLs to include in the package
 
 | | |
 | -------- | ----------------- |
@@ -2158,7 +2158,7 @@ Must be one of:
 
 ![Required](https://img.shields.io/badge/Required-red)
 
-**Description:** A path to a local folder or file to inject into the given target pod + container
+**Description:** Either a path to a local folder/file or a remote URL of a file to inject into the given target pod + container
 
 | | |
 | -------- | -------- |

examples/remote-manifests/README.md

@@ -0,0 +1,9 @@
+# Remote Manifests
+
+This example shows how you to specify remote resources for a component's `manifests`.
+
+:::info
+
+To view the example source code, select the `Edit this page` link below the article and select the parent folder.
+
+:::

examples/remote-manifests/zarf.yaml

@@ -0,0 +1,52 @@
+kind: ZarfPackageConfig
+metadata:
+ name: remote-manifests
+ version: 0.0.1
+
+components:
+ - name: remote-manifests-and-kustomizations
+ required: true
+ manifests:
+ - name: simple-nginx-deployment
+ namespace: nginx
+ files:
+ # to verify integrity of a remote manifest, you can use the sha256sum of the file
+ # append the sha256sum to the end of the url, separated by an @
+ # example:
+ - https://k8s.io/examples/application/deployment.yaml@c57f73449b26eae02ca2a549c388807d49ef6d3f2dc040a9bbb1290128d97157
+ # this sha256 can be discovered using the following:
+ # curl -s https://k8s.io/examples/application/deployment.yaml | sha256sum
+ # or
+ # zarf prepare sha256sum https://k8s.io/examples/application/deployment.yaml
+ - name: podinfo
+ namespace: podinfo
+ kustomizations:
+ # note this syntax: https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md
+ # using a specific commit hash (essentially the same thing as sha256summ'ing the kustomization)
+ # ?ref= is not a requirement, but it is recommended to use a specific commit hash / git tag to ensure
+ # that the kustomization is not changed in a way that breaks your deployment
+ # example:
+ - github.com/stefanprodan/podinfo//kustomize?ref=0647aea75b85755411b007a290b9321668370be5
+ actions:
+ onDeploy:
+ # the following checks were computed by viewing the success state of the package deployment
+ # and creating `wait` actions that match
+ after:
+ - wait:
+ cluster:
+ kind: deployment
+ name: nginx-deployment
+ namespace: nginx
+ condition: available
+ - wait:
+ cluster:
+ kind: deployment
+ name: podinfo
+ namespace: podinfo
+ condition: available
+ # image discovery is supported in remote manifests and kustomizations
+ # using the following command:
+ # zarf prepare find-images
+ images:
+ - nginx:1.14.2
+ - ghcr.io/stefanprodan/podinfo:6.3.5

src/cmd/initialize.go

@@ -122,19 +122,16 @@ func downloadInitPackage(initPackageName, downloadCacheTarget string) error {
  Message: lang.CmdInitDownloadConfirm,
  }
  if err := survey.AskOne(prompt, &confirmDownload); err != nil {
- message.Fatalf(nil, lang.CmdInitDownloadCancel, err.Error())
+ return fmt.Errorf(lang.CmdInitDownloadCancel, err.Error())
  }
  }
 
  // If the user wants to download the init-package, download it
  if confirmDownload {
- utils.DownloadToFile(url, downloadCacheTarget, "")
- } else {
- // Otherwise, exit and tell the user to manually download the init-package
- return errors.New(lang.CmdInitDownloadErrManual)
+ return utils.DownloadToFile(url, downloadCacheTarget, "")
  }
-
- return nil
+ // Otherwise, exit and tell the user to manually download the init-package
+ return errors.New(lang.CmdInitDownloadErrManual)
 }
 
 func validateInitFlags() error {

src/config/lang/english.go

@@ -22,6 +22,8 @@ const (
  ErrTunnelFailed = "Failed to create a tunnel to the Kubernetes cluster."
  ErrUnmarshal = "failed to unmarshal file: %w"
  ErrWritingFile = "failed to write the file %s: %s"
+ ErrDownloading = "failed to download %s: %s"
+ ErrCreatingDir = "failed to create directory %s: %s"
 )
 
 // Zarf CLI commands.
@@ -127,7 +129,7 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur
  CmdInitDownloadAsk = "It seems the init package could not be found locally, but can be downloaded from %s"
  CmdInitDownloadNote = "Note: This will require an internet connection."
  CmdInitDownloadConfirm = "Do you want to download this init package?"
- CmdInitDownloadCancel = "Confirm selection canceled: %s"
+ CmdInitDownloadCancel = "confirm selection canceled: %s"
  CmdInitDownloadErrManual = "download the init package manually and place it in the current working directory"
 
  CmdInitFlagSet = "Specify deployment variables to set on the command line (KEY=value)"

src/extensions/bigbang/flux.go

@@ -50,7 +50,7 @@ func getFlux(baseDir string, cfg *extensions.BigBang) (manifest types.ZarfManife
  remotePath := fmt.Sprintf("%s//base/flux?ref=%s", cfg.Repo, cfg.Version)
 
  // Perform Kustomzation now to get the flux.yaml file.
- if err := kustomize.BuildKustomization(remotePath, localPath, true); err != nil {
+ if err := kustomize.Build(remotePath, localPath, true); err != nil {
  return manifest, images, fmt.Errorf("unable to build kustomization: %w", err)
  }
 

src/internal/packager/kustomize/build.go

@@ -13,8 +13,8 @@ import (
  "sigs.k8s.io/kustomize/kyaml/filesys"
 )
 
-// BuildKustomization reads a kustomization and builds it into a single yaml file.
-func BuildKustomization(path string, destination string, kustomizeAllowAnyDirectory bool) error {
+// Build reads a kustomization and builds it into a single yaml file.
+func Build(path string, destination string, kustomizeAllowAnyDirectory bool) error {
  // Kustomize has to write to the filesystem on-disk
  fSys := filesys.MakeFsOnDisk()
 

src/pkg/message/progress.go

@@ -88,8 +88,8 @@ func (p *ProgressBar) Stop() {
  }
 }
 
-// Fatalf marks the ProgressBar as failed in the CLI.
-func (p *ProgressBar) Fatalf(err error, format string, a ...any) {
+// Errorf marks the ProgressBar as failed in the CLI.
+func (p *ProgressBar) Errorf(err error, format string, a ...any) {
  p.Stop()
- Fatalf(err, format, a...)
+ Errorf(err, format, a...)
 }

src/pkg/packager/create.go

@@ -17,6 +17,7 @@ import (
  "time"
 
  "github.com/defenseunicorns/zarf/src/config"
+ "github.com/defenseunicorns/zarf/src/config/lang"
  "github.com/defenseunicorns/zarf/src/internal/packager/git"
  "github.com/defenseunicorns/zarf/src/internal/packager/helm"
  "github.com/defenseunicorns/zarf/src/internal/packager/images"
@@ -313,45 +314,55 @@ func (p *Packager) addComponent(component types.ZarfComponent) (*types.Component
  }
 
  for idx, path := range chart.ValuesFiles {
- chartValueName := helm.StandardName(componentPath.Values, chart) + "-" + strconv.Itoa(idx)
- if err := utils.CreatePathAndCopy(path, chartValueName); err != nil {
- return nil, fmt.Errorf("unable to copy chart values file %s: %w", path, err)
+ dst := helm.StandardName(componentPath.Values, chart) + "-" + strconv.Itoa(idx)
+ if utils.IsURL(path) {
+ if err := utils.DownloadToFile(path, dst, component.CosignKeyPath); err != nil {
+ return nil, fmt.Errorf(lang.ErrDownloading, path, err.Error())
+ }
+ } else {
+ if err := utils.CreatePathAndCopy(path, dst); err != nil {
+ return nil, fmt.Errorf("unable to copy chart values file %s: %w", path, err)
+ }
  }
  }
  }
  }
 
  if len(component.Files) > 0 {
- _ = utils.CreateDirectory(componentPath.Files, 0700)
+ if err = utils.CreateDirectory(componentPath.Files, 0700); err != nil {
+ return nil, fmt.Errorf("unable to create the component files directory: %s", err.Error())
+ }
 
  for index, file := range component.Files {
  message.Debugf("Loading %#v", file)
- destinationFile := filepath.Join(componentPath.Files, strconv.Itoa(index))
+ dst := filepath.Join(componentPath.Files, strconv.Itoa(index))
 
  if utils.IsURL(file.Source) {
- utils.DownloadToFile(file.Source, destinationFile, component.CosignKeyPath)
+ if err := utils.DownloadToFile(file.Source, dst, component.CosignKeyPath); err != nil {
+ return nil, fmt.Errorf(lang.ErrDownloading, file.Source, err.Error())
+ }
  } else {
- if err := utils.CreatePathAndCopy(file.Source, destinationFile); err != nil {
+ if err := utils.CreatePathAndCopy(file.Source, dst); err != nil {
  return nil, fmt.Errorf("unable to copy file %s: %w", file.Source, err)
  }
  }
 
  // Abort packaging on invalid shasum (if one is specified).
  if file.Shasum != "" {
- if actualShasum, _ := utils.GetCryptoHash(destinationFile, crypto.SHA256); actualShasum != file.Shasum {
+ if actualShasum, _ := utils.GetCryptoHash(dst, crypto.SHA256); actualShasum != file.Shasum {
  return nil, fmt.Errorf("shasum mismatch for file %s: expected %s, got %s", file.Source, file.Shasum, actualShasum)
  }
  }
 
- info, _ := os.Stat(destinationFile)
+ info, _ := os.Stat(dst)
 
  if file.Executable || info.IsDir() {
- _ = os.Chmod(destinationFile, 0700)
+ _ = os.Chmod(dst, 0700)
  } else {
- _ = os.Chmod(destinationFile, 0600)
+ _ = os.Chmod(dst, 0600)
  }
 
- componentSBOM.Files = append(componentSBOM.Files, destinationFile)
+ componentSBOM.Files = append(componentSBOM.Files, dst)
  }
  }
 
@@ -361,19 +372,28 @@ func (p *Packager) addComponent(component types.ZarfComponent) (*types.Component
 
  for _, data := range component.DataInjections {
  spinner.Updatef("Copying data injection %s for %s", data.Target.Path, data.Target.Selector)
- destination := filepath.Join(componentPath.DataInjections, filepath.Base(data.Target.Path))
- if err := utils.CreatePathAndCopy(data.Source, destination); err != nil {
- return nil, fmt.Errorf("unable to copy data injection %s: %w", data.Source, err)
+ dst := filepath.Join(componentPath.DataInjections, filepath.Base(data.Target.Path))
+ if utils.IsURL(data.Source) {
+ if err := utils.DownloadToFile(data.Source, dst, component.CosignKeyPath); err != nil {
+ return nil, fmt.Errorf(lang.ErrDownloading, data.Source, err.Error())
+ }
+ } else {
+ if err := utils.CreatePathAndCopy(data.Source, dst); err != nil {
+ return nil, fmt.Errorf("unable to copy data injection %s: %s", data.Source, err.Error())
+ }
  }
 
  // Unwrap the dataInjection dir into individual files.
  pattern := regexp.MustCompile(`(?mi).+$`)
- files, _ := utils.RecursiveFileList(destination, pattern, false, true)
+ files, _ := utils.RecursiveFileList(dst, pattern, false, true)
  componentSBOM.Files = append(componentSBOM.Files, files...)
  }
  }
 
  if len(component.Manifests) > 0 {
+ if err := utils.CreateDirectory(componentPath.Manifests, 0700); err != nil {
+ return nil, fmt.Errorf("unable to create manifest directory %s: %s", componentPath.Manifests, err.Error())
+ }
  // Get the proper count of total manifests to add.
  manifestCount := 0
 
@@ -385,31 +405,39 @@ func (p *Packager) addComponent(component types.ZarfComponent) (*types.Component
  spinner := message.NewProgressSpinner("Loading %d K8s manifests", manifestCount)
  defer spinner.Success()
 
- if err := utils.CreateDirectory(componentPath.Manifests, 0700); err != nil {
- return nil, fmt.Errorf("unable to create manifest directory %s: %w", componentPath.Manifests, err)
- }
-
  // Iterate over all manifests.
  for _, manifest := range component.Manifests {
  for idx, f := range manifest.Files {
+ var trimmedPath string
+ var destination string
  // Copy manifests without any processing.
  spinner.Updatef("Copying manifest %s", f)
- // If using a temp directory, trim the temp directory from the path.
- trimmedPath := strings.TrimPrefix(f, componentPath.Temp)
- destination := path.Join(componentPath.Manifests, trimmedPath)
- if err := utils.CreatePathAndCopy(f, destination); err != nil {
- return nil, fmt.Errorf("unable to copy manifest %s: %w", f, err)
+ if utils.IsURL(f) {
+ mname := fmt.Sprintf("manifest-%s-%d.yaml", manifest.Name, idx)
+ destination = filepath.Join(componentPath.Manifests, mname)
+ if err := utils.DownloadToFile(f, destination, component.CosignKeyPath); err != nil {
+ return nil, fmt.Errorf(lang.ErrDownloading, f, err.Error())
+ }
+ // Update the manifest path to the new location.
+ manifest.Files[idx] = mname
+ } else {
+ // If using a temp directory, trim the temp directory from the path.
+ trimmedPath = strings.TrimPrefix(f, componentPath.Temp)
+ destination = filepath.Join(componentPath.Manifests, trimmedPath)
+ if err := utils.CreatePathAndCopy(f, destination); err != nil {
+ return nil, fmt.Errorf("unable to copy manifest %s: %w", f, err)
+ }
+ // Update the manifest path to the new location.
+ manifest.Files[idx] = trimmedPath
  }
-
- // Update the manifest path to the new location.
- manifest.Files[idx] = trimmedPath
  }
 
  for idx, k := range manifest.Kustomizations {
  // Generate manifests from kustomizations and place in the package.
  spinner.Updatef("Building kustomization for %s", k)
- destination := fmt.Sprintf("%s/kustomization-%s-%d.yaml", componentPath.Manifests, manifest.Name, idx)
- if err := kustomize.BuildKustomization(k, destination, manifest.KustomizeAllowAnyDirectory); err != nil {
+ kname := fmt.Sprintf("kustomization-%s-%d.yaml", manifest.Name, idx)
+ destination := filepath.Join(componentPath.Manifests, kname)
+ if err := kustomize.Build(k, destination, manifest.KustomizeAllowAnyDirectory); err != nil {
  return nil, fmt.Errorf("unable to build kustomization %s: %w", k, err)
  }
  }

src/pkg/packager/network.go

@@ -119,11 +119,8 @@ func (p *Packager) handleSgetPackage() error {
  p.cfg.DeployOpts.SGetKeyPath = "env://DU_SGET_KEY"
  }
 
- // Remove the 'sget://' header for the actual sget call
- remoteBlob := strings.TrimPrefix(opts.PackagePath, "sget://")
-
  // Sget the package
- err = utils.Sget(context.TODO(), remoteBlob, p.cfg.DeployOpts.SGetKeyPath, destinationFile)
+ err = utils.Sget(context.TODO(), opts.PackagePath, p.cfg.DeployOpts.SGetKeyPath, destinationFile)
  if err != nil {
  return fmt.Errorf("unable to get the remote package via sget: %w", err)
  }