feat: default-required feature flag

.github/workflows/test-upgrade.yml

@@ -69,7 +69,7 @@ jobs:
 
  - name: Initialize the cluster with the release version
  # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
- # in a previous step. This test run will the current release to create a K3s cluster.
+ # in a previous step. This test run will use the current release to create a K3s cluster.
  run: |
  sudo env "PATH=$PATH" CI=true zarf init --components k3s,git-server,logging --nodeport 31337 --confirm
 
@@ -79,10 +79,10 @@ jobs:
 
  - name: Create and deploy the upgrade test packages
  # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
- # in a previous step. This test run will the current release to create a K3s cluster.
+ # in a previous step. This test run will use the current release to create a K3s cluster.
  run: |
- zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.3 --confirm
- sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --confirm
+ zarf package create src/test/upgrade/pre-0.33 --set PODINFO_VERSION=6.3.3 --confirm
+ sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --components="test-upgrade-package" --confirm
  sudo env "PATH=$PATH" CI=true zarf tools kubectl describe deployments -n=podinfo-upgrade
  sudo env "PATH=$PATH" CI=true zarf tools kubectl describe pods -n=podinfo-upgrade
 
@@ -105,12 +105,10 @@ jobs:
 
  - name: Run the upgrade tests
  # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
- # in a previous step. This test run will the current release to create a K3s cluster.
+ # in a previous step. This test run will use the current release to create a K3s cluster.
  run: |
  sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
 
- zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.4 --confirm
-
  sudo env "PATH=$PATH" CI=true make test-upgrade ARCH=amd64
 
  - name: Save logs

Makefile

@@ -30,7 +30,7 @@ else
  endif
 endif
 
-CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion")
+CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"unset-development-only")
 BUILD_ARGS := -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)
 K8S_MODULES_VER=$(subst ., ,$(subst v,,$(shell go list -f '{{.Version}}' -m k8s.io/client-go)))
 K8S_MODULES_MAJOR_VER=$(shell echo $$(($(firstword $(K8S_MODULES_VER)) + 1)))
@@ -193,12 +193,13 @@ test-upgrade: ## Run the Zarf CLI E2E tests for an external registry and cluster
  @test -s $(ZARF_BIN) || $(MAKE) build-cli
  [ -n "$(shell zarf version)" ] || (echo "Zarf must be installed prior to the upgrade test" && exit 1)
  [ -n "$(shell zarf package list 2>&1 | grep test-upgrade-package)" ] || (echo "Zarf must be initialized and have the 6.3.3 upgrade-test package installed prior to the upgrade test" && exit 1)
- @test -s "zarf-package-test-upgrade-package-amd64-6.3.4.tar.zst" || zarf package create src/test/upgrade/ --set PODINFO_VERSION=6.3.4 --confirm
+ @test -s ./zarf-package-test-upgrade-package-$(ARCH)-6.3.4.tar.zst || zarf package create src/test/upgrade/pre-0.33 --set PODINFO_VERSION=6.3.4 --confirm
  cd src/test/upgrade && go test -failfast -v -timeout 30m
 
 .PHONY: test-unit
 test-unit: ## Run unit tests
  cd src/pkg && go test ./... -failfast -v -timeout 30m
+ cd src/types && go test ./... -failfast -v -timeout 30m
  cd src/internal && go test ./... -failfast -v timeout 30m
  cd src/extensions/bigbang && go test ./. -failfast -v timeout 30m
 

adr/0017-zarf-bundle.md

@@ -14,7 +14,6 @@ Currently there is no official way to enable the deployment, publishing, pulling
 
 ```yaml
 - name: init
- required: true
  files:
  - source: zarf-init-amd64-v0.27.0.tar.zst
  target: zarf-init-amd64-v0.27.0.tar.zst

adr/0023-required-optional.md

@@ -0,0 +1,39 @@
+# 23. Required -> Optional
+
+Date: 2024-01-02
+
+## Status
+
+Pending
+
+## Context
+
+> Feature request: <https://github.com/defenseunicorns/zarf/issues/2059>
+
+Currently, all Zarf components default to being optional due to the `required` key being _optional_ in the yaml. This leads to package authors needing to ensure that they annotate this key for each component, and since nothing in the current validations prompts them about this they may be confused about the "all things are optional" default state.
+
+When Zarf was first created, we didn't really know how it would evolve and this key was introduced in those very early days. At this point it would be better to require all components by default--especially with the introduction of composability and the OCI skeleton work, there is plenty of flexibility in the API to compose bespoke packages assembled from other packages.
+
+A few ways to handle this:
+
+1. Simply force the `required` key to be a non-optional, so that package authors would be forced to specify it for each component, thereby removing any ambiguity--but also force one more key for every single component ever created 🫠
+
+2. Deprecate `required` and introduce an optional `optional` key, which would default to _false_.
+
+3. Do something more significant like combine various condition-based things such as `only`, `optional` (instead of `required`), or `default`.
+
+## Decision
+
+> The change that we're proposing or have agreed to implement.
+
+Option 2: deprecate `required` and introduce an optional `optional` key, which defaults to _false_.
+
+Components are now **required** by default, instead of **optional**.
+
+## Consequences
+
+`zarf package create` will fail if any usage of `required` is detected in the `zarf.yaml`, resulting in some thrash for package creators.
+
+Packages created w/ Zarf v0.33.0+ will have their implicit _required_ logic flipped from previous versions (implicit `required: false` --> implicit `optional: false`).
+
+A `required-to-optional` migration.

docs-website/static/docs/tutorials/differential_package_edit.html

@@ -59,7 +59,6 @@
  version: "1.4.2"
 components:
  - name: longhorn-environment-check
- required: true
  files:
  - source: https://raw.githubusercontent.com/longhorn/longhorn/v1.4.2/scripts/environment_check.sh
  target: environment_check.sh
@@ -78,7 +77,6 @@
  chmod +x ./environment_check.sh
  ./environment_check.sh
  - name: longhorn
- required: true
  description: "Deploy Longhorn into a Kubernetes cluster. https://longhorn.io"
  actions:
  # Set the delete confirmation flag for Longhorn

docs-website/static/docs/tutorials/publish_and_deploy_manifest.html

@@ -62,7 +62,6 @@
 
 components:
  - name: helm-oci-chart
- required: true
  charts:
  - name: podinfo
  version: 6.3.3

docs-website/static/docs/tutorials/troubleshoot_version_required_no_version.html

@@ -57,7 +57,6 @@
 
 components:
  - name: helm-oci-chart
- required: true
  charts:
  - name: podinfo
  version: 6.3.3

docs/2-the-zarf-cli/100-cli-commands/zarf_dev.md

@@ -29,6 +29,7 @@ Commands useful for developing packages
 * [zarf dev find-images](zarf_dev_find-images.md) - Evaluates components in a Zarf file to identify images specified in their helm charts and manifests
 * [zarf dev generate-config](zarf_dev_generate-config.md) - Generates a config file for Zarf
 * [zarf dev lint](zarf_dev_lint.md) - Lints the given package for valid schema and recommended practices
+* [zarf dev migrate](zarf_dev_migrate.md) - [beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format
 * [zarf dev patch-git](zarf_dev_patch-git.md) - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE. NOTE:
 This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook.
 * [zarf dev sha256sum](zarf_dev_sha256sum.md) - Generates a SHA256SUM for the given file

docs/2-the-zarf-cli/100-cli-commands/zarf_dev_migrate.md

@@ -0,0 +1,38 @@
+# zarf dev migrate
+<!-- Auto-generated by hack/gen-cli-docs.sh -->
+
+[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format
+
+## Synopsis
+
+[beta] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format.
+
+This command modifies the original zarf.yaml file, performs a best effort attempt to preserve comments, and will format the contents in an opinionated manner.
+
+```
+zarf dev migrate [flags]
+```
+
+## Options
+
+```
+ -h, --help help for migrate
+ --run stringArray migrations to run (default: all, available: scripts-to-actions, pluralize-set-variable, required-to-optional)
+```
+
+## Options inherited from parent commands
+
+```
+ -a, --architecture string Architecture for OCI images and Zarf packages
+ --insecure Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture.
+ -l, --log-level string Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info")
+ --no-color Disable colors in output
+ --no-log-file Disable log file creation
+ --no-progress Disable fancy UI progress bars, spinners, logos, etc
+ --tmpdir string Specify the temporary directory to use for intermediate files
+ --zarf-cache string Specify the location of the Zarf cache directory (default "~/.zarf-cache")
+```
+
+## SEE ALSO
+
+* [zarf dev](zarf_dev.md) - Commands useful for developing packages