diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index 615161e157..93035f4135 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -975,6 +975,22 @@ Must be one of: +
+ + extractPath + +  +
+ +**Description:** Local folder or file to be extracted from a 'source' archive + +| | | +| -------- | -------- | +| **Type** | `string` | + +
+
+ diff --git a/packages/distros/eks/zarf.yaml b/packages/distros/eks/zarf.yaml index c09bbfd2c1..a537280be7 100644 --- a/packages/distros/eks/zarf.yaml +++ b/packages/distros/eks/zarf.yaml @@ -29,37 +29,34 @@ variables: components: - name: load-eksctl required: true - actions: - onDeploy: - after: - # Remove existing eksctl - - cmd: rm -f eksctl - # Extract the correct linux or mac binary from the tarball - - cmd: ./zarf tools archiver decompress archives/eksctl_$(uname -s)_$(uname -m).tar.gz . - # Cleanup temp files - - cmd: rm -fr archives files: - source: eks.yaml target: eks.yaml - source: https://github.com/weaveworks/eksctl/releases/download/v0.147.0/eksctl_Darwin_amd64.tar.gz - target: archives/eksctl_Darwin_x86_64.tar.gz - shasum: d3b2a204f68eaf151b8b79bb3a28857d45d5d56353b5c430a4cd34161c8fe6fe + target: binaries/eksctl_Darwin_x86_64 + executable: true + shasum: 6d72fe0bafa5ac62e1da3889b6987af6192b330abd9c50491bcf1c5966358f89 + extractPath: eksctl - source: https://github.com/weaveworks/eksctl/releases/download/v0.147.0/eksctl_Darwin_arm64.tar.gz - target: archives/eksctl_Darwin_arm64.tar.gz - shasum: bfc14880a3c5c8fec0e338726fdfa52e375dce0a8bfa766a34e4c4224ec5c929 + target: binaries/eksctl_Darwin_arm64 + executable: true + shasum: 1d7dd5b9907de1cb3fa7832659db29f50530444d10e77b4a8eb27aa648da6fab + extractPath: eksctl - source: https://github.com/weaveworks/eksctl/releases/download/v0.147.0/eksctl_Linux_amd64.tar.gz - target: archives/eksctl_Linux_x86_64.tar.gz - shasum: 56e5746160381a288d5ad70846f0f0b4cd7f5d51e1dfe0880043cf120a2eb10a + target: binaries/eksctl_Linux_x86_64 + executable: true + shasum: 2a47bb9c86c7531a166542aa2d8cb8e1e0be326308ebcfaf724d016abe31636b + extractPath: eksctl - name: deploy-eks-cluster description: Create an EKS cluster! actions: onDeploy: before: - - cmd: ./eksctl create cluster --dry-run -f eks.yaml + - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) create cluster --dry-run -f eks.yaml - cmd: sleep 15 - - cmd: ./eksctl create cluster -f eks.yaml - - cmd: ./eksctl utils write-kubeconfig -c ${ZARF_VAR_EKS_CLUSTER_NAME} + - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) create cluster -f eks.yaml + - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) utils write-kubeconfig -c ${ZARF_VAR_EKS_CLUSTER_NAME} - cmd: ./zarf tools kubectl create namespace zarf - cmd: ./zarf tools kubectl create secret generic zarf-eks-yaml -n zarf --from-file=eks.yaml @@ -72,8 +69,8 @@ components: - cmd: ./zarf tools kubectl get secret -n zarf zarf-eks-yaml -o jsonpath='{.data.*}' | base64 -d > eks.yaml # TODO: Error handling in case the eks.yaml isn't what we expect ??? # Use eksctl to delete the cluster - - cmd: ./eksctl delete cluster -f eks.yaml --disable-nodegroup-eviction --wait + - cmd: ./binaries/eksctl_$(uname -s)_$(uname -m) delete cluster -f eks.yaml --disable-nodegroup-eviction --wait after: # clean up after ourselves + - cmd: rm -rf binaries - cmd: rm -f eks.yaml - - cmd: rm -f eksctl diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 8e9ac8884f..10a65bb925 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -29,6 +29,8 @@ const ( ErrRemoveFile = "failed to remove file %s: %s" ErrUnarchive = "failed to unarchive %s: %s" ErrConfirmCancel = "confirm selection canceled: %s" + ErrFileExtract = "failed to extract filename %s from archive %s: %s" + ErrFileNameExtract = "failed to extract filename from URL %s: %s" ) // Zarf CLI commands. diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index ef1d712a00..62d7177347 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -5,7 +5,6 @@ package packager import ( - "crypto" "errors" "fmt" "os" @@ -295,7 +294,6 @@ func (p *Packager) getFilesToSBOM(component types.ZarfComponent) (*types.Compone for filesIdx, file := range component.Files { path := filepath.Join(componentPath.Files, strconv.Itoa(filesIdx), filepath.Base(file.Target)) - appendSBOMFiles(path) } @@ -384,28 +382,78 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent, isSkel rel := filepath.Join(types.FilesFolder, strconv.Itoa(filesIdx), filepath.Base(file.Target)) dst := filepath.Join(componentPath.Base, rel) + destinationDir := filepath.Dir(dst) if helpers.IsURL(file.Source) { if isSkeleton { continue } - if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { - return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + + if file.ExtractPath != "" { + + // get the compressedFileName from the source + compressedFileName, err := helpers.ExtractBasePathFromURL(file.Source) + if err != nil { + return fmt.Errorf(lang.ErrFileNameExtract, file.Source, err.Error()) + } + + compressedFile := filepath.Join(componentPath.Temp, compressedFileName) + + // If the file is an archive, download it to the componentPath.Temp + if err := utils.DownloadToFile(file.Source, compressedFile, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } + + err = archiver.Extract(compressedFile, file.ExtractPath, destinationDir) + if err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, compressedFileName, err.Error()) + } + + } else { + if err := utils.DownloadToFile(file.Source, dst, component.DeprecatedCosignKeyPath); err != nil { + return fmt.Errorf(lang.ErrDownloading, file.Source, err.Error()) + } } + } else { - if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { - return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + if file.ExtractPath != "" { + err = archiver.Extract(file.Source, file.ExtractPath, destinationDir) + if err != nil { + return fmt.Errorf(lang.ErrFileExtract, file.ExtractPath, file.Source, err.Error()) + } + } else { + if err := utils.CreatePathAndCopy(file.Source, dst); err != nil { + return fmt.Errorf("unable to copy file %s: %w", file.Source, err) + } } - if isSkeleton { - p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel + + } + + if file.ExtractPath != "" { + // Make sure dst reflects the actual file or directory. + updatedExtractedFileOrDir := filepath.Join(destinationDir, file.ExtractPath) + if updatedExtractedFileOrDir != dst { + err = os.Rename(updatedExtractedFileOrDir, dst) + if err != nil { + return fmt.Errorf(lang.ErrWritingFile, dst, err) + } } } + if isSkeleton { + // Change the source to the new relative source directory (any remote files will have been skipped above) + p.cfg.Pkg.Components[index].Files[filesIdx].Source = rel + // Remove the extractPath from a skeleton since it will already extract it + p.cfg.Pkg.Components[index].Files[filesIdx].ExtractPath = "" + } + // Abort packaging on invalid shasum (if one is specified). if file.Shasum != "" { - if actualShasum, _ := utils.GetCryptoHashFromFile(dst, crypto.SHA256); actualShasum != file.Shasum { - return fmt.Errorf("shasum mismatch for file %s: expected %s, got %s", file.Source, file.Shasum, actualShasum) + actualShasum, _ := utils.GetSHA256OfFile(dst) + if actualShasum != file.Shasum { + return fmt.Errorf("shasum mismatch for file %s: expected %s, got %s", dst, file.Shasum, actualShasum) } + } if file.Executable || utils.IsDir(dst) { diff --git a/src/pkg/utils/helpers/url.go b/src/pkg/utils/helpers/url.go index 1b7278484b..7ad3017ecd 100644 --- a/src/pkg/utils/helpers/url.go +++ b/src/pkg/utils/helpers/url.go @@ -7,6 +7,9 @@ package helpers import ( "fmt" "net/url" + "path" + + "github.com/defenseunicorns/zarf/src/config/lang" ) // Nonstandard URL schemes or prefixes @@ -40,3 +43,17 @@ func DoHostnamesMatch(url1 string, url2 string) (bool, error) { return parsedURL1.Hostname() == parsedURL2.Hostname(), nil } + +// ExtractBasePathFromURL returns filename from URL string +func ExtractBasePathFromURL(urlStr string) (string, error) { + if !IsURL(urlStr) { + return "", fmt.Errorf(lang.PkgValidateErrImportURLInvalid, urlStr) + } + parsedURL, err := url.Parse(urlStr) + if err != nil { + return "", err + } + + filename := path.Base(parsedURL.Path) + return filename, nil +} diff --git a/src/pkg/utils/helpers/url_test.go b/src/pkg/utils/helpers/url_test.go index b5955d1a2d..6c81c4279c 100644 --- a/src/pkg/utils/helpers/url_test.go +++ b/src/pkg/utils/helpers/url_test.go @@ -5,6 +5,7 @@ package helpers import ( + "fmt" "testing" "github.com/stretchr/testify/require" @@ -90,6 +91,43 @@ func (suite *TestURLSuite) Test_2_DoHostnamesMatch() { suite.False(b) } +func (suite *TestURLSuite) Test_3_ExtractBasePathFromURL() { + goodURLs := []string{ + "https://zarf.dev/file.txt", + "https://docs.zarf.dev/file.txt", + "https://zarf.dev/docs/file.tar.gz", + "https://defenseunicorns.com/file.yaml", + "https://google.com/file.md", + } + badURLs := []string{ + "invalid-url", + "am", + "not", + "a url", + "info@defenseunicorns.com", + "12345", + "kubernetes.svc.default.svc.cluster.local", + } + expectations := []string{ + "file.txt", + "file.txt", + "file.tar.gz", + "file.yaml", + "file.md", + } + + for idx, url := range goodURLs { + actualURL, err := ExtractBasePathFromURL(url) + suite.NoError(err) + suite.Equal(actualURL, expectations[idx]) + } + for _, url := range badURLs { + url, err := ExtractBasePathFromURL(url) + fmt.Println(url) + suite.Error(err) + } + +} func TestURL(t *testing.T) { suite.Run(t, new(TestURLSuite)) } diff --git a/src/test/e2e/00_use_cli_test.go b/src/test/e2e/00_use_cli_test.go index dff59f04fa..33ebcf41b7 100644 --- a/src/test/e2e/00_use_cli_test.go +++ b/src/test/e2e/00_use_cli_test.go @@ -116,6 +116,22 @@ func TestUseCLI(t *testing.T) { require.Error(t, err, stdOut, stdErr) }) + t.Run("zarf package to test archive path", func(t *testing.T) { + t.Parallel() + stdOut, stdErr, err := e2e.Zarf("package", "create", "packages/distros/eks", "--confirm") + require.NoError(t, err, stdOut, stdErr) + + path := "zarf-package-distro-eks-multi-0.0.2.tar.zst" + stdOut, stdErr, err = e2e.Zarf("package", "deploy", path, "--confirm") + require.NoError(t, err, stdOut, stdErr) + + require.FileExists(t, "binaries/eksctl_Darwin_x86_64") + require.FileExists(t, "binaries/eksctl_Darwin_arm64") + require.FileExists(t, "binaries/eksctl_Linux_x86_64") + + e2e.CleanFiles("binaries/eksctl_Darwin_x86_64", "binaries/eksctl_Darwin_arm64", "binaries/eksctl_Linux_x86_64") + }) + t.Run("zarf package create with tmpdir and cache", func(t *testing.T) { t.Parallel() tmpdir := t.TempDir() diff --git a/src/types/component.go b/src/types/component.go index b6297925e5..a6fb4a4ebd 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -79,11 +79,12 @@ type ZarfComponentOnlyCluster struct { // ZarfFile defines a file to deploy. type ZarfFile struct { - Source string `json:"source" jsonschema:"description=Local folder or file path or remote URL to pull into the package"` - Shasum string `json:"shasum,omitempty" jsonschema:"description=(files only) Optional SHA256 checksum of the file"` - Target string `json:"target" jsonschema:"description=The absolute or relative path where the file or folder should be copied to during package deploy"` - Executable bool `json:"executable,omitempty" jsonschema:"description=(files only) Determines if the file should be made executable during package deploy"` - Symlinks []string `json:"symlinks,omitempty" jsonschema:"description=List of symlinks to create during package deploy"` + Source string `json:"source" jsonschema:"description=Local folder or file path or remote URL to pull into the package"` + Shasum string `json:"shasum,omitempty" jsonschema:"description=(files only) Optional SHA256 checksum of the file"` + Target string `json:"target" jsonschema:"description=The absolute or relative path where the file or folder should be copied to during package deploy"` + Executable bool `json:"executable,omitempty" jsonschema:"description=(files only) Determines if the file should be made executable during package deploy"` + Symlinks []string `json:"symlinks,omitempty" jsonschema:"description=List of symlinks to create during package deploy"` + ExtractPath string `json:"extractPath,omitempty" jsonschema:"description=Local folder or file to be extracted from a 'source' archive"` } // ZarfChart defines a helm chart to be deployed. diff --git a/src/ui/lib/api-types.ts b/src/ui/lib/api-types.ts index 56780c6c9e..4bfa2abecf 100644 --- a/src/ui/lib/api-types.ts +++ b/src/ui/lib/api-types.ts @@ -719,6 +719,10 @@ export interface ZarfFile { * (files only) Determines if the file should be made executable during package deploy */ executable?: boolean; + /** + * Local folder or file to be extracted from a 'source' archive + */ + extractPath?: string; /** * (files only) Optional SHA256 checksum of the file */ @@ -1590,6 +1594,7 @@ const typeMap: any = { ], false), "ZarfFile": o([ { json: "executable", js: "executable", typ: u(undefined, true) }, + { json: "extractPath", js: "extractPath", typ: u(undefined, "") }, { json: "shasum", js: "shasum", typ: u(undefined, "") }, { json: "source", js: "source", typ: "" }, { json: "symlinks", js: "symlinks", typ: u(undefined, a("")) }, diff --git a/zarf.schema.json b/zarf.schema.json index d05e36f41b..8bc6520679 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -791,6 +791,10 @@ }, "type": "array", "description": "List of symlinks to create during package deploy" + }, + "extractPath": { + "type": "string", + "description": "Local folder or file to be extracted from a 'source' archive" } }, "additionalProperties": false,