From 6ef3b498e3960a893a2c949706c685e446757e7e Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 25 Apr 2023 18:32:37 -0500 Subject: [PATCH] Discover jobs/hooks in find images and diff git repo patches (#1645) ## Description Implements hook discovery for images and shows changes from patch-git. ## Related Issue Fixes #1246 Relates to #1625 ## Type of change - [X] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [X] Other (security config, docs update, etc) ## Checklist before merging - [ ] Test, docs, adr added or updated as needed - [X] [Contributor Guide Steps](https://github.com/defenseunicorns/zarf/blob/main/CONTRIBUTING.md#developer-workflow) followed --- go.mod | 2 +- src/cmd/prepare.go | 3 +++ src/internal/packager/helm/chart.go | 8 +++++++- src/pkg/message/message.go | 12 ++++++++++++ src/pkg/packager/prepare.go | 30 +++++++++++++++++++---------- src/pkg/transform/git.go | 2 +- src/pkg/transform/git_test.go | 10 +++++----- 7 files changed, 49 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 9233ef38f3..cd9a81fab3 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/otiai10/copy v1.11.0 github.com/pkg/errors v0.9.1 github.com/pterm/pterm v0.12.59 + github.com/sergi/go-diff v1.3.1 github.com/sigstore/cosign v1.13.1 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.15.0 @@ -320,7 +321,6 @@ require ( github.com/scylladb/go-set v1.0.3-0.20200225121959-cc7b2070d91e // indirect github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect github.com/segmentio/ksuid v1.0.4 // indirect - github.com/sergi/go-diff v1.3.1 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/sigstore/fulcio v0.6.0 // indirect diff --git a/src/cmd/prepare.go b/src/cmd/prepare.go index ce55d3ef2d..ea44917160 100644 --- a/src/cmd/prepare.go +++ b/src/cmd/prepare.go @@ -47,6 +47,9 @@ var prepareTransformGitLinks = &cobra.Command{ text := string(content) processedText := transform.MutateGitURLsInText(pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername) + // Print the differences + message.PrintDiff(text, processedText) + // Ask the user before this destructive action confirm := false prompt := &survey.Confirm{ diff --git a/src/internal/packager/helm/chart.go b/src/internal/packager/helm/chart.go index e851adc17c..e30feb0151 100644 --- a/src/internal/packager/helm/chart.go +++ b/src/internal/packager/helm/chart.go @@ -182,9 +182,15 @@ func (h *Helm) TemplateChart() (string, error) { return "", fmt.Errorf("error generating helm chart template: %w", err) } + manifest := templatedChart.Manifest + + for _, hook := range templatedChart.Hooks { + manifest += fmt.Sprintf("\n---\n%s", hook.Manifest) + } + spinner.Success() - return templatedChart.Manifest, nil + return manifest, nil } // GenerateChart generates a helm chart for a given Zarf manifest. diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index e304670f8a..c97d496729 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -15,6 +15,7 @@ import ( "time" "github.com/pterm/pterm" + "github.com/sergi/go-diff/diffmatchpatch" ) // LogLevel is the level of logging to display. @@ -243,6 +244,17 @@ func Paragraphn(n int, format string, a ...any) string { return pterm.DefaultParagraph.WithMaxWidth(n).Sprintf(format, a...) } +// PrintDiff prints the differences between a and b with a as original and b as new +func PrintDiff(textA, textB string) { + dmp := diffmatchpatch.New() + + diffs := dmp.DiffMain(textA, textB, true) + + diffs = dmp.DiffCleanupSemantic(diffs) + + pterm.Println(dmp.DiffPrettyText(diffs)) +} + func debugPrinter(offset int, a ...any) { printer := pterm.Debug.WithShowLineNumber(logLevel > 2).WithLineNumberOffset(offset) now := time.Now().Format(time.RFC3339) diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index 8b9aeed087..45d4f49d12 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -20,6 +20,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" "github.com/google/go-containerregistry/pkg/crane" v1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" ) @@ -58,7 +59,7 @@ func (p *Packager) FindImages(baseDir, repoHelmChartPath string) error { } } - fmt.Printf("components:\n") + componentDefinition := "\ncomponents:\n" for _, component := range p.cfg.Pkg.Components { @@ -204,16 +205,16 @@ func (p *Packager) FindImages(baseDir, repoHelmChartPath string) error { if sortedImages := k8s.SortImages(matchedImages, nil); len(sortedImages) > 0 { // Log the header comment - fmt.Printf("\n - name: %s\n images:\n", component.Name) + componentDefinition += fmt.Sprintf("\n - name: %s\n images:\n", component.Name) for _, image := range sortedImages { // Use print because we want this dumped to stdout - fmt.Println(" - " + image) + componentDefinition += fmt.Sprintf(" - %s\n", image) } } // Handle the "maybes" if sortedImages := k8s.SortImages(maybeImages, matchedImages); len(sortedImages) > 0 { - var realImages []string + var validImages []string for _, image := range sortedImages { if descriptor, err := crane.Head(image, config.GetCraneOptions(config.CommonOptions.Insecure)...); err != nil { // Test if this is a real image, if not just quiet log to debug, this is normal @@ -221,19 +222,21 @@ func (p *Packager) FindImages(baseDir, repoHelmChartPath string) error { } else { // Otherwise, add to the list of images message.Debugf("Imaged digest found: %s", descriptor.Digest) - realImages = append(realImages, image) + validImages = append(validImages, image) } } - if len(realImages) > 0 { - fmt.Printf(" # Possible images - %s - %s\n", p.cfg.Pkg.Metadata.Name, component.Name) - for _, image := range realImages { - fmt.Println(" - " + image) + if len(validImages) > 0 { + componentDefinition += fmt.Sprintf(" # Possible images - %s - %s\n", p.cfg.Pkg.Metadata.Name, component.Name) + for _, image := range validImages { + componentDefinition += fmt.Sprintf(" - %s\n", image) } } } } + fmt.Println(componentDefinition) + // In case the directory was changed, reset to prevent breaking relative target paths if originalDir != "" { _ = os.Chdir(originalDir) @@ -244,7 +247,7 @@ func (p *Packager) FindImages(baseDir, repoHelmChartPath string) error { func (p *Packager) processUnstructured(resource *unstructured.Unstructured, matchedImages, maybeImages k8s.ImageMap) (k8s.ImageMap, k8s.ImageMap, error) { var imageSanityCheck = regexp.MustCompile(`(?mi)"image":"([^"]+)"`) - var imageFuzzyCheck = regexp.MustCompile(`(?mi)"([a-z0-9\-./]+:[\w][\w.\-]{0,127})"`) + var imageFuzzyCheck = regexp.MustCompile(`(?mi)"([a-z0-9\-.\/]+:[\w][\w.\-]{0,127})"`) var json string contents := resource.UnstructuredContent() @@ -282,6 +285,13 @@ func (p *Packager) processUnstructured(resource *unstructured.Unstructured, matc } matchedImages = k8s.BuildImageMap(matchedImages, replicaSet.Spec.Template.Spec) + case "Job": + var job batchv1.Job + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(contents, &job); err != nil { + return matchedImages, maybeImages, fmt.Errorf("could not parse job: %w", err) + } + matchedImages = k8s.BuildImageMap(matchedImages, job.Spec.Template.Spec) + default: // Capture any custom images matches := imageSanityCheck.FindAllStringSubmatch(json, -1) diff --git a/src/pkg/transform/git.go b/src/pkg/transform/git.go index 076f4b081b..84bb23386d 100644 --- a/src/pkg/transform/git.go +++ b/src/pkg/transform/git.go @@ -18,7 +18,7 @@ var gitURLRegex = regexp.MustCompile(`^(?P[a-z]+:\/\/)(?P.+?)\/ // MutateGitURLsInText changes the gitURL hostname to use the repository Zarf is configured to use. func MutateGitURLsInText(targetBaseURL string, text string, pushUser string) string { - extractPathRegex := regexp.MustCompilePOSIX(`https?://[^/]+/(.*\.git)`) + extractPathRegex := regexp.MustCompile(`[a-z]+:\/\/[^\/]+\/(.*\.git)`) output := extractPathRegex.ReplaceAllStringFunc(text, func(match string) string { output, err := GitTransformURL(targetBaseURL, match, pushUser) if err != nil { diff --git a/src/pkg/transform/git_test.go b/src/pkg/transform/git_test.go index f7bac2c0f4..e4c28c2c44 100644 --- a/src/pkg/transform/git_test.go +++ b/src/pkg/transform/git_test.go @@ -46,8 +46,8 @@ func TestMutateGitURLsInText(t *testing.T) { # We transform https://*/*.git URLs https://github.com/defenseunicorns/zarf.git # Even URLs with things on either side - stuffhttps://github.com/defenseunicorns/zarf.gitandthings - # But not ssh://*/*.git URLs + stuff https://github.com/defenseunicorns/zarf.git andthings + # Including ssh://*/*.git URLs ssh://git@github.com/defenseunicorns/zarf.git # Or non .git URLs https://www.defenseunicorns.com/ @@ -58,9 +58,9 @@ func TestMutateGitURLsInText(t *testing.T) { # We transform https://*/*.git URLs https://gitlab.com/repo-owner/zarf-1211668992.git # Even URLs with things on either side - stuffhttps://gitlab.com/repo-owner/zarf-1211668992.gitandthings - # But not ssh://*/*.git URLs - ssh://git@github.com/defenseunicorns/zarf.git + stuff https://gitlab.com/repo-owner/zarf-1211668992.git andthings + # Including ssh://*/*.git URLs + https://gitlab.com/repo-owner/zarf-2566185087.git # Or non .git URLs https://www.defenseunicorns.com/ `