From 3750a0224c8a97618f68c5cc52ee10763d40290e Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 5 Apr 2024 09:55:44 -0700 Subject: [PATCH 01/49] propogate error Signed-off-by: ashnamehrotra --- pkg/patch/patch.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index 9f153929..51a02cfb 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -178,8 +178,8 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat // determine OS family fileBytes, err := buildkit.ExtractFileFromState(ctx, c, &config.ImageState, "/etc/os-release") if err != nil { - ch <- err - return nil, fmt.Errorf("unable to extract /etc/os-release file from state %w", err) + ch <- fmt.Errorf("unable to extract /etc/os-release file from state %w", err) + return nil, err } osType, err := getOSType(ctx, fileBytes) @@ -194,6 +194,7 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat ch <- err return nil, err } + // do not specify updates, will update all updates = nil } else { From d5a1059e0f0baf6f5ad63983f1deecbd4cb87ede Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 5 Apr 2024 09:58:18 -0700 Subject: [PATCH 02/49] do not need to set unassigned updates to nil Signed-off-by: ashnamehrotra --- pkg/patch/patch.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index 51a02cfb..7358a068 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -195,8 +195,6 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return nil, err } - // do not specify updates, will update all - updates = nil } else { // get package manager based on os family type manager, err = pkgmgr.GetPackageManager(updates.Metadata.OS.Type, config, workingFolder) From 5f7e590459011d9208d7b233618434501c9a2770 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 5 Apr 2024 10:00:25 -0700 Subject: [PATCH 03/49] specify linux for OS in ExtractFileFromState and get archi Signed-off-by: ashnamehrotra --- pkg/buildkit/buildkit.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 2e79eb6e..5775726d 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -3,6 +3,7 @@ package buildkit import ( "bytes" "context" + "fmt" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/sourceresolver" @@ -29,6 +30,7 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri // Initialize buildkit config for the target image config := Config{ ImageName: image, + Platform: nil, } // Resolve and pull the config for the target image @@ -60,7 +62,12 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri // Extracts the bytes of the file denoted by `path` from the state `st`. func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, path string) ([]byte, error) { - def, err := st.Marshal(ctx) + platform, err := st.GetPlatform(ctx) + if err != nil { + return nil, fmt.Errorf("unable to get platform from ImageState %w", err) + } + + def, err := st.Marshal(ctx, llb.Platform(ispec.Platform{OS: "linux", Architecture: platform.Architecture})) if err != nil { return nil, err } From a8a6f57a4cd53081321008b68474e53dc6957963 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 5 Apr 2024 10:30:17 -0700 Subject: [PATCH 04/49] save debugging Signed-off-by: ashnamehrotra --- pkg/buildkit/buildkit.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 5775726d..2ca3e407 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -5,6 +5,7 @@ import ( "context" "fmt" + "github.com/davecgh/go-spew/spew" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/sourceresolver" gwclient "github.com/moby/buildkit/frontend/gateway/client" @@ -30,7 +31,6 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri // Initialize buildkit config for the target image config := Config{ ImageName: image, - Platform: nil, } // Resolve and pull the config for the target image @@ -68,8 +68,9 @@ func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, } def, err := st.Marshal(ctx, llb.Platform(ispec.Platform{OS: "linux", Architecture: platform.Architecture})) + spew.Dump("DEF", def) if err != nil { - return nil, err + return nil, fmt.Errorf("ONE %w", err) } resp, err := c.Solve(ctx, gwclient.SolveRequest{ @@ -77,12 +78,12 @@ func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, Definition: def.ToPB(), }) if err != nil { - return nil, err + return nil, fmt.Errorf("TWO %w", err) } ref, err := resp.SingleRef() if err != nil { - return nil, err + return nil, fmt.Errorf("THREE %w", err) } return ref.ReadFile(ctx, gwclient.ReadRequest{ From b4aa1e368ee186662060a6c08da300a622c97f3c Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 9 Apr 2024 14:54:51 -0700 Subject: [PATCH 05/49] take OSVersion as pkgmgr arg Signed-off-by: ashnamehrotra --- pkg/patch/patch.go | 48 +++++++++++++++++++++++++++++++++++---- pkg/pkgmgr/pkgmgr.go | 4 ++-- pkg/pkgmgr/pkgmgr_test.go | 12 +++++----- 3 files changed, 51 insertions(+), 13 deletions(-) diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index 7358a068..7c3e4a2d 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/davecgh/go-spew/spew" "github.com/docker/buildx/build" "github.com/docker/cli/cli/config" log "github.com/sirupsen/logrus" @@ -19,11 +20,13 @@ import ( "github.com/distribution/reference" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" gwclient "github.com/moby/buildkit/frontend/gateway/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" "github.com/moby/buildkit/util/progress/progressui" + ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/project-copacetic/copacetic/pkg/buildkit" "github.com/project-copacetic/copacetic/pkg/pkgmgr" "github.com/project-copacetic/copacetic/pkg/report" @@ -188,8 +191,14 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return nil, err } + osVersion, err := getOSVersion(ctx, fileBytes) + if err != nil { + ch <- err + return nil, err + } + // get package manager based on os family type - manager, err = pkgmgr.GetPackageManager(osType, config, workingFolder) + manager, err = pkgmgr.GetPackageManager(osType, osVersion, config, workingFolder) if err != nil { ch <- err return nil, err @@ -197,7 +206,7 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat } else { // get package manager based on os family type - manager, err = pkgmgr.GetPackageManager(updates.Metadata.OS.Type, config, workingFolder) + manager, err = pkgmgr.GetPackageManager(updates.Metadata.OS.Type, updates.Metadata.OS.Version, config, workingFolder) if err != nil { ch <- err return nil, err @@ -212,12 +221,31 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return nil, err } - def, err := patchedImageState.Marshal(ctx) + spew.Dump("Installed with patched image") + + platform, err := patchedImageState.GetPlatform(ctx) if err != nil { - ch <- err - return nil, err + return nil, fmt.Errorf("unable to get platform from ImageState %w", err) + } + + var def *llb.Definition + if platform != nil { + def, err = patchedImageState.Marshal(ctx, llb.Platform(ispec.Platform{OS: "linux", Architecture: platform.Architecture})) + if err != nil { + ch <- err + return nil, fmt.Errorf("unable to get platform from ImageState %w", err) + } + } else { + // TO DO remove this + def, err = patchedImageState.Marshal(ctx, llb.Platform(ispec.Platform{OS: "linux", Architecture: "arm64"})) + if err != nil { + ch <- err + return nil, fmt.Errorf("unable to get platform from ImageState %w", err) + } } + spew.Dump("Marshalled patched image state") + res, err := c.Solve(ctx, gwclient.SolveRequest{ Definition: def.ToPB(), Evaluate: true, @@ -318,6 +346,16 @@ func getOSType(ctx context.Context, osreleaseBytes []byte) (string, error) { } } +func getOSVersion(ctx context.Context, osreleaseBytes []byte) (string, error) { + r := bytes.NewReader(osreleaseBytes) + osData, err := osrelease.Parse(ctx, r) + if err != nil { + return "", fmt.Errorf("unable to parse os-release data %w", err) + } + + return osData["VERSION_ID"], nil +} + func dockerLoad(ctx context.Context, pipeR io.Reader) error { cmd := exec.CommandContext(ctx, "docker", "load") cmd.Stdin = pipeR diff --git a/pkg/pkgmgr/pkgmgr.go b/pkg/pkgmgr/pkgmgr.go index 80355b9a..ab0073a4 100644 --- a/pkg/pkgmgr/pkgmgr.go +++ b/pkg/pkgmgr/pkgmgr.go @@ -27,12 +27,12 @@ type PackageManager interface { GetPackageType() string } -func GetPackageManager(osType string, config *buildkit.Config, workingFolder string) (PackageManager, error) { +func GetPackageManager(osType string, osVersion string, config *buildkit.Config, workingFolder string) (PackageManager, error) { switch osType { case "alpine": return &apkManager{config: config, workingFolder: workingFolder}, nil case "debian", "ubuntu": - return &dpkgManager{config: config, workingFolder: workingFolder}, nil + return &dpkgManager{config: config, workingFolder: workingFolder, osVersion: osVersion}, nil case "cbl-mariner", "centos", "redhat", "amazon": return &rpmManager{config: config, workingFolder: workingFolder}, nil default: diff --git a/pkg/pkgmgr/pkgmgr_test.go b/pkg/pkgmgr/pkgmgr_test.go index d72b0bad..03229a41 100644 --- a/pkg/pkgmgr/pkgmgr_test.go +++ b/pkg/pkgmgr/pkgmgr_test.go @@ -17,7 +17,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an apkManager for alpine", func(t *testing.T) { // Call the GetPackageManager function with "alpine" as osType - manager, err := GetPackageManager("alpine", config, workingFolder) + manager, err := GetPackageManager("alpine", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -29,7 +29,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return a dpkgManager for debian", func(t *testing.T) { // Call the GetPackageManager function with "debian" as osType - manager, err := GetPackageManager("debian", config, workingFolder) + manager, err := GetPackageManager("debian", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -41,7 +41,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return a dpkgManager for ubuntu", func(t *testing.T) { // Call the GetPackageManager function with "ubuntu" as osType - manager, err := GetPackageManager("ubuntu", config, workingFolder) + manager, err := GetPackageManager("ubuntu", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -53,7 +53,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an rpmManager for cbl-mariner", func(t *testing.T) { // Call the GetPackageManager function with "cbl-mariner" as osType - manager, err := GetPackageManager("cbl-mariner", config, workingFolder) + manager, err := GetPackageManager("cbl-mariner", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -65,7 +65,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an rpmManager for redhat", func(t *testing.T) { // Call the GetPackageManager function with "redhat" as osType - manager, err := GetPackageManager("redhat", config, workingFolder) + manager, err := GetPackageManager("redhat", "1.0", config, workingFolder) // Assert that there is no error and the manager is not nil assert.NoError(t, err) @@ -77,7 +77,7 @@ func TestGetPackageManager(t *testing.T) { t.Run("should return an error for unsupported osType", func(t *testing.T) { // Call the GetPackageManager function with "unsupported" as osType - manager, err := GetPackageManager("unsupported", config, workingFolder) + manager, err := GetPackageManager("unsupported", "", config, workingFolder) // Assert that there is an error and the manager is nil assert.Error(t, err) From 3c1c92e873348925be063b0f83fc48c44f3e8c7c Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 9 Apr 2024 15:32:59 -0700 Subject: [PATCH 06/49] working debian changes Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 72 +++++++++++++++++++++++++++-------------- pkg/pkgmgr/dpkg_test.go | 2 +- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index e0e56abc..93ee268d 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -31,6 +31,7 @@ type dpkgManager struct { workingFolder string isDistroless bool statusdNames string + osVersion string } type dpkgStatusType uint @@ -72,15 +73,19 @@ func isLessThanDebianVersion(v1, v2 string) bool { } // Map the target image OSType & OSVersion to an appropriate tooling image. -func getAPTImageName(manifest *unversioned.UpdateManifest) string { - version := manifest.Metadata.OS.Version - if manifest.Metadata.OS.Type == "debian" { - version = strings.Split(manifest.Metadata.OS.Version, ".")[0] + "-slim" +func getAPTImageName(manifest *unversioned.UpdateManifest, osVersion string) string { + version := osVersion + osType := "debian" + + if manifest == nil || manifest.Metadata.OS.Type == "debian" { + version = strings.Split(version, ".")[0] + "-slim" + } else { + osType = manifest.Metadata.OS.Type } // TODO: support qualifying image name with designated repository - log.Debugf("Using %s:%s as basis for tooling image", manifest.Metadata.OS.Type, version) - return fmt.Sprintf("%s:%s", manifest.Metadata.OS.Type, version) + log.Debugf("Using %s:%s as basis for tooling image", osType, version) + return fmt.Sprintf("%s:%s", osType, version) } func getDPKGStatusType(b []byte) dpkgStatusType { @@ -103,16 +108,32 @@ func getDPKGStatusType(b []byte) dpkgStatusType { } func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned.UpdateManifest, ignoreErrors bool) (*llb.State, []string, error) { - // If manifest nil, update all packages (only for non-distroless right now) + // Probe for additional information to execute the appropriate update install graphs + toolImageName := getAPTImageName(manifest, dm.osVersion) + if err := dm.probeDPKGStatus(ctx, toolImageName); err != nil { + return nil, nil, err + } + + // If manifest nil, update all packages if manifest == nil { - updatedImageState, _, err := dm.installUpdates(ctx, nil) - if err != nil { - return updatedImageState, nil, err + if dm.isDistroless { + updatedImageState, _, err := dm.unpackAndMergeUpdates(ctx, nil, toolImageName) + if err != nil { + return updatedImageState, nil, err + } + // add validation in the future + return updatedImageState, nil, nil + } else { + updatedImageState, _, err := dm.installUpdates(ctx, nil) + if err != nil { + return updatedImageState, nil, err + } + // add validation in the future + return updatedImageState, nil, nil } - // add validation in the future - return updatedImageState, nil, nil } + // Else update according to specified updates // Validate and extract unique updates listed in input manifest debComparer := VersionComparer{isValidDebianVersion, isLessThanDebianVersion} updates, err := GetUniqueLatestUpdates(manifest.Updates, debComparer, ignoreErrors) @@ -124,12 +145,6 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned return &dm.config.ImageState, nil, nil } - // Probe for additional information to execute the appropriate update install graphs - toolImageName := getAPTImageName(manifest) - if err := dm.probeDPKGStatus(ctx, toolImageName); err != nil { - return nil, nil, err - } - var updatedImageState *llb.State var resultManifestBytes []byte if dm.isDistroless { @@ -306,18 +321,27 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers // Download all requested update packages without specifying the version. This works around: // - Reports being slightly out of date, where a newer security revision has displaced the one specified leading to not found errors. // - Reports not specifying version epochs correct (e.g. bsdutils=2.36.1-8+deb11u1 instead of with epoch as 1:2.36.1-8+dev11u1) - const aptDownloadTemplate = "apt download --no-install-recommends %s" - pkgStrings := []string{} - for _, u := range updates { - pkgStrings = append(pkgStrings, u.Name) + + var downloadCmd string + if updates != nil { + aptDownloadTemplate := "apt download --no-install-recommends %s" + pkgStrings := []string{} + for _, u := range updates { + pkgStrings = append(pkgStrings, u.Name) + } + downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) + } else { + downloadCmd = `sh -c "apt upgrade --download-only -y"` } - downloadCmd := fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) + + downloadPath := "/var/cache/apt/archives" downloaded := updated.Dir(downloadPath).Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() + diffState := llb.Diff(updated, downloaded) // Scripted enumeration and dpkg unpack of all downloaded packages [layer to merge with target] const extractTemplate = `find %s -name '*.deb' -exec dpkg-deb -x '{}' %s \;` extractCmd := fmt.Sprintf(extractTemplate, downloadPath, unpackPath) - unpacked := downloaded.Run(llb.Shlex(extractCmd)).Root() + unpacked := downloaded.Run(llb.AddMount(downloadPath, diffState), llb.Shlex(extractCmd)).Root() unpackedToRoot := llb.Scratch().File(llb.Copy(unpacked, unpackPath, "/", &llb.CopyInfo{CopyDirContentsOnly: true})) // Scripted extraction of all debinfo for version checking to separate layer into local mount diff --git a/pkg/pkgmgr/dpkg_test.go b/pkg/pkgmgr/dpkg_test.go index 68052e5e..e6128255 100644 --- a/pkg/pkgmgr/dpkg_test.go +++ b/pkg/pkgmgr/dpkg_test.go @@ -125,7 +125,7 @@ func TestGetAPTImageName(t *testing.T) { // Run test cases for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got := getAPTImageName(tc.manifest) + got := getAPTImageName(tc.manifest, "") if got != tc.want { t.Errorf("getAPTImageName() = %v, want %v", got, tc.want) } From a65c1737354e24f88dc53be7849d5d6db73180f0 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 9 Apr 2024 15:33:19 -0700 Subject: [PATCH 07/49] update vex unit tests for pkgmgr changes Signed-off-by: ashnamehrotra --- pkg/vex/openvex_test.go | 4 ++-- pkg/vex/vex_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/vex/openvex_test.go b/pkg/vex/openvex_test.go index a7fdedef..92716719 100644 --- a/pkg/vex/openvex_test.go +++ b/pkg/vex/openvex_test.go @@ -12,8 +12,8 @@ import ( func TestOpenVex_CreateVEXDocument(t *testing.T) { config := &buildkit.Config{} workingFolder := "/tmp" - alpineManager, _ := pkgmgr.GetPackageManager("alpine", config, workingFolder) - debianManager, _ := pkgmgr.GetPackageManager("debian", config, workingFolder) + alpineManager, _ := pkgmgr.GetPackageManager("alpine", "", config, workingFolder) + debianManager, _ := pkgmgr.GetPackageManager("debian", "", config, workingFolder) patchedImageName := "foo.io/bar:latest" t.Setenv("COPA_VEX_AUTHOR", "test author") diff --git a/pkg/vex/vex_test.go b/pkg/vex/vex_test.go index 6baef3e9..04e8717c 100644 --- a/pkg/vex/vex_test.go +++ b/pkg/vex/vex_test.go @@ -11,7 +11,7 @@ import ( func TestTryOutputVexDocument(t *testing.T) { config := &buildkit.Config{} workingFolder := "/tmp" - alpineManager, _ := pkgmgr.GetPackageManager("alpine", config, workingFolder) + alpineManager, _ := pkgmgr.GetPackageManager("alpine", "", config, workingFolder) patchedImageName := "patched" type args struct { From 5a037d3cf24f7d1b86dcafdec5a38edc0786b47e Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 19 Apr 2024 15:46:17 -0700 Subject: [PATCH 08/49] saving part of debian changes Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 91 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 93ee268d..30e1927b 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -4,11 +4,14 @@ import ( "bufio" "bytes" "context" + "encoding/json" "fmt" "path/filepath" + "regexp" "strconv" "strings" + "github.com/davecgh/go-spew/spew" "github.com/hashicorp/go-multierror" debVer "github.com/knqyf263/go-deb-version" "github.com/moby/buildkit/client/llb" @@ -31,6 +34,7 @@ type dpkgManager struct { workingFolder string isDistroless bool statusdNames string + packageInfo map[string]string osVersion string } @@ -110,7 +114,7 @@ func getDPKGStatusType(b []byte) dpkgStatusType { func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned.UpdateManifest, ignoreErrors bool) (*llb.State, []string, error) { // Probe for additional information to execute the appropriate update install graphs toolImageName := getAPTImageName(manifest, dm.osVersion) - if err := dm.probeDPKGStatus(ctx, toolImageName); err != nil { + if err := dm.probeDPKGStatus(ctx, toolImageName, (manifest == nil)); err != nil { return nil, nil, err } @@ -171,7 +175,7 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned // Probe the target image for: // - DPKG status type to distinguish between regular and distroless images. // - Whether status.d contains base64-encoded package names. -func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) error { +func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, updateAll bool) error { imagePlatform, err := dm.config.ImageState.GetPlatform(ctx) if err != nil { return fmt.Errorf("unable to get image platform %w", err) @@ -211,6 +215,7 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) er elif [ -d "$DPKG_STATUS_FOLDER" ]; then status="$DPKG_STATUS_IS_DIRECTORY" ls -1 "$DPKG_STATUS_FOLDER" > "$RESULT_STATUSD_PATH" + mv "$DPKG_STATUS_FOLDER"/* "$RESULTS_PATH" fi echo -n "$status" > "${RESULTS_PATH}/${STATUSD_OUTPUT_FILENAME}" `, @@ -230,8 +235,30 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) er if err != nil { return err } + dm.statusdNames = strings.ReplaceAll(string(statusdNamesBytes), "\n", " ") dm.statusdNames = strings.TrimSpace(dm.statusdNames) + + // In the case of updating all packages, save package names and versions + if updateAll { + namesList := strings.Fields(dm.statusdNames) + packageInfo := make(map[string]string) + for _, name := range namesList { + fileBtyes, err := buildkit.ExtractFileFromState(ctx, dm.config.Client, &resultsState, name) + if err != nil { + return err + } + pkgName, pkgVersion, err := getPackageInfo(string(fileBtyes)) + if err != nil { + return err + } + + packageInfo[pkgName] = pkgVersion + } + + dm.packageInfo = packageInfo + } + log.Infof("Processed status.d: %s", dm.statusdNames) dm.isDistroless = true return nil @@ -242,6 +269,30 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string) er } } +func getPackageInfo(file string) (string, string, error) { + var packageName string + var packageVersion string + + packagePattern := regexp.MustCompile(`^Package:\s*(.*)`) + match := packagePattern.FindStringSubmatch(file) + if len(match) > 1 { + packageName = match[1] + } else { + return "", "", fmt.Errorf("no package name found for package") + } + + versionPattern := regexp.MustCompile(`Version:\s*(.*)`) + match = versionPattern.FindStringSubmatch(file) + if len(match) > 1 { + packageVersion = match[1] + } else { + spew.Dump(file) + return "", "", fmt.Errorf("no version found for package") + } + + return packageName, packageVersion, nil +} + // Patch a regular debian image with: // - sh and apt installed on the image // - valid dpkg status on the image @@ -318,20 +369,50 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers llb.IgnoreCache, ).Root() + // In the case of update all packages, only update packages that are not latest version + if updates == nil { + jsonPackageData, err := json.Marshal(dm.packageInfo) + if err != nil { + return nil, nil, fmt.Errorf("unable to marshal dm.packageInfo %w", err) + } + + updated = updated.Run( + llb.AddEnv("PACKAGES_PRESENT", string(jsonPackageData)), + llb.Args([]string{ + `bash`, `-c`, ` + json_str=$PACKAGES_PRESENT + update_packages="" + + while IFS=':' read -r package version; do + pkg_name=$(echo "$package" | sed 's/^"\(.*\)"$/\1/') + pkg_version=$(echo "$version" | sed 's/^"\(.*\)"$/\1/') + installed_version=$(apt show $pkg_name 2>/dev/null | awk -F ': ' '/Version:/{print $2}') + + if [ "$installed_version" != "$pkg_version" ]; then + update_packages="$update_packages $pkg_name" + fi + done <<< "$(echo "$json_str" | tr -d '{}\n' | tr ',' '\n')" + + # export UPDATE_PACKAGES=$update_packages + # or + echo "$update_packages" > packages.txt + `, + })).Root() + } + // Download all requested update packages without specifying the version. This works around: // - Reports being slightly out of date, where a newer security revision has displaced the one specified leading to not found errors. // - Reports not specifying version epochs correct (e.g. bsdutils=2.36.1-8+deb11u1 instead of with epoch as 1:2.36.1-8+dev11u1) - var downloadCmd string + pkgStrings := []string{} if updates != nil { aptDownloadTemplate := "apt download --no-install-recommends %s" - pkgStrings := []string{} for _, u := range updates { pkgStrings = append(pkgStrings, u.Name) } downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) } else { - downloadCmd = `sh -c "apt upgrade --download-only -y"` + downloadCmd = "apt download --no-install-recommends $UPDATE_PACKAGES" } downloadPath := "/var/cache/apt/archives" From eefb00f8024f983ee7641a5fc29ac012d0aed90b Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 23 Apr 2024 10:17:27 -0700 Subject: [PATCH 09/49] debian changes to read only outdated packages via packages.txt Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 30e1927b..180c830f 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -392,9 +392,9 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers update_packages="$update_packages $pkg_name" fi done <<< "$(echo "$json_str" | tr -d '{}\n' | tr ',' '\n')" - - # export UPDATE_PACKAGES=$update_packages - # or + + mkdir /var/cache/apt/archives + cd /var/cache/apt/archives echo "$update_packages" > packages.txt `, })).Root() @@ -412,11 +412,11 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers } downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) } else { - downloadCmd = "apt download --no-install-recommends $UPDATE_PACKAGES" + downloadCmd = "xargs -a packages.txt -n 1 apt download --no-install-recommends" } downloadPath := "/var/cache/apt/archives" - downloaded := updated.Dir(downloadPath).Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() + downloaded := updated.Dir(downloadPath).Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() diffState := llb.Diff(updated, downloaded) // Scripted enumeration and dpkg unpack of all downloaded packages [layer to merge with target] From c8031c80da56f7b3bea1fd706d29ecd9f8249e5c Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 23 Apr 2024 13:49:48 -0700 Subject: [PATCH 10/49] cleanup Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 180c830f..e2008b11 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -369,7 +369,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers llb.IgnoreCache, ).Root() - // In the case of update all packages, only update packages that are not latest version + // In the case of update all packages, only update packages that are not latest version. Store these packages in packages.txt. if updates == nil { jsonPackageData, err := json.Marshal(dm.packageInfo) if err != nil { @@ -386,9 +386,9 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers while IFS=':' read -r package version; do pkg_name=$(echo "$package" | sed 's/^"\(.*\)"$/\1/') pkg_version=$(echo "$version" | sed 's/^"\(.*\)"$/\1/') - installed_version=$(apt show $pkg_name 2>/dev/null | awk -F ': ' '/Version:/{print $2}') + latest_version=$(apt show $pkg_name 2>/dev/null | awk -F ': ' '/Version:/{print $2}') - if [ "$installed_version" != "$pkg_version" ]; then + if [ "$latest_version" != "$pkg_version" ]; then update_packages="$update_packages $pkg_name" fi done <<< "$(echo "$json_str" | tr -d '{}\n' | tr ',' '\n')" @@ -412,6 +412,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers } downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) } else { + // only updated the outdated pacakges from packages.txt downloadCmd = "xargs -a packages.txt -n 1 apt download --no-install-recommends" } From c0a9904b3b06b461d3ffb196a9cdae554655801a Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 23 Apr 2024 13:52:42 -0700 Subject: [PATCH 11/49] rpm changes - read from manifest2 and compare w yum info Signed-off-by: ashnamehrotra --- pkg/pkgmgr/rpm.go | 83 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index fee883d1..7d18bf59 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "path/filepath" @@ -43,6 +44,7 @@ type rpmManager struct { workingFolder string rpmTools rpmToolPaths isDistroless bool + packageInfo map[string]string } type rpmDBType uint @@ -286,6 +288,15 @@ func (rm *rpmManager) probeRPMStatus(ctx context.Context, toolImage string) erro switch rpmDB { case RPMDBManifests: rm.isDistroless = true + rpmManifest2File, err := buildkit.ExtractFileFromState(ctx, rm.config.Client, &outState, rpmManifest2) + if err != nil { + return err + } + pkgInfo, err := parseManifestFile(string(rpmManifest2File)) + if err != nil { + return err + } + rm.packageInfo = pkgInfo case RPMDBNone, RPMDBMixed: err := fmt.Errorf("could not find determine RPM DB type of target image: %v", rpmDB) log.Error(err) @@ -326,6 +337,30 @@ func (rm *rpmManager) probeRPMStatus(ctx context.Context, toolImage string) erro return nil } +func parseManifestFile(file string) (map[string]string, error) { + // split into lines + file = strings.TrimSuffix(file, "\n") + lines := strings.Split(file, "\n") + + resultMap := make(map[string]string) + + // iterate over lines + for _, line := range lines { + // split line into columns + columns := strings.Split(line, "\t") + + if len(columns) >= 2 { + // get package name and version + name := columns[0] + version := strings.TrimSuffix(columns[1], ".cm2") + resultMap[name] = version + } else { + return nil, errors.New("unable to parse rpm manifest file") + } + } + return resultMap, nil +} + // Patch a regular RPM-based image with: // - sh and an appropriate tool installed on the image (yum, dnf, microdnf) // - valid rpm database on the image @@ -394,16 +429,54 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi toolsInstalled := toolingBase.Run(llb.Shlex(installToolsCmd), llb.WithProxy(utils.GetProxy())).Root() busyboxCopied := toolsInstalled.Dir(downloadPath).Run(llb.Shlex("cp /usr/sbin/busybox .")).Root() + // In the case of update all packages, only update packages that are not latest version. Store these packages in packages.txt. + if updates == nil { + jsonPackageData, err := json.Marshal(rm.packageInfo) + if err != nil { + return nil, nil, fmt.Errorf("unable to marshal dm.packageInfo %w", err) + } + + busyboxCopied = busyboxCopied.Run( + llb.AddEnv("PACKAGES_PRESENT", string(jsonPackageData)), + llb.Args([]string{ + `bash`, `-c`, ` + json_str=$PACKAGES_PRESENT + update_packages="" + + echo "TESTING" + while IFS=':' read -r package version; do + pkg_name=$(echo "$package" | sed 's/^"\(.*\)"$/\1/') + + pkg_version=$(echo "$version" | sed 's/^"\(.*\)"$/\1/') + latest_version=$(yum info $pkg_name 2>/dev/null | grep "Version" | sed -n '$s/Version *: //p') + + if [ "$latest_version" != "$pkg_version" ]; then + update_packages="$update_packages $pkg_name" + fi + done <<< "$(echo "$json_str" | tr -d '{}\n' | tr ',' '\n')" + + echo "$update_packages" > packages.txt + `, + })).Root() + } + // Download all requested update packages without specifying the version. This works around: // - Reports being slightly out of date, where a newer security revision has displaced the one specified leading to not found errors. // - Reports not specifying version epochs correct (e.g. bsdutils=2.36.1-8+deb11u1 instead of with epoch as 1:2.36.1-8+dev11u1) // - Reports specifying remediation packages for cbl-mariner v1 instead of v2 (e.g. *.cm1.aarch64 instead of *.cm2.aarch64) - const rpmDownloadTemplate = `yumdownloader --downloadonly --downloaddir=. --best -y %s` - pkgStrings := []string{} - for _, u := range updates { - pkgStrings = append(pkgStrings, u.Name) + var downloadCmd string + if updates != nil { + const rpmDownloadTemplate = `yumdownloader --downloadonly --downloaddir=. --best -y %s` + pkgStrings := []string{} + for _, u := range updates { + pkgStrings = append(pkgStrings, u.Name) + } + downloadCmd = fmt.Sprintf(rpmDownloadTemplate, strings.Join(pkgStrings, " ")) + } else { + // only updated the outdated pacakges from packages.txt + downloadCmd = `xargs -a packages.txt -n 1 yumdownloader --downloadonly --downloaddir=. --best -y` } - downloadCmd := fmt.Sprintf(rpmDownloadTemplate, strings.Join(pkgStrings, " ")) + downloaded := busyboxCopied.Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() // Scripted enumeration and rpm install of all downloaded packages under the download folder as root From 843f264a0a850ec59bd4cd303cec6f3db3ffccbe Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 23 Apr 2024 14:18:24 -0700 Subject: [PATCH 12/49] cleanup Signed-off-by: ashnamehrotra --- pkg/pkgmgr/rpm.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 7d18bf59..a2941231 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -443,7 +443,6 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi json_str=$PACKAGES_PRESENT update_packages="" - echo "TESTING" while IFS=':' read -r package version; do pkg_name=$(echo "$package" | sed 's/^"\(.*\)"$/\1/') From 435fab5eb7f77783ae6deef2f2ca3c8243d6f33d Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 23 Apr 2024 15:42:24 -0700 Subject: [PATCH 13/49] handle platform OS case, more cleanup Signed-off-by: ashnamehrotra --- pkg/buildkit/buildkit.go | 18 ++++++++---------- pkg/patch/patch.go | 29 +++++++---------------------- pkg/pkgmgr/dpkg.go | 2 -- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 2ca3e407..fbf41de2 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -3,9 +3,8 @@ package buildkit import ( "bytes" "context" - "fmt" - "github.com/davecgh/go-spew/spew" + "github.com/containerd/containerd/platforms" "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/client/llb/sourceresolver" gwclient "github.com/moby/buildkit/frontend/gateway/client" @@ -62,15 +61,14 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri // Extracts the bytes of the file denoted by `path` from the state `st`. func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, path string) ([]byte, error) { - platform, err := st.GetPlatform(ctx) - if err != nil { - return nil, fmt.Errorf("unable to get platform from ImageState %w", err) + platform := platforms.Normalize(platforms.DefaultSpec()) + if platform.OS != "linux" { + platform.OS = "linux" } - def, err := st.Marshal(ctx, llb.Platform(ispec.Platform{OS: "linux", Architecture: platform.Architecture})) - spew.Dump("DEF", def) + def, err := st.Marshal(ctx, llb.Platform(platform)) if err != nil { - return nil, fmt.Errorf("ONE %w", err) + return nil, err } resp, err := c.Solve(ctx, gwclient.SolveRequest{ @@ -78,12 +76,12 @@ func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, Definition: def.ToPB(), }) if err != nil { - return nil, fmt.Errorf("TWO %w", err) + return nil, err } ref, err := resp.SingleRef() if err != nil { - return nil, fmt.Errorf("THREE %w", err) + return nil, err } return ref.ReadFile(ctx, gwclient.ReadRequest{ diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index 7c3e4a2d..b8174180 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -11,7 +11,7 @@ import ( "strings" "time" - "github.com/davecgh/go-spew/spew" + "github.com/containerd/containerd/platforms" "github.com/docker/buildx/build" "github.com/docker/cli/cli/config" log "github.com/sirupsen/logrus" @@ -26,7 +26,6 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" "github.com/moby/buildkit/util/progress/progressui" - ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/project-copacetic/copacetic/pkg/buildkit" "github.com/project-copacetic/copacetic/pkg/pkgmgr" "github.com/project-copacetic/copacetic/pkg/report" @@ -221,31 +220,17 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat return nil, err } - spew.Dump("Installed with patched image") + platform := platforms.Normalize(platforms.DefaultSpec()) + if platform.OS != "linux" { + platform.OS = "linux" + } - platform, err := patchedImageState.GetPlatform(ctx) + def, err := patchedImageState.Marshal(ctx, llb.Platform(platform)) if err != nil { + ch <- err return nil, fmt.Errorf("unable to get platform from ImageState %w", err) } - var def *llb.Definition - if platform != nil { - def, err = patchedImageState.Marshal(ctx, llb.Platform(ispec.Platform{OS: "linux", Architecture: platform.Architecture})) - if err != nil { - ch <- err - return nil, fmt.Errorf("unable to get platform from ImageState %w", err) - } - } else { - // TO DO remove this - def, err = patchedImageState.Marshal(ctx, llb.Platform(ispec.Platform{OS: "linux", Architecture: "arm64"})) - if err != nil { - ch <- err - return nil, fmt.Errorf("unable to get platform from ImageState %w", err) - } - } - - spew.Dump("Marshalled patched image state") - res, err := c.Solve(ctx, gwclient.SolveRequest{ Definition: def.ToPB(), Evaluate: true, diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index e2008b11..eec29d30 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" - "github.com/davecgh/go-spew/spew" "github.com/hashicorp/go-multierror" debVer "github.com/knqyf263/go-deb-version" "github.com/moby/buildkit/client/llb" @@ -286,7 +285,6 @@ func getPackageInfo(file string) (string, string, error) { if len(match) > 1 { packageVersion = match[1] } else { - spew.Dump(file) return "", "", fmt.Errorf("no version found for package") } From aae6520cc0bbb599e0f623483ae7fc06462bd7e2 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 23 Apr 2024 15:45:51 -0700 Subject: [PATCH 14/49] modify integration test to include distroless Signed-off-by: ashnamehrotra --- .../fixtures/test-images-non-distroless.json | 92 ------------------- integration/patch_test.go | 28 ++---- 2 files changed, 8 insertions(+), 112 deletions(-) delete mode 100644 integration/fixtures/test-images-non-distroless.json diff --git a/integration/fixtures/test-images-non-distroless.json b/integration/fixtures/test-images-non-distroless.json deleted file mode 100644 index d5887b30..00000000 --- a/integration/fixtures/test-images-non-distroless.json +++ /dev/null @@ -1,92 +0,0 @@ -[ - { - "image": "docker.io/grafana/grafana", - "tag": "8.5.0", - "digest": "sha256:42d3e6bc186572245aded5a0be381012adba6d89355fa9486dd81b0c634695b5", - "distro": "Alpine", - "description": "Valid apk/db, apk present", - "ignoreErrors": false - }, - { - "image": "docker.io/grafana/grafana", - "tag": "8.5.0", - "localName": "registry.copacetic.test/repo/image:tag", - "digest": "sha256:42d3e6bc186572245aded5a0be381012adba6d89355fa9486dd81b0c634695b5", - "distro": "Alpine", - "description": "Valid apk/db, apk present, locally tagged with fully-qualified name", - "ignoreErrors": false - }, - { - "image": "docker.io/library/nginx", - "tag": "1.21.6", - "digest": "sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", - "distro": "Debian", - "description": "Valid dpkg/status, apt present", - "ignoreErrors": false - }, - { - "image": "docker.io/library/nginx", - "tag": "1.21.6", - "digest": "sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", - "localName": "local/image:tag", - "distro": "Debian", - "description": "Valid dpkg/status, apt present, locally tagged with repo and image name", - "ignoreErrors": false - }, - { - "image": "registry.k8s.io/kube-proxy", - "tag": "v1.23.4", - "digest": "sha256:30116c7218264d95623d3918a50da703675755cae866cd4c324586611fcd50ea", - "distro": "Debian", - "description": "Valid dpkg/status, apt present, custom network config", - "ignoreErrors": false - }, - { - "image": "quay.io/calico/cni", - "tag": "v3.15.1", - "digest": "sha256:a925b445c2688fc9c149b20ea04faabd40610d3304a6efda68e5dada7a41b813", - "distro": "Redhat", - "description": "Valid rpm DB, microdnf & rpm present", - "ignoreErrors": false - }, - { - "image": "mcr.microsoft.com/cbl-mariner/base/core", - "tag": "2.0.20240112", - "digest": "sha256:60323975ec3aabe1840920a65237950a54c5fef6ffc811a5d26bb6bd130f1cc3", - "distro": "Mariner", - "description": "Valid rpm DB, no dnf, yum & rpm present", - "ignoreErrors": false - }, - { - "image": "mcr.microsoft.com/cbl-mariner/base/core", - "tag": "2.0.20240112-arm64", - "digest": "sha256:c85680df0ddccfd5bf0cd60ff7d0c07b0ea783bcee9ce5dc748b68c0d36e280a", - "distro": "Mariner", - "description": "Valid rpm DB, no dnf, yum & rpm present, arm64 cross-arch", - "ignoreErrors": false - }, - { - "image": "docker.io/library/centos", - "tag": "7.6.1810", - "digest": "sha256:62d9e1c2daa91166139b51577fe4f4f6b4cc41a3a2c7fc36bd895e2a17a3e4e6", - "distro": "CentOS", - "description": "Valid rpm DB, yum present", - "ignoreErrors": false - }, - { - "image": "docker.io/library/amazonlinux", - "tag": "2.0.20210326.0", - "digest": "sha256:06380711d6a8ac0b6989f7e2a4419e560796791d9c7c843753a719c73552dc30", - "distro": "Amazon Linux", - "description": "Valid rpm DB, yum present", - "ignoreErrors": false - }, - { - "image": "docker.io/grafana/grafana-image-renderer", - "tag" : "3.4.0", - "digest": "sha256:205a39f5b58f96b9ff81a0b523a60c26c86e88e76575696fcd6debde9de02197", - "distro": "Alpine", - "description": "Valid apk/db, apk present, fail to patch libssl/libcryto", - "ignoreErrors": false - } -] diff --git a/integration/patch_test.go b/integration/patch_test.go index 5304febb..e5ae02b1 100644 --- a/integration/patch_test.go +++ b/integration/patch_test.go @@ -18,8 +18,13 @@ import ( "github.com/stretchr/testify/require" ) -//go:embed fixtures/trivy_ignore.rego -var trivyIgnore []byte +var ( + //go:embed fixtures/test-images.json + testImages []byte + + //go:embed fixtures/trivy_ignore.rego + trivyIgnore []byte +) type testImage struct { Image string `json:"image"` @@ -32,25 +37,8 @@ type testImage struct { } func TestPatch(t *testing.T) { - var file []byte - var err error - - // test distroless and non-distroless - if reportFile { - file, err = os.ReadFile("fixtures/test-images.json") - if err != nil { - t.Error("Unable to read test-images", err) - } - } else { - // only test non-distroless - file, err = os.ReadFile("fixtures/test-images-non-distroless.json") - if err != nil { - t.Error("Unable to read test-images", err) - } - } - var images []testImage - err = json.Unmarshal(file, &images) + err := json.Unmarshal(testImages, &images) require.NoError(t, err) tmp := t.TempDir() From 84e90f36819d38cb338a7fc1f6f8500682a2d795 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 23 Apr 2024 15:57:13 -0700 Subject: [PATCH 15/49] clarifications Signed-off-by: ashnamehrotra --- pkg/patch/patch.go | 4 ++-- pkg/pkgmgr/dpkg.go | 4 ++-- pkg/pkgmgr/rpm.go | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index b8174180..f8591854 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -180,8 +180,8 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat // determine OS family fileBytes, err := buildkit.ExtractFileFromState(ctx, c, &config.ImageState, "/etc/os-release") if err != nil { - ch <- fmt.Errorf("unable to extract /etc/os-release file from state %w", err) - return nil, err + ch <- err + return nil, fmt.Errorf("unable to extract /etc/os-release file from state %w", err) } osType, err := getOSType(ctx, fileBytes) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index eec29d30..1bfe0432 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -238,7 +238,7 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, up dm.statusdNames = strings.ReplaceAll(string(statusdNamesBytes), "\n", " ") dm.statusdNames = strings.TrimSpace(dm.statusdNames) - // In the case of updating all packages, save package names and versions + // In the case of updating all packages, read each file to save package names and versions if updateAll { namesList := strings.Fields(dm.statusdNames) packageInfo := make(map[string]string) @@ -367,7 +367,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers llb.IgnoreCache, ).Root() - // In the case of update all packages, only update packages that are not latest version. Store these packages in packages.txt. + // In the case of update all packages, only update packages that are not already latest version. Store these packages in packages.txt. if updates == nil { jsonPackageData, err := json.Marshal(dm.packageInfo) if err != nil { diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index a2941231..19a91442 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -292,6 +292,7 @@ func (rm *rpmManager) probeRPMStatus(ctx context.Context, toolImage string) erro if err != nil { return err } + // parse container-manifest-2 to get installed package names and versions pkgInfo, err := parseManifestFile(string(rpmManifest2File)) if err != nil { return err @@ -355,7 +356,7 @@ func parseManifestFile(file string) (map[string]string, error) { version := strings.TrimSuffix(columns[1], ".cm2") resultMap[name] = version } else { - return nil, errors.New("unable to parse rpm manifest file") + return nil, errors.New("unexpected format when parsing rpm manifest file") } } return resultMap, nil From 4327c96f779681a500415c15d2414671a3b6775d Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 08:59:09 -0700 Subject: [PATCH 16/49] test all but custom dpkg/status.d Signed-off-by: ashnamehrotra --- .../fixtures/test-images-non-distroless.json | 125 ++++++++++++++++++ integration/patch_test.go | 28 ++-- 2 files changed, 145 insertions(+), 8 deletions(-) create mode 100644 integration/fixtures/test-images-non-distroless.json diff --git a/integration/fixtures/test-images-non-distroless.json b/integration/fixtures/test-images-non-distroless.json new file mode 100644 index 00000000..12cc39b8 --- /dev/null +++ b/integration/fixtures/test-images-non-distroless.json @@ -0,0 +1,125 @@ +[ + { + "image": "docker.io/grafana/grafana", + "tag": "8.5.0", + "digest": "sha256:42d3e6bc186572245aded5a0be381012adba6d89355fa9486dd81b0c634695b5", + "distro": "Alpine", + "description": "Valid apk/db, apk present", + "ignoreErrors": false + }, + { + "image": "docker.io/grafana/grafana", + "tag": "8.5.0", + "localName": "registry.copacetic.test/repo/image:tag", + "digest": "sha256:42d3e6bc186572245aded5a0be381012adba6d89355fa9486dd81b0c634695b5", + "distro": "Alpine", + "description": "Valid apk/db, apk present, locally tagged with fully-qualified name", + "ignoreErrors": false + }, + { + "image": "docker.io/library/nginx", + "tag": "1.21.6", + "digest": "sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", + "distro": "Debian", + "description": "Valid dpkg/status, apt present", + "ignoreErrors": false + }, + { + "image": "docker.io/library/nginx", + "tag": "1.21.6", + "digest": "sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", + "localName": "local/image:tag", + "distro": "Debian", + "description": "Valid dpkg/status, apt present, locally tagged with repo and image name", + "ignoreErrors": false + }, + { + "image": "registry.k8s.io/kube-proxy", + "tag": "v1.23.4", + "digest": "sha256:30116c7218264d95623d3918a50da703675755cae866cd4c324586611fcd50ea", + "distro": "Debian", + "description": "Valid dpkg/status, apt present, custom network config", + "ignoreErrors": false + }, + { + "image": "docker.io/fluent/fluent-bit", + "tag": "1.8.4", + "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", + "distro": "Google Distroless", + "description": "Custom dpkg/status.d with base64 names, no apt", + "ignoreErrors": false + }, + { + "image": "docker.io/fluent/fluent-bit", + "tag": "1.8.4", + "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", + "localName": "localimage:tag", + "distro": "Google Distroless", + "description": "Custom dpkg/status.d with base64 names, no apt, locally tagged with image name only", + "ignoreErrors": false + }, + { + "image": "docker.io/openpolicyagent/opa", + "tag": "0.46.0", + "digest": "sha256:c4b11c9b86eaba41276ae682bb6875332316242010b7523efe30f365ad0c3cb8", + "distro": "Google Distroless", + "description": "Custom dpkg/status.d with text names, no apt, libssl1", + "ignoreErrors": false + }, + { + "image": "quay.io/calico/cni", + "tag": "v3.15.1", + "digest": "sha256:a925b445c2688fc9c149b20ea04faabd40610d3304a6efda68e5dada7a41b813", + "distro": "Redhat", + "description": "Valid rpm DB, microdnf & rpm present", + "ignoreErrors": false + }, + { + "image": "mcr.microsoft.com/cbl-mariner/base/core", + "tag": "2.0.20240112", + "digest": "sha256:60323975ec3aabe1840920a65237950a54c5fef6ffc811a5d26bb6bd130f1cc3", + "distro": "Mariner", + "description": "Valid rpm DB, no dnf, yum & rpm present", + "ignoreErrors": false + }, + { + "image": "mcr.microsoft.com/cbl-mariner/base/core", + "tag": "2.0.20240112-arm64", + "digest": "sha256:c85680df0ddccfd5bf0cd60ff7d0c07b0ea783bcee9ce5dc748b68c0d36e280a", + "distro": "Mariner", + "description": "Valid rpm DB, no dnf, yum & rpm present, arm64 cross-arch", + "ignoreErrors": false + }, + { + "image": "mcr.microsoft.com/cbl-mariner/distroless/base", + "tag": "2.0.20220527", + "digest": "sha256:f550c5428df17b145851ad75983aca6d613ad4b51ca7983b2a83e67d0ac91a5d", + "distro": "Mariner Distroless", + "description": "Custom rpmmanifest files, no yum/dnf/microdnf/rpm", + "ignoreErrors": false + }, + { + "image": "docker.io/library/centos", + "tag": "7.6.1810", + "digest": "sha256:62d9e1c2daa91166139b51577fe4f4f6b4cc41a3a2c7fc36bd895e2a17a3e4e6", + "distro": "CentOS", + "description": "Valid rpm DB, yum present", + "ignoreErrors": false + }, + { + "image": "docker.io/library/amazonlinux", + "tag": "2.0.20210326.0", + "digest": "sha256:06380711d6a8ac0b6989f7e2a4419e560796791d9c7c843753a719c73552dc30", + "distro": "Amazon Linux", + "description": "Valid rpm DB, yum present", + "ignoreErrors": false + }, + { + "image": "docker.io/grafana/grafana-image-renderer", + "tag" : "3.4.0", + "digest": "sha256:205a39f5b58f96b9ff81a0b523a60c26c86e88e76575696fcd6debde9de02197", + "distro": "Alpine", + "description": "Valid apk/db, apk present, fail to patch libssl/libcryto", + "ignoreErrors": true + } +] diff --git a/integration/patch_test.go b/integration/patch_test.go index e5ae02b1..5304febb 100644 --- a/integration/patch_test.go +++ b/integration/patch_test.go @@ -18,13 +18,8 @@ import ( "github.com/stretchr/testify/require" ) -var ( - //go:embed fixtures/test-images.json - testImages []byte - - //go:embed fixtures/trivy_ignore.rego - trivyIgnore []byte -) +//go:embed fixtures/trivy_ignore.rego +var trivyIgnore []byte type testImage struct { Image string `json:"image"` @@ -37,8 +32,25 @@ type testImage struct { } func TestPatch(t *testing.T) { + var file []byte + var err error + + // test distroless and non-distroless + if reportFile { + file, err = os.ReadFile("fixtures/test-images.json") + if err != nil { + t.Error("Unable to read test-images", err) + } + } else { + // only test non-distroless + file, err = os.ReadFile("fixtures/test-images-non-distroless.json") + if err != nil { + t.Error("Unable to read test-images", err) + } + } + var images []testImage - err := json.Unmarshal(testImages, &images) + err = json.Unmarshal(file, &images) require.NoError(t, err) tmp := t.TempDir() From cfe7f56f58246edc7ce8c2416d6e653ca693c314 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 09:08:40 -0700 Subject: [PATCH 17/49] rename test file Signed-off-by: ashnamehrotra --- ...mages-non-distroless.json => test-images-update-all.json} | 0 integration/patch_test.go | 5 ++--- 2 files changed, 2 insertions(+), 3 deletions(-) rename integration/fixtures/{test-images-non-distroless.json => test-images-update-all.json} (100%) diff --git a/integration/fixtures/test-images-non-distroless.json b/integration/fixtures/test-images-update-all.json similarity index 100% rename from integration/fixtures/test-images-non-distroless.json rename to integration/fixtures/test-images-update-all.json diff --git a/integration/patch_test.go b/integration/patch_test.go index 5304febb..f2de2021 100644 --- a/integration/patch_test.go +++ b/integration/patch_test.go @@ -35,15 +35,14 @@ func TestPatch(t *testing.T) { var file []byte var err error - // test distroless and non-distroless if reportFile { file, err = os.ReadFile("fixtures/test-images.json") if err != nil { t.Error("Unable to read test-images", err) } } else { - // only test non-distroless - file, err = os.ReadFile("fixtures/test-images-non-distroless.json") + // test all images besides custom dpkg/status.d + file, err = os.ReadFile("fixtures/test-images-update-all.json") if err != nil { t.Error("Unable to read test-images", err) } From c6face3dd616c07246ade752ed5aae1cbffdd55d Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 09:16:28 -0700 Subject: [PATCH 18/49] run go mod tidy Signed-off-by: ashnamehrotra --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 00087386..63439efb 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.21.5 require ( github.com/aquasecurity/trivy v0.45.1 + github.com/containerd/containerd v1.7.13 github.com/cpuguy83/dockercfg v0.3.1 github.com/cpuguy83/go-docker v0.3.0 github.com/distribution/reference v0.5.0 @@ -57,7 +58,6 @@ require ( github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/console v1.0.4 // indirect - github.com/containerd/containerd v1.7.13 // indirect github.com/containerd/continuity v0.4.3 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.2 // indirect From e6cb18bd9009773be888414f17fe8d3c07241f60 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 09:17:28 -0700 Subject: [PATCH 19/49] revert ignore errors test case Signed-off-by: ashnamehrotra --- integration/fixtures/test-images-update-all.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/fixtures/test-images-update-all.json b/integration/fixtures/test-images-update-all.json index 12cc39b8..b2dd86c6 100644 --- a/integration/fixtures/test-images-update-all.json +++ b/integration/fixtures/test-images-update-all.json @@ -120,6 +120,6 @@ "digest": "sha256:205a39f5b58f96b9ff81a0b523a60c26c86e88e76575696fcd6debde9de02197", "distro": "Alpine", "description": "Valid apk/db, apk present, fail to patch libssl/libcryto", - "ignoreErrors": true + "ignoreErrors": false } ] From 8d332923b88224142fd916293620ae80c6a1de73 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 09:35:03 -0700 Subject: [PATCH 20/49] fix description - fluent bit container does have apt Signed-off-by: ashnamehrotra --- integration/fixtures/test-images-update-all.json | 4 ++-- integration/fixtures/test-images.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/integration/fixtures/test-images-update-all.json b/integration/fixtures/test-images-update-all.json index b2dd86c6..0430e657 100644 --- a/integration/fixtures/test-images-update-all.json +++ b/integration/fixtures/test-images-update-all.json @@ -46,7 +46,7 @@ "tag": "1.8.4", "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names, no apt", + "description": "Custom dpkg/status.d with base64 names", "ignoreErrors": false }, { @@ -55,7 +55,7 @@ "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", "localName": "localimage:tag", "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names, no apt, locally tagged with image name only", + "description": "Custom dpkg/status.d with base64 names, locally tagged with image name only", "ignoreErrors": false }, { diff --git a/integration/fixtures/test-images.json b/integration/fixtures/test-images.json index d0e801ef..b62b9e8e 100644 --- a/integration/fixtures/test-images.json +++ b/integration/fixtures/test-images.json @@ -54,7 +54,7 @@ "tag": "1.8.4", "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names, no apt", + "description": "Custom dpkg/status.d with base64 names", "ignoreErrors": false }, { @@ -63,7 +63,7 @@ "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", "localName": "localimage:tag", "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names, no apt, locally tagged with image name only", + "description": "Custom dpkg/status.d with base64 names, locally tagged with image name only", "ignoreErrors": false }, { From c89f37983f61ed6514f9606825eaf0f28a2e596f Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 09:39:12 -0700 Subject: [PATCH 21/49] fix unit test dpkg Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgr/dpkg_test.go b/pkg/pkgmgr/dpkg_test.go index e6128255..51d55158 100644 --- a/pkg/pkgmgr/dpkg_test.go +++ b/pkg/pkgmgr/dpkg_test.go @@ -125,7 +125,7 @@ func TestGetAPTImageName(t *testing.T) { // Run test cases for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got := getAPTImageName(tc.manifest, "") + got := getAPTImageName(tc.manifest, tc.manifest.Metadata.OS.Version) if got != tc.want { t.Errorf("getAPTImageName() = %v, want %v", got, tc.want) } From 3d96ff2b5240a7800233b77bb0c8600b57f79f0e Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 09:46:35 -0700 Subject: [PATCH 22/49] lint fix Signed-off-by: ashnamehrotra --- pkg/patch/patch.go | 1 - pkg/pkgmgr/dpkg.go | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/patch/patch.go b/pkg/patch/patch.go index f8591854..562839b3 100644 --- a/pkg/patch/patch.go +++ b/pkg/patch/patch.go @@ -202,7 +202,6 @@ func patchWithContext(ctx context.Context, ch chan error, image, reportFile, pat ch <- err return nil, err } - } else { // get package manager based on os family type manager, err = pkgmgr.GetPackageManager(updates.Metadata.OS.Type, updates.Metadata.OS.Version, config, workingFolder) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 1bfe0432..d1a380f0 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -46,6 +46,7 @@ const ( DPKGStatusMixed DPKGStatusInvalid // must always be the last listed + Debian = "debian" ) func (st dpkgStatusType) String() string { @@ -78,9 +79,9 @@ func isLessThanDebianVersion(v1, v2 string) bool { // Map the target image OSType & OSVersion to an appropriate tooling image. func getAPTImageName(manifest *unversioned.UpdateManifest, osVersion string) string { version := osVersion - osType := "debian" + osType := Debian - if manifest == nil || manifest.Metadata.OS.Type == "debian" { + if manifest == nil || manifest.Metadata.OS.Type == Debian { version = strings.Split(version, ".")[0] + "-slim" } else { osType = manifest.Metadata.OS.Type @@ -126,14 +127,14 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned } // add validation in the future return updatedImageState, nil, nil - } else { - updatedImageState, _, err := dm.installUpdates(ctx, nil) - if err != nil { - return updatedImageState, nil, err - } - // add validation in the future - return updatedImageState, nil, nil } + + updatedImageState, _, err := dm.installUpdates(ctx, nil) + if err != nil { + return updatedImageState, nil, err + } + // add validation in the future + return updatedImageState, nil, nil } // Else update according to specified updates From c42224b13162cc04f86b3c6d9caa4ff48afe827d Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 10:44:44 -0700 Subject: [PATCH 23/49] update golang.org/x/net to v0.23.0 Signed-off-by: ashnamehrotra --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 63439efb..1064164a 100644 --- a/go.mod +++ b/go.mod @@ -152,12 +152,12 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.19.0 // indirect + golang.org/x/crypto v0.21.0 // indirect golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.18.0 // indirect diff --git a/go.sum b/go.sum index a2e4e5df..cfd322d9 100644 --- a/go.sum +++ b/go.sum @@ -544,8 +544,8 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu+kQvCqcsoVaQgSV60o= golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= @@ -572,8 +572,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= @@ -614,13 +614,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= From 6855c3a4712df61b26b25ebe51433c7b0ff5dd53 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 10:54:59 -0700 Subject: [PATCH 24/49] downloadPath to dpkgDownloadPath const Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index d1a380f0..3a299dc6 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -24,6 +24,7 @@ const ( dpkgLibPath = "/var/lib/dpkg" dpkgStatusPath = dpkgLibPath + "/status" dpkgStatusFolder = dpkgLibPath + "/status.d" + dpkgDownloadPath = "/var/cache/apt/archives" statusdOutputFilename = "statusd_type" ) @@ -415,14 +416,13 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers downloadCmd = "xargs -a packages.txt -n 1 apt download --no-install-recommends" } - downloadPath := "/var/cache/apt/archives" - downloaded := updated.Dir(downloadPath).Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() + downloaded := updated.Dir(dpkgDownloadPath).Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() diffState := llb.Diff(updated, downloaded) // Scripted enumeration and dpkg unpack of all downloaded packages [layer to merge with target] const extractTemplate = `find %s -name '*.deb' -exec dpkg-deb -x '{}' %s \;` - extractCmd := fmt.Sprintf(extractTemplate, downloadPath, unpackPath) - unpacked := downloaded.Run(llb.AddMount(downloadPath, diffState), llb.Shlex(extractCmd)).Root() + extractCmd := fmt.Sprintf(extractTemplate, dpkgDownloadPath, unpackPath) + unpacked := downloaded.Run(llb.AddMount(dpkgDownloadPath, diffState), llb.Shlex(extractCmd)).Root() unpackedToRoot := llb.Scratch().File(llb.Copy(unpacked, unpackPath, "/", &llb.CopyInfo{CopyDirContentsOnly: true})) // Scripted extraction of all debinfo for version checking to separate layer into local mount @@ -430,7 +430,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers mkFolders := downloaded.File(llb.Mkdir(resultsPath, 0o744, llb.WithParents(true))).File(llb.Mkdir(dpkgStatusFolder, 0o744, llb.WithParents(true))) const writeFieldsTemplate = `find . -name '*.deb' -exec sh -c "dpkg-deb -f {} > %s" \;` writeFieldsCmd := fmt.Sprintf(writeFieldsTemplate, filepath.Join(resultsPath, "{}.fields")) - fieldsWritten := mkFolders.Dir(downloadPath).Run(llb.Shlex(writeFieldsCmd)).Root() + fieldsWritten := mkFolders.Dir(dpkgDownloadPath).Run(llb.Shlex(writeFieldsCmd)).Root() // Write the name and version of the packages applied to the results.manifest file for the host const outputResultsTemplate = `find . -name '*.fields' -exec sh -c 'grep "^Package:\|^Version:" {} >> %s' \;` From ab28700ca4912a2ca49d615ed5e85938af394321 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 12:09:13 -0700 Subject: [PATCH 25/49] check for .md5sums file Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 3a299dc6..98825354 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -249,12 +249,15 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, up if err != nil { return err } - pkgName, pkgVersion, err := getPackageInfo(string(fileBtyes)) - if err != nil { - return err - } - packageInfo[pkgName] = pkgVersion + if !strings.HasSuffix(name, ".md5sums") { + pkgName, pkgVersion, err := getPackageInfo(string(fileBtyes)) + if err != nil { + return err + } + + packageInfo[pkgName] = pkgVersion + } } dm.packageInfo = packageInfo From 611d610bb44b4119e4807133d3aaac49983b3949 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 12:14:07 -0700 Subject: [PATCH 26/49] patch test changes to test same set of images with ignore-errors false Signed-off-by: ashnamehrotra --- .../fixtures/test-images-update-all.json | 125 ------------------ integration/patch_test.go | 31 ++--- 2 files changed, 12 insertions(+), 144 deletions(-) delete mode 100644 integration/fixtures/test-images-update-all.json diff --git a/integration/fixtures/test-images-update-all.json b/integration/fixtures/test-images-update-all.json deleted file mode 100644 index 0430e657..00000000 --- a/integration/fixtures/test-images-update-all.json +++ /dev/null @@ -1,125 +0,0 @@ -[ - { - "image": "docker.io/grafana/grafana", - "tag": "8.5.0", - "digest": "sha256:42d3e6bc186572245aded5a0be381012adba6d89355fa9486dd81b0c634695b5", - "distro": "Alpine", - "description": "Valid apk/db, apk present", - "ignoreErrors": false - }, - { - "image": "docker.io/grafana/grafana", - "tag": "8.5.0", - "localName": "registry.copacetic.test/repo/image:tag", - "digest": "sha256:42d3e6bc186572245aded5a0be381012adba6d89355fa9486dd81b0c634695b5", - "distro": "Alpine", - "description": "Valid apk/db, apk present, locally tagged with fully-qualified name", - "ignoreErrors": false - }, - { - "image": "docker.io/library/nginx", - "tag": "1.21.6", - "digest": "sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", - "distro": "Debian", - "description": "Valid dpkg/status, apt present", - "ignoreErrors": false - }, - { - "image": "docker.io/library/nginx", - "tag": "1.21.6", - "digest": "sha256:2bcabc23b45489fb0885d69a06ba1d648aeda973fae7bb981bafbb884165e514", - "localName": "local/image:tag", - "distro": "Debian", - "description": "Valid dpkg/status, apt present, locally tagged with repo and image name", - "ignoreErrors": false - }, - { - "image": "registry.k8s.io/kube-proxy", - "tag": "v1.23.4", - "digest": "sha256:30116c7218264d95623d3918a50da703675755cae866cd4c324586611fcd50ea", - "distro": "Debian", - "description": "Valid dpkg/status, apt present, custom network config", - "ignoreErrors": false - }, - { - "image": "docker.io/fluent/fluent-bit", - "tag": "1.8.4", - "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", - "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names", - "ignoreErrors": false - }, - { - "image": "docker.io/fluent/fluent-bit", - "tag": "1.8.4", - "digest": "sha256:2d80c13c2e7e06aa6a2e54a1825c6adbb3829c8a133ff617a0a61790bd61c53d", - "localName": "localimage:tag", - "distro": "Google Distroless", - "description": "Custom dpkg/status.d with base64 names, locally tagged with image name only", - "ignoreErrors": false - }, - { - "image": "docker.io/openpolicyagent/opa", - "tag": "0.46.0", - "digest": "sha256:c4b11c9b86eaba41276ae682bb6875332316242010b7523efe30f365ad0c3cb8", - "distro": "Google Distroless", - "description": "Custom dpkg/status.d with text names, no apt, libssl1", - "ignoreErrors": false - }, - { - "image": "quay.io/calico/cni", - "tag": "v3.15.1", - "digest": "sha256:a925b445c2688fc9c149b20ea04faabd40610d3304a6efda68e5dada7a41b813", - "distro": "Redhat", - "description": "Valid rpm DB, microdnf & rpm present", - "ignoreErrors": false - }, - { - "image": "mcr.microsoft.com/cbl-mariner/base/core", - "tag": "2.0.20240112", - "digest": "sha256:60323975ec3aabe1840920a65237950a54c5fef6ffc811a5d26bb6bd130f1cc3", - "distro": "Mariner", - "description": "Valid rpm DB, no dnf, yum & rpm present", - "ignoreErrors": false - }, - { - "image": "mcr.microsoft.com/cbl-mariner/base/core", - "tag": "2.0.20240112-arm64", - "digest": "sha256:c85680df0ddccfd5bf0cd60ff7d0c07b0ea783bcee9ce5dc748b68c0d36e280a", - "distro": "Mariner", - "description": "Valid rpm DB, no dnf, yum & rpm present, arm64 cross-arch", - "ignoreErrors": false - }, - { - "image": "mcr.microsoft.com/cbl-mariner/distroless/base", - "tag": "2.0.20220527", - "digest": "sha256:f550c5428df17b145851ad75983aca6d613ad4b51ca7983b2a83e67d0ac91a5d", - "distro": "Mariner Distroless", - "description": "Custom rpmmanifest files, no yum/dnf/microdnf/rpm", - "ignoreErrors": false - }, - { - "image": "docker.io/library/centos", - "tag": "7.6.1810", - "digest": "sha256:62d9e1c2daa91166139b51577fe4f4f6b4cc41a3a2c7fc36bd895e2a17a3e4e6", - "distro": "CentOS", - "description": "Valid rpm DB, yum present", - "ignoreErrors": false - }, - { - "image": "docker.io/library/amazonlinux", - "tag": "2.0.20210326.0", - "digest": "sha256:06380711d6a8ac0b6989f7e2a4419e560796791d9c7c843753a719c73552dc30", - "distro": "Amazon Linux", - "description": "Valid rpm DB, yum present", - "ignoreErrors": false - }, - { - "image": "docker.io/grafana/grafana-image-renderer", - "tag" : "3.4.0", - "digest": "sha256:205a39f5b58f96b9ff81a0b523a60c26c86e88e76575696fcd6debde9de02197", - "distro": "Alpine", - "description": "Valid apk/db, apk present, fail to patch libssl/libcryto", - "ignoreErrors": false - } -] diff --git a/integration/patch_test.go b/integration/patch_test.go index f2de2021..2776e81f 100644 --- a/integration/patch_test.go +++ b/integration/patch_test.go @@ -18,8 +18,13 @@ import ( "github.com/stretchr/testify/require" ) -//go:embed fixtures/trivy_ignore.rego -var trivyIgnore []byte +var ( + //go:embed fixtures/test-images.json + testImages []byte + + //go:embed fixtures/trivy_ignore.rego + trivyIgnore []byte +) type testImage struct { Image string `json:"image"` @@ -32,24 +37,8 @@ type testImage struct { } func TestPatch(t *testing.T) { - var file []byte - var err error - - if reportFile { - file, err = os.ReadFile("fixtures/test-images.json") - if err != nil { - t.Error("Unable to read test-images", err) - } - } else { - // test all images besides custom dpkg/status.d - file, err = os.ReadFile("fixtures/test-images-update-all.json") - if err != nil { - t.Error("Unable to read test-images", err) - } - } - var images []testImage - err = json.Unmarshal(file, &images) + err := json.Unmarshal(testImages, &images) require.NoError(t, err) tmp := t.TempDir() @@ -59,6 +48,10 @@ func TestPatch(t *testing.T) { for _, img := range images { img := img + if !reportFile { + img.IgnoreErrors = false + } + t.Run(img.Description, func(t *testing.T) { t.Parallel() From 64763fff4d17691afe1bd45e3b266287e81c16b3 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Wed, 24 Apr 2024 12:34:28 -0700 Subject: [PATCH 27/49] add kube-proxy v1.26.1 to integration test Signed-off-by: ashnamehrotra --- integration/fixtures/test-images.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/integration/fixtures/test-images.json b/integration/fixtures/test-images.json index b62b9e8e..c3a71dab 100644 --- a/integration/fixtures/test-images.json +++ b/integration/fixtures/test-images.json @@ -49,6 +49,14 @@ "description": "Custom dpkg/status.d with text names, no apt, libssl1.1", "ignoreErrors": false }, + { + "image": "registry.k8s.io/kube-proxy", + "tag": "v1.26.1", + "digest": "sha256:1e4f13f5f5c215813fb9c9c6f56da1c0354363f2a69bd12732658f79d585864f", + "distro": "Custom Google Distroless", + "description": "Custom dpkg/status.d with text names, with both libssl1.1 and libss1", + "ignoreErrors": false + }, { "image": "docker.io/fluent/fluent-bit", "tag": "1.8.4", From 81495833105627e16508588ae4f9a19cca802f90 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Thu, 25 Apr 2024 09:07:59 -0700 Subject: [PATCH 28/49] libss1 to libssl1 Signed-off-by: ashnamehrotra --- integration/fixtures/test-images.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/fixtures/test-images.json b/integration/fixtures/test-images.json index c3a71dab..92987abf 100644 --- a/integration/fixtures/test-images.json +++ b/integration/fixtures/test-images.json @@ -54,7 +54,7 @@ "tag": "v1.26.1", "digest": "sha256:1e4f13f5f5c215813fb9c9c6f56da1c0354363f2a69bd12732658f79d585864f", "distro": "Custom Google Distroless", - "description": "Custom dpkg/status.d with text names, with both libssl1.1 and libss1", + "description": "Custom dpkg/status.d with text names, with both libssl1.1 and libssl1", "ignoreErrors": false }, { From e1284e332dab5626f2b7aa257a5d87f6855599bb Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Thu, 25 Apr 2024 11:22:09 -0700 Subject: [PATCH 29/49] add documentation Signed-off-by: ashnamehrotra --- website/docs/update-all.md | 11 +++++++++++ website/sidebars.js | 1 + 2 files changed, 12 insertions(+) create mode 100644 website/docs/update-all.md diff --git a/website/docs/update-all.md b/website/docs/update-all.md new file mode 100644 index 00000000..25ab6858 --- /dev/null +++ b/website/docs/update-all.md @@ -0,0 +1,11 @@ +--- +title: Update All Packages +--- + +Copa can be used to update all outdated packages in a container, regardless of vulnerability status. + +Simply run Copa as usual and omit the report flag: + + ```bash + copa patch -i docker.io/library/nginx:1.21.6 + ``` diff --git a/website/sidebars.js b/website/sidebars.js index 8f166204..c8bf6515 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -36,6 +36,7 @@ const sidebars = { 'custom-address', 'output', 'scanner-plugins', + 'update-all', ], }, { From 3cd095344314e44ced73d831bc0ed2321742c4f2 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 26 Apr 2024 16:19:53 -0700 Subject: [PATCH 30/49] GetPackageInfo unit test Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 4 +-- pkg/pkgmgr/dpkg_test.go | 65 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 98825354..e20ccabc 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -251,7 +251,7 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, up } if !strings.HasSuffix(name, ".md5sums") { - pkgName, pkgVersion, err := getPackageInfo(string(fileBtyes)) + pkgName, pkgVersion, err := GetPackageInfo(string(fileBtyes)) if err != nil { return err } @@ -273,7 +273,7 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, up } } -func getPackageInfo(file string) (string, string, error) { +func GetPackageInfo(file string) (string, string, error) { var packageName string var packageVersion string diff --git a/pkg/pkgmgr/dpkg_test.go b/pkg/pkgmgr/dpkg_test.go index 51d55158..64c0037b 100644 --- a/pkg/pkgmgr/dpkg_test.go +++ b/pkg/pkgmgr/dpkg_test.go @@ -339,7 +339,7 @@ func TestValidateDebianPackageVersions(t *testing.T) { } } -func Test_dpkgManager_GetPackageType(t *testing.T) { +func TestGetPackageType(t *testing.T) { type fields struct { config *buildkit.Config workingFolder string @@ -376,3 +376,66 @@ func Test_dpkgManager_GetPackageType(t *testing.T) { }) } } + +func Test_GetPackageInfo(t *testing.T) { + type fields struct { + name string + version string + errMsg string + } + tests := []struct { + name string + file string + want fields + }{ + { + name: "valid package file format", + file: `Package: tzdata + Version: 2021a-1+deb11u8 + Architecture: all + Maintainer: GNU Libc Maintainers + Installed-Size: 3393 + Depends: debconf (>= 0.5) | debconf-2.0 + Provides: tzdata-bullseye + Section: localization + Priority: required + Multi-Arch: foreign + Homepage: https://www.iana.org/time-zones + Description: time zone and daylight-saving time data + This package contains data required for the implementation of + standard local time for many representative locations around the + globe. It is updated periodically to reflect changes made by + political bodies to time zone boundaries, UTC offsets, and + daylight-saving rules.`, + want: fields{ + name: "tzdata", + version: "2021a-1+deb11u8", + errMsg: "", + }, + }, + { + name: "invalid package file format", + file: "PackageVersion", + want: fields{ + name: "", + version: "", + errMsg: "no package name found for package", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + name, version, err := GetPackageInfo(tt.file) + var errMsg string + if err == nil { + errMsg = "" + } else { + errMsg = err.Error() + } + + if name != tt.want.name || version != tt.want.version || errMsg != tt.want.errMsg { + t.Errorf("GetPackageInfo() = Name: %v, Version: %v Error: %v, want Name: %v, Version: %v, Error: %v", name, version, err, tt.want.name, tt.want.version, tt.want.errMsg) + } + }) + } +} From 6038534d14962a29b0c1a88e46c93528ad5a414f Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 26 Apr 2024 16:21:14 -0700 Subject: [PATCH 31/49] space formatting txt block Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index e20ccabc..2784c91e 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -216,7 +216,7 @@ func (dm *dpkgManager) probeDPKGStatus(ctx context.Context, toolImage string, up elif [ -d "$DPKG_STATUS_FOLDER" ]; then status="$DPKG_STATUS_IS_DIRECTORY" ls -1 "$DPKG_STATUS_FOLDER" > "$RESULT_STATUSD_PATH" - mv "$DPKG_STATUS_FOLDER"/* "$RESULTS_PATH" + mv "$DPKG_STATUS_FOLDER"/* "$RESULTS_PATH" fi echo -n "$status" > "${RESULTS_PATH}/${STATUSD_OUTPUT_FILENAME}" `, From 88656d465184d15438ff89501c024464ba5c4e15 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 26 Apr 2024 16:35:10 -0700 Subject: [PATCH 32/49] getOsVersion unit test Signed-off-by: ashnamehrotra --- pkg/patch/patch_test.go | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/pkg/patch/patch_test.go b/pkg/patch/patch_test.go index 7f0e3fac..af8da6d8 100644 --- a/pkg/patch/patch_test.go +++ b/pkg/patch/patch_test.go @@ -182,3 +182,48 @@ func TestGetOSType(t *testing.T) { }) } } + +func TestGetOSVersion(t *testing.T) { + testCases := []struct { + osRelease []byte + errMsg string + expectedOSVersion string + }{ + { + osRelease: []byte(`PRETTY_NAME="Debian GNU/Linux 11 (bullseye)" + NAME="Debian GNU/Linux" + VERSION_ID="11" + VERSION="11 (bullseye)" + VERSION_CODENAME=bullseye + ID=debian + HOME_URL="https://www.debian.org/" + SUPPORT_URL="https://www.debian.org/support" + BUG_REPORT_URL="https://bugs.debian.org/" + `), + errMsg: "", + expectedOSVersion: "11", + }, + { + osRelease: []byte("Cannot Parse Version_ID"), + errMsg: "unable to parse os-release data osrelease: malformed line \"Cannot Parse Version_ID\"", + expectedOSVersion: "", + }, + } + + for _, tc := range testCases { + t.Run("TestGetOSVersion", func(t *testing.T) { + osVersion, err := getOSVersion(context.TODO(), tc.osRelease) + + var errMsg string + if err == nil { + errMsg = "" + } else { + errMsg = err.Error() + } + + // Use testify package to assert that the output manifest and error match the expected ones + assert.Equal(t, tc.expectedOSVersion, osVersion) + assert.Equal(t, tc.errMsg, errMsg) + }) + } +} From 04bd6bffc4d40c3ff0170a65fa842dda6a4fdb19 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 26 Apr 2024 16:39:28 -0700 Subject: [PATCH 33/49] add darwin comment Signed-off-by: ashnamehrotra --- pkg/buildkit/buildkit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index fbf41de2..0555d694 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -61,6 +61,7 @@ func InitializeBuildkitConfig(ctx context.Context, c gwclient.Client, image stri // Extracts the bytes of the file denoted by `path` from the state `st`. func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, path string) ([]byte, error) { + // since platform is obtained from host, override it in the case of Darwin platform := platforms.Normalize(platforms.DefaultSpec()) if platform.OS != "linux" { platform.OS = "linux" From c1081b03bc0ee4082eeb49e312098c38cb3a7209 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 26 Apr 2024 17:39:36 -0700 Subject: [PATCH 34/49] update quickstart w documentation Signed-off-by: ashnamehrotra --- website/docs/quick-start.md | 33 +++++++++++++++++++++++++++++++++ website/docs/update-all.md | 11 ----------- website/sidebars.js | 1 - 3 files changed, 33 insertions(+), 12 deletions(-) delete mode 100644 website/docs/update-all.md diff --git a/website/docs/quick-start.md b/website/docs/quick-start.md index fb76e10e..72e399f7 100644 --- a/website/docs/quick-start.md +++ b/website/docs/quick-start.md @@ -16,7 +16,15 @@ This sample illustrates how to patch containers using vulnerability reports with * Alternatively, see [scanner plugins](#scanner-plugins) for custom scanner support. ## Sample Steps +Copa can patch images in two ways: +- Update only vulnerable packages as detected by a supported scanner report. +- Update all outdated packages in a container, regardless of vulnerability status. +:::note +The update all functionality allows you to address discrepancies that may arise between scanners and the packages they flag as vulnerable. It is important to note, however, that some upgrades can introduce dependency or compatibility conflicts. +::: + +### Patch with scanner report: 1. Scan the container image for patchable OS vulnerabilities, outputting the results to a JSON file: ```bash @@ -102,5 +110,30 @@ This sample illustrates how to patch containers using vulnerability reports with 2024/01/22 23:32:54 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576 2024/01/22 23:32:54 [notice] 1#1: start worker processes ``` + You can stop the container by opening a new shell instance and running: `docker stop nginx-test` + +### Patch all outdated packages: +1. Run Copa with a buildkit connection as described above, and omit the report flag: + ```bash + copa patch -i docker.io/library/nginx:1.21.6 + ``` +2. Run the container to verify that the image has no regressions: + ```bash + $ docker run -it --rm --name nginx-test docker.io/library/nginx:1.21.6-patched + /docker-entrypoint.sh: /docker-entrypoint.d/ is not empty, will attempt to perform configuration + /docker-entrypoint.sh: Looking for shell scripts in /docker-entrypoint.d/ + /docker-entrypoint.sh: Launching /docker-entrypoint.d/10-listen-on-ipv6-by-default.sh + 10-listen-on-ipv6-by-default.sh: info: Getting the checksum of /etc/nginx/conf.d/default.conf + 10-listen-on-ipv6-by-default.sh: info: Enabled listen on IPv6 in /etc/nginx/conf.d/default.conf + /docker-entrypoint.sh: Launching /docker-entrypoint.d/20-envsubst-on-templates.sh + /docker-entrypoint.sh: Launching /docker-entrypoint.d/30-tune-worker-processes.sh + /docker-entrypoint.sh: Configuration complete; ready for start up + 2024/01/22 23:32:54 [notice] 1#1: using the "epoll" event method + 2024/01/22 23:32:54 [notice] 1#1: nginx/1.21.6 + 2024/01/22 23:32:54 [notice] 1#1: built by gcc 10.2.1 20210110 (Debian 10.2.1-6) + 2024/01/22 23:32:54 [notice] 1#1: OS: Linux 6.2.0-1018-azure + 2024/01/22 23:32:54 [notice] 1#1: getrlimit(RLIMIT_NOFILE): 1048576:1048576 + 2024/01/22 23:32:54 [notice] 1#1: start worker processes + ``` You can stop the container by opening a new shell instance and running: `docker stop nginx-test` diff --git a/website/docs/update-all.md b/website/docs/update-all.md deleted file mode 100644 index 25ab6858..00000000 --- a/website/docs/update-all.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Update All Packages ---- - -Copa can be used to update all outdated packages in a container, regardless of vulnerability status. - -Simply run Copa as usual and omit the report flag: - - ```bash - copa patch -i docker.io/library/nginx:1.21.6 - ``` diff --git a/website/sidebars.js b/website/sidebars.js index c8bf6515..8f166204 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -36,7 +36,6 @@ const sidebars = { 'custom-address', 'output', 'scanner-plugins', - 'update-all', ], }, { From 2dbc3de3bf41f4f8861e123f014df26a5ae8eb58 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 06:32:50 -0700 Subject: [PATCH 35/49] dpkg validation update all regular and distroless Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 2784c91e..71116d6e 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -122,19 +122,17 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned // If manifest nil, update all packages if manifest == nil { if dm.isDistroless { - updatedImageState, _, err := dm.unpackAndMergeUpdates(ctx, nil, toolImageName) + updatedImageState, _, err := dm.unpackAndMergeUpdates(ctx, nil, toolImageName, ignoreErrors) if err != nil { return updatedImageState, nil, err } - // add validation in the future return updatedImageState, nil, nil } - updatedImageState, _, err := dm.installUpdates(ctx, nil) + updatedImageState, _, err := dm.installUpdates(ctx, nil, ignoreErrors) if err != nil { return updatedImageState, nil, err } - // add validation in the future return updatedImageState, nil, nil } @@ -153,12 +151,12 @@ func (dm *dpkgManager) InstallUpdates(ctx context.Context, manifest *unversioned var updatedImageState *llb.State var resultManifestBytes []byte if dm.isDistroless { - updatedImageState, resultManifestBytes, err = dm.unpackAndMergeUpdates(ctx, updates, toolImageName) + updatedImageState, resultManifestBytes, err = dm.unpackAndMergeUpdates(ctx, updates, toolImageName, ignoreErrors) if err != nil { return nil, nil, err } } else { - updatedImageState, resultManifestBytes, err = dm.installUpdates(ctx, updates) + updatedImageState, resultManifestBytes, err = dm.installUpdates(ctx, updates, ignoreErrors) if err != nil { return nil, nil, err } @@ -304,7 +302,7 @@ func GetPackageInfo(file string) (string, string, error) { // // TODO: Support Debian images with valid dpkg status but missing tools. No current examples exist in test set // i.e. extra RunOption to mount a copy of busybox-static or full apt install into the image and invoking that. -func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages) (*llb.State, []byte, error) { +func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages, ignoreErrors bool) (*llb.State, []byte, error) { // TODO: Add support for custom APT config and gpg key injection // Since this takes place in the target container, it can interfere with install actions // such as the installation of the updated debian-archive-keyring package, so it's probably best @@ -330,11 +328,16 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U installCmd = fmt.Sprintf(aptInstallTemplate, strings.Join(pkgStrings, " ")) } else { // if updates is not specified, update all packages - installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove"` + installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove 2>>"` } aptInstalled := aptUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() + // Validate no errors were encountered if updating all + if updates != nil && !ignoreErrors { + aptInstalled = aptInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + } + // Write results.manifest to host for post-patch validation const outputResultsTemplate = `sh -c 'grep "^Package:\|^Version:" "%s" >> "%s"'` outputResultsCmd := fmt.Sprintf(outputResultsTemplate, dpkgStatusPath, resultManifest) @@ -352,7 +355,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U return &patchMerge, resultsBytes, nil } -func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string) (*llb.State, []byte, error) { +func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string, ignoreErrors bool) (*llb.State, []byte, error) { imagePlatform, err := dm.config.ImageState.GetPlatform(ctx) if err != nil { return nil, nil, fmt.Errorf("unable to get image platform %w", err) @@ -415,11 +418,22 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers } downloadCmd = fmt.Sprintf(aptDownloadTemplate, strings.Join(pkgStrings, " ")) } else { - // only updated the outdated pacakges from packages.txt - downloadCmd = "xargs -a packages.txt -n 1 apt download --no-install-recommends" + // only update the outdated pacakges from packages.txt + downloadCmd = ` + packages=$(>error_log.txt + done + ` } downloaded := updated.Dir(dpkgDownloadPath).Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() + + // Validate no errors were encountered if updating all + if updates != nil && !ignoreErrors { + downloaded = downloaded.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + } + diffState := llb.Diff(updated, downloaded) // Scripted enumeration and dpkg unpack of all downloaded packages [layer to merge with target] From ef483794626462216da5b822e2f095dbb2d37b9c Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 07:31:05 -0700 Subject: [PATCH 36/49] add error_log.txt Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 71116d6e..4add9d36 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -328,7 +328,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U installCmd = fmt.Sprintf(aptInstallTemplate, strings.Join(pkgStrings, " ")) } else { // if updates is not specified, update all packages - installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove 2>>"` + installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove 2>>error_log.txt"` } aptInstalled := aptUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() From 8fa43a1953ef00d1e5ed559be183950c57cb6239 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 07:39:12 -0700 Subject: [PATCH 37/49] rpm validation update all regular and distroless Signed-off-by: ashnamehrotra --- pkg/pkgmgr/rpm.go | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 19a91442..35b9edab 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -196,12 +196,12 @@ func (rm *rpmManager) InstallUpdates(ctx context.Context, manifest *unversioned. var updatedImageState *llb.State var resultManifestBytes []byte if rm.isDistroless { - updatedImageState, resultManifestBytes, err = rm.unpackAndMergeUpdates(ctx, updates, toolImageName) + updatedImageState, resultManifestBytes, err = rm.unpackAndMergeUpdates(ctx, updates, toolImageName, ignoreErrors) if err != nil { return nil, nil, err } } else { - updatedImageState, resultManifestBytes, err = rm.installUpdates(ctx, updates) + updatedImageState, resultManifestBytes, err = rm.installUpdates(ctx, updates, ignoreErrors) if err != nil { return nil, nil, err } @@ -368,7 +368,7 @@ func parseManifestFile(file string) (map[string]string, error) { // // TODO: Support RPM-based images with valid rpm status but missing tools. (e.g. calico images > v3.21.0) // i.e. extra RunOption to mount a copy of rpm tools installed into the image and invoking that. -func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages) (*llb.State, []byte, error) { +func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.UpdatePackages, ignoreErrors bool) (*llb.State, []byte, error) { pkgs := "" // If specific updates, provided, parse into pkg names, else will update all @@ -385,13 +385,13 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up var installCmd string switch { case rm.rpmTools["dnf"] != "": - const dnfInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all'` + const dnfInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>>error_log.txt'` installCmd = fmt.Sprintf(dnfInstallTemplate, rm.rpmTools["dnf"], pkgs) case rm.rpmTools["yum"] != "": - const yumInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all'` + const yumInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>>error_log.txt'` installCmd = fmt.Sprintf(yumInstallTemplate, rm.rpmTools["yum"], pkgs) case rm.rpmTools["microdnf"] != "": - const microdnfInstallTemplate = `sh -c '%[1]s update %[2]s && %[1]s clean all'` + const microdnfInstallTemplate = `sh -c '%[1]s update %[2]s && %[1]s clean all 2>>error_log.txt'` installCmd = fmt.Sprintf(microdnfInstallTemplate, rm.rpmTools["microdnf"], pkgs) default: err := errors.New("unexpected: no package manager tools were found for patching") @@ -399,6 +399,11 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up } installed := rm.config.ImageState.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() + // Validate no errors were encountered if updating all + if updates != nil && !ignoreErrors { + installed = installed.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + } + // Write results.manifest to host for post-patch validation var resultBytes []byte var err error @@ -419,7 +424,7 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up return &patchMerge, resultBytes, nil } -func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string) (*llb.State, []byte, error) { +func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversioned.UpdatePackages, toolImage string, ignoreErrors bool) (*llb.State, []byte, error) { // Spin up a build tooling container to fetch and unpack packages to create patch layer. // Pull family:version -> need to create version to base image map toolingBase := llb.Image(toolImage, @@ -474,11 +479,21 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi downloadCmd = fmt.Sprintf(rpmDownloadTemplate, strings.Join(pkgStrings, " ")) } else { // only updated the outdated pacakges from packages.txt - downloadCmd = `xargs -a packages.txt -n 1 yumdownloader --downloadonly --downloaddir=. --best -y` + downloadCmd = ` + packages=$(>error_log.txt + done + ` } downloaded := busyboxCopied.Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() + // Validate no errors were encountered if updating all + if updates != nil && !ignoreErrors { + downloaded = downloaded.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + } + // Scripted enumeration and rpm install of all downloaded packages under the download folder as root const extractTemplate = `sh -c 'for f in %[1]s/*.rpm ; do rpm2cpio "$f" | cpio -idmv -D %[1]s ; done'` extractCmd := fmt.Sprintf(extractTemplate, downloadPath) From 98f89fdb8f85f68cbbddb577b5d62d72248e3043 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 07:40:42 -0700 Subject: [PATCH 38/49] apk validation update all regular Signed-off-by: ashnamehrotra --- pkg/pkgmgr/apk.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index 9ef4b5a1..26dd4b0f 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -120,7 +120,7 @@ func validateAPKPackageVersions(updates unversioned.UpdatePackages, cmp VersionC func (am *apkManager) InstallUpdates(ctx context.Context, manifest *unversioned.UpdateManifest, ignoreErrors bool) (*llb.State, []string, error) { // If manifest is nil, update all packages if manifest == nil { - updatedImageState, _, err := am.upgradePackages(ctx, nil) + updatedImageState, _, err := am.upgradePackages(ctx, nil, ignoreErrors) if err != nil { return updatedImageState, nil, err } @@ -140,7 +140,7 @@ func (am *apkManager) InstallUpdates(ctx context.Context, manifest *unversioned. } log.Debugf("latest unique APKs: %v", updates) - updatedImageState, resultsBytes, err := am.upgradePackages(ctx, updates) + updatedImageState, resultsBytes, err := am.upgradePackages(ctx, updates, ignoreErrors) if err != nil { return nil, nil, err } @@ -161,7 +161,7 @@ func (am *apkManager) InstallUpdates(ctx context.Context, manifest *unversioned. // TODO: support "distroless" Alpine images (e.g. APKO images) // Still assumes that APK exists in the target image and is pathed, which can be addressed by // mounting a copy of apk-tools-static into the image and invoking apk-static directly. -func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.UpdatePackages) (*llb.State, []byte, error) { +func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.UpdatePackages, ignoreErrors bool) (*llb.State, []byte, error) { // TODO: Add support for custom APK config apkUpdated := am.config.ImageState.Run(llb.Shlex("apk update"), llb.WithProxy(utils.GetProxy())).Root() @@ -200,8 +200,12 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U } } else { // if updates is not specified, update all packages - installCmd := `apk upgrade --no-cache` + installCmd := `apk upgrade --no-cache 2>>error_log.txt` apkInstalled = apkUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() + // Validate no errors were encountered if updating all + if !ignoreErrors { + apkInstalled = apkInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + } } // Diff the installed updates and merge that into the target image From 2b79b9a45e92579e7a69218fdbefb8de94663ac6 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 07:57:11 -0700 Subject: [PATCH 39/49] change bash to sh dpkg Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 4add9d36..e99a4611 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -335,7 +335,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U // Validate no errors were encountered if updating all if updates != nil && !ignoreErrors { - aptInstalled = aptInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + aptInstalled = aptInstalled.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() } // Write results.manifest to host for post-patch validation @@ -431,7 +431,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers // Validate no errors were encountered if updating all if updates != nil && !ignoreErrors { - downloaded = downloaded.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + downloaded = downloaded.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() } diffState := llb.Diff(updated, downloaded) From 3666ec3d1ee0ee621af4a6c36d65d8a497df510e Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 08:36:30 -0700 Subject: [PATCH 40/49] check for error to filter out warning Signed-off-by: ashnamehrotra --- pkg/pkgmgr/apk.go | 2 +- pkg/pkgmgr/dpkg.go | 7 +++++-- pkg/pkgmgr/rpm.go | 11 +++++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index 26dd4b0f..a2bbb0cb 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -200,7 +200,7 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U } } else { // if updates is not specified, update all packages - installCmd := `apk upgrade --no-cache 2>>error_log.txt` + installCmd := `apk upgrade --no-cache 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` apkInstalled = apkUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all if !ignoreErrors { diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index e99a4611..5d746c22 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -328,7 +328,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U installCmd = fmt.Sprintf(aptInstallTemplate, strings.Join(pkgStrings, " ")) } else { // if updates is not specified, update all packages - installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove 2>>error_log.txt"` + installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi"` } aptInstalled := aptUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() @@ -422,7 +422,10 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers downloadCmd = ` packages=$(>error_log.txt + output=$(apt download --no-install-recommends "$package" 2>&1) + if [ $? -ne 0 ]; then + echo "$output" >>error_log.txt + fi done ` } diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 35b9edab..35e055e9 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -385,13 +385,13 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up var installCmd string switch { case rm.rpmTools["dnf"] != "": - const dnfInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>>error_log.txt'` + const dnfInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(dnfInstallTemplate, rm.rpmTools["dnf"], pkgs) case rm.rpmTools["yum"] != "": - const yumInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>>error_log.txt'` + const yumInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(yumInstallTemplate, rm.rpmTools["yum"], pkgs) case rm.rpmTools["microdnf"] != "": - const microdnfInstallTemplate = `sh -c '%[1]s update %[2]s && %[1]s clean all 2>>error_log.txt'` + const microdnfInstallTemplate = `sh -c '%[1]s update %[2]s && %[1]s clean all 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(microdnfInstallTemplate, rm.rpmTools["microdnf"], pkgs) default: err := errors.New("unexpected: no package manager tools were found for patching") @@ -482,7 +482,10 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi downloadCmd = ` packages=$(>error_log.txt + output=$(yumdownloader --downloadonly --downloaddir=. --best -y "$package" 2>&1) + if [ $? -ne 0 ]; then + echo "$output" >>error_log.txt + fi done ` } From b86087548c13bd06d6392a3c295eb1fb68ce8f9b Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 09:05:40 -0700 Subject: [PATCH 41/49] revert apk changes Signed-off-by: ashnamehrotra --- pkg/pkgmgr/apk.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index a2bbb0cb..d595c6ff 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -200,12 +200,15 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U } } else { // if updates is not specified, update all packages - installCmd := `apk upgrade --no-cache 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` + //installCmd := `apk upgrade --no-cache 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` + installCmd := `apk upgrade --no-cache` apkInstalled = apkUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() - // Validate no errors were encountered if updating all - if !ignoreErrors { - apkInstalled = apkInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() - } + /* + // Validate no errors were encountered if updating all + if !ignoreErrors { + apkInstalled = apkInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + } + */ } // Diff the installed updates and merge that into the target image From fb286123db7f1cea9863c90360d0ef370870368a Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 09:30:37 -0700 Subject: [PATCH 42/49] use llb args instead Signed-off-by: ashnamehrotra --- pkg/pkgmgr/rpm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 35e055e9..42c92f23 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -490,7 +490,7 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi ` } - downloaded := busyboxCopied.Run(llb.Shlex(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() + downloaded := busyboxCopied.Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all if updates != nil && !ignoreErrors { From 7c4f2dfae42711d2532653d57a03b52a9b280398 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 10:11:00 -0700 Subject: [PATCH 43/49] apk changes Signed-off-by: ashnamehrotra --- pkg/pkgmgr/apk.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index d595c6ff..38ad7c35 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -200,15 +200,13 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U } } else { // if updates is not specified, update all packages - //installCmd := `apk upgrade --no-cache 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` - installCmd := `apk upgrade --no-cache` + installCmd := `apk upgrade --no-cache 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` apkInstalled = apkUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() - /* - // Validate no errors were encountered if updating all - if !ignoreErrors { - apkInstalled = apkInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() - } - */ + + // Validate no errors were encountered if updating all + if !ignoreErrors { + apkInstalled = apkInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + } } // Diff the installed updates and merge that into the target image From 800db9746411dcf07f8421b61ea2e3142db20662 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 10:51:30 -0700 Subject: [PATCH 44/49] output fixes Signed-off-by: ashnamehrotra --- pkg/pkgmgr/apk.go | 4 ++-- pkg/pkgmgr/dpkg.go | 2 +- pkg/pkgmgr/rpm.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index f0af2364..0110d8fa 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -200,8 +200,8 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U } } else { // if updates is not specified, update all packages - installCmd := `apk upgrade --no-cache 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` - apkInstalled = apkUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() + installCmd := `output=$(apk upgrade --no-cache 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` + apkInstalled = apkUpdated.Run(llb.Args([]string{"sh", "-c", installCmd}), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all if !ignoreErrors { diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 5d746c22..555f3aa4 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -328,7 +328,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U installCmd = fmt.Sprintf(aptInstallTemplate, strings.Join(pkgStrings, " ")) } else { // if updates is not specified, update all packages - installCmd = `sh -c "apt upgrade -y && apt clean -y && apt autoremove 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi"` + installCmd = `sh -c "output=$(apt upgrade -y && apt clean -y && apt autoremove 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi"` } aptInstalled := aptUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 42c92f23..751ef66e 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -385,13 +385,13 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up var installCmd string switch { case rm.rpmTools["dnf"] != "": - const dnfInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` + const dnfInstallTemplate = `sh -c 'output=$(%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(dnfInstallTemplate, rm.rpmTools["dnf"], pkgs) case rm.rpmTools["yum"] != "": - const yumInstallTemplate = `sh -c '%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` + const yumInstallTemplate = `sh -c 'output=$(%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(yumInstallTemplate, rm.rpmTools["yum"], pkgs) case rm.rpmTools["microdnf"] != "": - const microdnfInstallTemplate = `sh -c '%[1]s update %[2]s && %[1]s clean all 2>&1; if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` + const microdnfInstallTemplate = `output=$(sh -c '%[1]s update %[2]s && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(microdnfInstallTemplate, rm.rpmTools["microdnf"], pkgs) default: err := errors.New("unexpected: no package manager tools were found for patching") From d5fe32b576e33bf70b3ba658625d0879cfb3073d Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 11:01:57 -0700 Subject: [PATCH 45/49] fix microdnf Signed-off-by: ashnamehrotra --- pkg/pkgmgr/rpm.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 751ef66e..6b0dc804 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -391,7 +391,7 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up const yumInstallTemplate = `sh -c 'output=$(%[1]s upgrade %[2]s -y && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(yumInstallTemplate, rm.rpmTools["yum"], pkgs) case rm.rpmTools["microdnf"] != "": - const microdnfInstallTemplate = `output=$(sh -c '%[1]s update %[2]s && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` + const microdnfInstallTemplate = `sh -c 'output=$(%[1]s update %[2]s && %[1]s clean all 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi'` installCmd = fmt.Sprintf(microdnfInstallTemplate, rm.rpmTools["microdnf"], pkgs) default: err := errors.New("unexpected: no package manager tools were found for patching") From f8d65f43fc1aafb0fe1520711590e98fdbae4163 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Fri, 10 May 2024 11:11:43 -0700 Subject: [PATCH 46/49] try sh instead of bash Signed-off-by: ashnamehrotra --- pkg/pkgmgr/apk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index 0110d8fa..a62def44 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -205,7 +205,7 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U // Validate no errors were encountered if updating all if !ignoreErrors { - apkInstalled = apkInstalled.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + apkInstalled = apkInstalled.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() } } From 92718bba9ba1d65cc23f42aa7cf8137bcdb92b35 Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 21 May 2024 13:15:20 -0700 Subject: [PATCH 47/49] fix if statement Signed-off-by: ashnamehrotra --- pkg/pkgmgr/dpkg.go | 4 ++-- pkg/pkgmgr/rpm.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index 555f3aa4..f949c4ae 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -334,7 +334,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U aptInstalled := aptUpdated.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all - if updates != nil && !ignoreErrors { + if updates == nil && !ignoreErrors { aptInstalled = aptInstalled.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() } @@ -433,7 +433,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers downloaded := updated.Dir(dpkgDownloadPath).Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all - if updates != nil && !ignoreErrors { + if updates == nil && !ignoreErrors { downloaded = downloaded.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() } diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 6b0dc804..c9c0d0c4 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -400,7 +400,7 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up installed := rm.config.ImageState.Run(llb.Shlex(installCmd), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all - if updates != nil && !ignoreErrors { + if updates == nil && !ignoreErrors { installed = installed.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() } @@ -493,7 +493,7 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi downloaded := busyboxCopied.Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all - if updates != nil && !ignoreErrors { + if updates == nil && !ignoreErrors { downloaded = downloaded.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() } From c57f9e098f821801c10e064cfdf429b82f1b738e Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 25 Jun 2024 09:54:57 -0700 Subject: [PATCH 48/49] create and use util function for sh c Signed-off-by: ashnamehrotra --- pkg/buildkit/buildkit.go | 4 ++++ pkg/pkgmgr/apk.go | 2 +- pkg/pkgmgr/dpkg.go | 4 ++-- pkg/pkgmgr/rpm.go | 6 +++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/pkg/buildkit/buildkit.go b/pkg/buildkit/buildkit.go index 0555d694..9cd80990 100644 --- a/pkg/buildkit/buildkit.go +++ b/pkg/buildkit/buildkit.go @@ -90,6 +90,10 @@ func ExtractFileFromState(ctx context.Context, c gwclient.Client, st *llb.State, }) } +func Sh(cmd string) llb.RunOption { + return llb.Args([]string{"/bin/sh", "-c", cmd}) +} + func ArrayFile(input []string) []byte { var b bytes.Buffer for _, s := range input { diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index a62def44..35c8b6dc 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -205,7 +205,7 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U // Validate no errors were encountered if updating all if !ignoreErrors { - apkInstalled = apkInstalled.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + apkInstalled = apkInstalled.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root() } } diff --git a/pkg/pkgmgr/dpkg.go b/pkg/pkgmgr/dpkg.go index f949c4ae..29ff1d92 100644 --- a/pkg/pkgmgr/dpkg.go +++ b/pkg/pkgmgr/dpkg.go @@ -335,7 +335,7 @@ func (dm *dpkgManager) installUpdates(ctx context.Context, updates unversioned.U // Validate no errors were encountered if updating all if updates == nil && !ignoreErrors { - aptInstalled = aptInstalled.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + aptInstalled = aptInstalled.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root() } // Write results.manifest to host for post-patch validation @@ -434,7 +434,7 @@ func (dm *dpkgManager) unpackAndMergeUpdates(ctx context.Context, updates unvers // Validate no errors were encountered if updating all if updates == nil && !ignoreErrors { - downloaded = downloaded.Run(llb.Args([]string{"sh", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + downloaded = downloaded.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root() } diffState := llb.Diff(updated, downloaded) diff --git a/pkg/pkgmgr/rpm.go b/pkg/pkgmgr/rpm.go index 24e9dcfc..32e1a196 100644 --- a/pkg/pkgmgr/rpm.go +++ b/pkg/pkgmgr/rpm.go @@ -416,7 +416,7 @@ func (rm *rpmManager) installUpdates(ctx context.Context, updates unversioned.Up // Validate no errors were encountered if updating all if updates == nil && !ignoreErrors { - installed = installed.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + installed = installed.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root() } // Write results.manifest to host for post-patch validation @@ -505,11 +505,11 @@ func (rm *rpmManager) unpackAndMergeUpdates(ctx context.Context, updates unversi ` } - downloaded := busyboxCopied.Run(llb.Args([]string{"bash", "-c", downloadCmd}), llb.WithProxy(utils.GetProxy())).Root() + downloaded := busyboxCopied.Run(buildkit.Sh(downloadCmd), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all if updates == nil && !ignoreErrors { - downloaded = downloaded.Run(llb.Args([]string{"bash", "-c", "if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi"})).Root() + downloaded = downloaded.Run(buildkit.Sh("if [ -s error_log.txt ]; then cat error_log.txt; exit 1; fi")).Root() } // Scripted enumeration and rpm install of all downloaded packages under the download folder as root From 22ef97d38eb1c830a091cc75c5c12681f8ddfa2b Mon Sep 17 00:00:00 2001 From: ashnamehrotra Date: Tue, 25 Jun 2024 09:57:14 -0700 Subject: [PATCH 49/49] use util func Signed-off-by: ashnamehrotra --- pkg/pkgmgr/apk.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/pkgmgr/apk.go b/pkg/pkgmgr/apk.go index 35c8b6dc..314bd2c6 100644 --- a/pkg/pkgmgr/apk.go +++ b/pkg/pkgmgr/apk.go @@ -201,7 +201,7 @@ func (am *apkManager) upgradePackages(ctx context.Context, updates unversioned.U } else { // if updates is not specified, update all packages installCmd := `output=$(apk upgrade --no-cache 2>&1); if [ $? -ne 0 ]; then echo "$output" >>error_log.txt; fi` - apkInstalled = apkUpdated.Run(llb.Args([]string{"sh", "-c", installCmd}), llb.WithProxy(utils.GetProxy())).Root() + apkInstalled = apkUpdated.Run(buildkit.Sh(installCmd), llb.WithProxy(utils.GetProxy())).Root() // Validate no errors were encountered if updating all if !ignoreErrors {