ci: cache common images during tests

Signed-off-by: Justin Chadwell <[email protected]>

ci/go.mod

@@ -13,6 +13,7 @@ require (
  github.com/99designs/gqlgen v0.17.44
  github.com/Khan/genqlient v0.7.0
  github.com/containerd/containerd v1.7.15-0.20240329193453-0dcf21c1528a
+ github.com/docker/cli v26.0.0-rc1+incompatible
  github.com/magefile/mage v1.15.0
  github.com/moby/buildkit v0.13.0-rc3.0.20240403135707-dc23e43dc15c
  github.com/opencontainers/image-spec v1.1.0
@@ -28,13 +29,29 @@ require (
 )
 
 require (
+ github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
  github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+ github.com/containerd/typeurl/v2 v2.1.1 // indirect
+ github.com/distribution/reference v0.5.0 // indirect
+ github.com/docker/docker-credential-helpers v0.8.0 // indirect
+ github.com/felixge/httpsnoop v1.0.4 // indirect
  github.com/go-logr/logr v1.4.1 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect
+ github.com/gogo/googleapis v1.4.1 // indirect
+ github.com/gogo/protobuf v1.3.2 // indirect
  github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
+ github.com/in-toto/in-toto-golang v0.5.0 // indirect
+ github.com/klauspost/compress v1.17.4 // indirect
+ github.com/moby/locker v1.0.1 // indirect
+ github.com/secure-systems-lab/go-securesystemslib v0.4.0 // indirect
+ github.com/shibumi/go-pathspec v1.3.0 // indirect
+ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.46.1 // indirect
+ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
  go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
  go.opentelemetry.io/otel/metric v1.24.0 // indirect
  go.opentelemetry.io/proto/otlp v1.1.0 // indirect
+ golang.org/x/crypto v0.22.0 // indirect
  golang.org/x/net v0.24.0 // indirect
  golang.org/x/text v0.14.0 // indirect
  google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect

ci/test.go

@@ -2,10 +2,21 @@ package main
 
 import (
  "context"
+ "fmt"
+ "net/http"
  "path/filepath"
+ "strings"
+ "time"
 
+ "github.com/containerd/containerd/content"
+ "github.com/containerd/containerd/remotes/docker"
+ "github.com/docker/cli/cli/config/configfile"
  "github.com/moby/buildkit/identity"
+ "github.com/moby/buildkit/util/contentutil"
+ "github.com/moby/buildkit/version"
+ ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 
+ "github.com/dagger/dagger/ci/internal/dagger"
  "github.com/dagger/dagger/ci/util"
  "github.com/dagger/dagger/engine/distconsts"
 )
@@ -97,11 +108,78 @@ func (t *Test) test(
  return err
 }
 
+func toCredentialsFunc(dt string) func(string) (string, string, error) {
+ cfg := configfile.New("config.json")
+ err := cfg.LoadFromReader(strings.NewReader(dt))
+ if err != nil {
+ panic(err)
+ }
+
+ return func(host string) (string, string, error) {
+ if host == "registry-1.docker.io" {
+ host = "https://index.docker.io/v1/"
+ }
+ ac, err := cfg.GetAuthConfig(host)
+ if err != nil {
+ return "", "", err
+ }
+ if ac.IdentityToken != "" {
+ return "", ac.IdentityToken, nil
+ }
+ return ac.Username, ac.Password, nil
+ }
+}
+
 func (t *Test) testCmd(ctx context.Context) (*Container, error) {
+ mirror := registryMirror()
+ mirror, err := mirror.Start(ctx)
+ if err != nil {
+ return nil, err
+ }
+ go mirror.Up(ctx, dagger.ServiceUpOpts{
+ Ports: []dagger.PortForward{{
+ Backend: 5000,
+ Frontend: 5000,
+ }},
+ })
+
+ var auth docker.Authorizer
+ if t.Dagger.HostDockerConfig != nil {
+ hostConfig, err := t.Dagger.HostDockerConfig.Plaintext(ctx)
+ if err != nil {
+ return nil, err
+ }
+ auth = docker.NewDockerAuthorizer(docker.WithAuthCreds(toCredentialsFunc(hostConfig)), docker.WithAuthClient(http.DefaultClient))
+ }
+
+ err = copyImagesLocal(auth, "localhost:5000", map[string]string{
+ "library/alpine:latest": "docker.io/library/alpine:3.18.2",
+ "library/alpine:3.18": "docker.io/library/alpine:3.18.2",
+ "library/alpine:3.18.2": "docker.io/library/alpine:3.18.2",
+
+ "library/registry:2": "docker.io/library/registry:2",
+
+ "library/golang:latest": "docker.io/library/golang:1.22.2-alpine",
+ "library/golang:1.22": "docker.io/library/golang:1.22.2-alpine",
+ "library/golang:1.22.2": "docker.io/library/golang:1.22.2-alpine",
+ "library/golang:alpine": "docker.io/library/golang:1.22.2-alpine",
+ "library/golang:1.22-alpine": "docker.io/library/golang:1.22.2-alpine",
+ "library/golang:1.22.2-alpine": "docker.io/library/golang:1.22.2-alpine",
+
+ "library/python:latest": "docker.io/library/python:3.11-slim",
+ "library/python:3": "docker.io/library/python:3.11-slim",
+ "library/python:3.11": "docker.io/library/python:3.11-slim",
+ "library/python:3.11-slim": "docker.io/library/python:3.11-slim",
+ })
+ if err != nil {
+ return nil, err
+ }
+
  engine := t.Dagger.Engine().
  WithConfig(`registry."registry:5000"`, `http = true`).
  WithConfig(`registry."privateregistry:5000"`, `http = true`).
- WithConfig(`registry."docker.io"`, `mirrors = ["mirror.gcr.io"]`).
+ WithConfig(`registry."docker.io"`, `mirrors = ["mirror.dagger.test:5000"]`).
+ WithConfig(`registry."mirror.dagger.test:5000"`, `http = true`).
  WithConfig(`grpc`, `address=["unix:///var/run/buildkit/buildkitd.sock", "tcp://0.0.0.0:1234"]`).
  WithArg(`network-name`, `dagger-dev`).
  WithArg(`network-cidr`, `10.88.0.0/16`)
@@ -132,6 +210,7 @@ func (t *Test) testCmd(ctx context.Context) (*Container, error) {
  devEngineSvc := devEngine.
  WithServiceBinding("registry", registrySvc).
  WithServiceBinding("privateregistry", privateRegistry()).
+ WithServiceBinding("mirror.dagger.test", mirror).
  WithExposedPort(1234, ContainerWithExposedPortOpts{Protocol: Tcp}).
  WithMountedCache(distconsts.EngineDefaultStateDir, dag.CacheVolume("dagger-dev-engine-test-state"+identity.NewID())).
  WithExec(nil, ContainerWithExecOpts{
@@ -192,3 +271,78 @@ func privateRegistry() *Service {
  WithExec(nil).
  AsService()
 }
+
+func registryMirror() *Service {
+ return dag.Container().
+ From("registry:2").
+ WithEnvVariable("LOG_LEVEL", "warn").
+ WithEnvVariable("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/data/registry").
+ WithMountedCache("/data/registry", dag.CacheVolume("dagger-registry-mirror-cache")).
+ WithExposedPort(5000, ContainerWithExposedPortOpts{Protocol: Tcp}).
+ WithExec(nil).
+ AsService()
+}
+
+var localImageCache map[string]map[string]struct{}
+
+func copyImagesLocal(auth docker.Authorizer, host string, images map[string]string) error {
+ for to, from := range images {
+ if localImageCache == nil {
+ localImageCache = map[string]map[string]struct{}{}
+ }
+ if _, ok := localImageCache[host]; !ok {
+ localImageCache[host] = map[string]struct{}{}
+ }
+ if _, ok := localImageCache[host][to]; ok {
+ continue
+ }
+ localImageCache[host][to] = struct{}{}
+
+ start := time.Now()
+
+ var desc ocispecs.Descriptor
+ var provider content.Provider
+ var err error
+ desc, provider, err = ProviderFromRef(from, auth)
+ if err != nil {
+ return err
+ }
+
+ // already exists check
+ _, _, err = docker.NewResolver(docker.ResolverOptions{}).Resolve(context.TODO(), host+"/"+to)
+ if err == nil {
+ fmt.Printf("copied %s to local mirror %s (skipped)\n", from, host+"/"+to)
+ continue
+ }
+
+ ingester, err := contentutil.IngesterFromRef(host + "/" + to)
+ if err != nil {
+ return err
+ }
+ if err := contentutil.CopyChain(context.TODO(), ingester, provider, desc); err != nil {
+ return err
+ }
+ fmt.Printf("copied %s to local mirror %s in %s\n", from, host+"/"+to, time.Since(start))
+ }
+ return nil
+}
+
+func ProviderFromRef(ref string, auth docker.Authorizer) (ocispecs.Descriptor, content.Provider, error) {
+ headers := http.Header{}
+ headers.Set("User-Agent", version.UserAgent())
+ remote := docker.NewResolver(docker.ResolverOptions{
+ Headers: headers,
+ Authorizer: auth,
+ })
+
+ name, desc, err := remote.Resolve(context.TODO(), ref)
+ if err != nil {
+ return ocispecs.Descriptor{}, nil, err
+ }
+
+ fetcher, err := remote.Fetcher(context.TODO(), name)
+ if err != nil {
+ return ocispecs.Descriptor{}, nil, err
+ }
+ return desc, contentutil.FromFetcher(fetcher), nil
+}

core/integration/container_test.go

@@ -98,7 +98,7 @@ func main() {
  t.Run("default Dockerfile location", func(t *testing.T) {
  src := contextDir.
  WithNewFile("Dockerfile",
- `FROM golang:1.18.2-alpine
+ `FROM golang
 WORKDIR /src
 COPY main.go .
 RUN go mod init hello
@@ -115,7 +115,7 @@ CMD goenv
  t.Run("custom Dockerfile location", func(t *testing.T) {
  src := contextDir.
  WithNewFile("subdir/Dockerfile.whee",
- `FROM golang:1.18.2-alpine
+ `FROM golang
 WORKDIR /src
 COPY main.go .
 RUN go mod init hello
@@ -134,7 +134,7 @@ CMD goenv
  t.Run("subdirectory with default Dockerfile location", func(t *testing.T) {
  src := contextDir.
  WithNewFile("Dockerfile",
- `FROM golang:1.18.2-alpine
+ `FROM golang
 WORKDIR /src
 COPY main.go .
 RUN go mod init hello
@@ -153,7 +153,7 @@ CMD goenv
  t.Run("subdirectory with custom Dockerfile location", func(t *testing.T) {
  src := contextDir.
  WithNewFile("subdir/Dockerfile.whee",
- `FROM golang:1.18.2-alpine
+ `FROM golang
 WORKDIR /src
 COPY main.go .
 RUN go mod init hello
@@ -174,7 +174,7 @@ CMD goenv
  t.Run("with build args", func(t *testing.T) {
  src := contextDir.
  WithNewFile("Dockerfile",
- `FROM golang:1.18.2-alpine
+ `FROM golang
 ARG FOOARG=bar
 WORKDIR /src
 COPY main.go .
@@ -196,7 +196,7 @@ CMD goenv
  t.Run("with target", func(t *testing.T) {
  src := contextDir.
  WithNewFile("Dockerfile",
- `FROM golang:1.18.2-alpine AS base
+ `FROM golang AS base
 CMD echo "base"
 
 FROM base AS stage1
@@ -221,7 +221,7 @@ CMD echo "stage2"
 
  src := contextDir.
  WithNewFile("Dockerfile",
- `FROM golang:1.18.2-alpine
+ `FROM golang
 WORKDIR /src
 RUN --mount=type=secret,id=my-secret,required=true test "$(cat /run/secrets/my-secret)" = "barbar"
 RUN --mount=type=secret,id=my-secret,required=true cp /run/secrets/my-secret /secret
@@ -929,7 +929,7 @@ func TestContainerVariables(t *testing.T) {
  err := testutil.Query(
  `{
  container {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  envVariables {
  name
  value
@@ -963,7 +963,7 @@ func TestContainerVariable(t *testing.T) {
  err := testutil.Query(
  `{
  container {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  envVariable(name: "GOLANG_VERSION")
  }
  }
@@ -975,7 +975,7 @@ func TestContainerVariable(t *testing.T) {
  err = testutil.Query(
  `{
  container {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  envVariable(name: "UNKNOWN")
  }
  }
@@ -1003,7 +1003,7 @@ func TestContainerWithoutVariable(t *testing.T) {
  err := testutil.Query(
  `{
  container {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  withoutEnvVariable(name: "GOLANG_VERSION") {
  envVariables {
  name
@@ -1043,7 +1043,7 @@ func TestContainerEnvVariablesReplace(t *testing.T) {
  err := testutil.Query(
  `{
  container {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  withEnvVariable(name: "GOPATH", value: "/gone") {
  envVariables {
  name
@@ -1217,7 +1217,7 @@ func TestContainerWorkdir(t *testing.T) {
  err := testutil.Query(
  `{
  container {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  workdir
  withExec(args: ["pwd"]) {
  stdout
@@ -1249,7 +1249,7 @@ func TestContainerWithWorkdir(t *testing.T) {
  err := testutil.Query(
  `{
  container {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  withWorkdir(path: "/usr") {
  workdir
  withExec(args: ["pwd"]) {
@@ -2694,7 +2694,7 @@ func TestContainerMultiFrom(t *testing.T) {
  from(address: "node:18.10.0-alpine") {
  withMountedDirectory(path: "/mnt", source: $id) {
  withExec(args: ["sh", "-c", "node --version >> /mnt/versions"]) {
- from(address: "golang:1.18.2-alpine") {
+ from(address: "golang") {
  withExec(args: ["sh", "-c", "go version >> /mnt/versions"]) {
  withExec(args: ["cat", "/mnt/versions"]) {
  stdout