Skip to content

Commit 14508f7

Browse files
stmcginniscurtbushko
stmcginnis
and
curtbushko
committed
Add ability to include images in custom node build
This adds a new `kind build add-image` command that provides the ability to "preload" container images in a custom node image. Signed-off-by: Sean McGinnis <[email protected]> Co-authored-by: Curt Bushko <[email protected]>
1 parent 06b98aa commit 14508f7

File tree

18 files changed

+602
-60
lines changed

18 files changed

+602
-60
lines changed

‎pkg/build/addimage/build.go

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
// Package addimage implements functionality to build a node image with container images included.
18+
package addimage
19+
20+
import (
21+
"runtime"
22+
23+
"sigs.k8s.io/kind/pkg/apis/config/defaults"
24+
"sigs.k8s.io/kind/pkg/build/internal/build"
25+
"sigs.k8s.io/kind/pkg/log"
26+
)
27+
28+
// Build creates a new node image by combining an existing node image with a collection
29+
// of additional images.
30+
func Build(options ...Option) error {
31+
// default options
32+
ctx := &buildContext{
33+
image: DefaultImage,
34+
baseImage: defaults.Image,
35+
logger: log.NoopLogger{},
36+
arch: runtime.GOARCH,
37+
}
38+
39+
// apply user options
40+
for _, option := range options {
41+
if err := option.apply(ctx); err != nil {
42+
return err
43+
}
44+
}
45+
46+
// verify that we're using a supported arch
47+
if !build.SupportedArch(ctx.arch) {
48+
ctx.logger.Warnf("unsupported architecture %q", ctx.arch)
49+
}
50+
return ctx.Build()
51+
}

‎pkg/build/addimage/buildcontext.go

+184
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*
2+
Copyrigh. 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package addimage
18+
19+
import (
20+
"fmt"
21+
"math/rand"
22+
"os"
23+
"path/filepath"
24+
"strings"
25+
"time"
26+
27+
"sigs.k8s.io/kind/pkg/build/internal/build"
28+
"sigs.k8s.io/kind/pkg/build/internal/container/docker"
29+
"sigs.k8s.io/kind/pkg/errors"
30+
"sigs.k8s.io/kind/pkg/exec"
31+
"sigs.k8s.io/kind/pkg/fs"
32+
"sigs.k8s.io/kind/pkg/log"
33+
)
34+
35+
const (
36+
// httpProxy is the HTTP_PROXY environment variable key
37+
httpProxy = "HTTP_PROXY"
38+
// httpsProxy is the HTTPS_PROXY environment variable key
39+
httpsProxy = "HTTPS_PROXY"
40+
// noProxy is the NO_PROXY environment variable key
41+
noProxy = "NO_PROXY"
42+
)
43+
44+
// buildContext the settings to use for rebuilding the node image.
45+
type buildContext struct {
46+
// option fields
47+
image string
48+
baseImage string
49+
additionalImages []string
50+
logger log.Logger
51+
arch string
52+
alwaysPull bool
53+
}
54+
55+
// Build rebuilds the cluster node image using the buildContext to determine
56+
// which base image and additional images to package into a new node image.
57+
func (c *buildContext) Build() (err error) {
58+
c.logger.V(0).Infof("Adding %v images to base image", c.additionalImages)
59+
60+
// Retrieve any necessary images
61+
for _, imageName := range c.additionalImages {
62+
if !c.alwaysPull {
63+
// Check if the image already exists locally
64+
if _, err := docker.ImageID(imageName, c.arch); err == nil {
65+
continue
66+
}
67+
}
68+
69+
// Pull the image for the requested architecture, which may be different than this host
70+
err = docker.Pull(c.logger, imageName, build.DockerBuildOsAndArch(c.arch), 3)
71+
if err != nil {
72+
c.logger.Errorf("add image build failed, unable to pull image %q: %v", imageName, err)
73+
return err
74+
}
75+
}
76+
77+
c.logger.V(0).Infof("Creating build container based on %q", c.baseImage)
78+
79+
containerID, err := c.createBuildContainer()
80+
if containerID != "" {
81+
defer func() {
82+
_ = exec.Command("docker", "rm", "-f", "-v", containerID).Run()
83+
}()
84+
}
85+
if err != nil {
86+
c.logger.Errorf("add image build failed, unable to create build container: %v", err)
87+
return err
88+
}
89+
c.logger.V(1).Infof("Building in %s", containerID)
90+
91+
// Tar up the images to make the load easier (and follow the current load pattern)
92+
// Setup the tar path where the images will be saved
93+
dir, err := fs.TempDir("", "images-tar")
94+
if err != nil {
95+
return errors.Wrap(err, "failed to create tempdir")
96+
}
97+
defer os.RemoveAll(dir)
98+
99+
// Save the images into a tar file
100+
imagesTarFile := filepath.Join(dir, "images.tar")
101+
c.logger.V(1).Infof("Saving images into tar file at %q", imagesTarFile)
102+
err = docker.SaveImages(c.additionalImages, imagesTarFile)
103+
if err != nil {
104+
return err
105+
}
106+
107+
// Import the images from the tarfile into our build container
108+
cmder := docker.ContainerCmder(containerID)
109+
importer := build.NewContainerdImporter(cmder)
110+
111+
f, err := os.Open(imagesTarFile)
112+
if err != nil {
113+
return err
114+
}
115+
defer f.Close()
116+
117+
c.logger.V(0).Infof("Importing images into build container %s", containerID)
118+
if err := importer.LoadCommand().SetStdout(os.Stdout).SetStderr(os.Stderr).SetStdin(f).Run(); err != nil {
119+
return err
120+
}
121+
122+
// Save the image changes to a new image
123+
c.logger.V(0).Info("Saving new image " + c.image)
124+
saveCmd := exec.Command(
125+
"docker", "commit",
126+
// we need to put this back after changing it when running the image
127+
"--change", `ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ]`,
128+
// remove proxy settings since they're for the building process
129+
// and should not be carried with the built image
130+
"--change", `ENV HTTP_PROXY="" HTTPS_PROXY="" NO_PROXY=""`,
131+
containerID, c.image,
132+
)
133+
exec.InheritOutput(saveCmd)
134+
if err = saveCmd.Run(); err != nil {
135+
c.logger.Errorf("add image build failed, unable to save destination image: %v", err)
136+
return err
137+
}
138+
139+
c.logger.V(0).Info("Add image build completed.")
140+
return nil
141+
}
142+
143+
func (c *buildContext) createBuildContainer() (id string, err error) {
144+
// Attempt to explicitly pull the image if it doesn't exist locally
145+
// we don't care if this errors, we'll still try to run which also pulls
146+
_ = docker.Pull(c.logger, c.baseImage, build.DockerBuildOsAndArch(c.arch), 4)
147+
148+
// This should be good enough: a specific prefix, the current unix time,
149+
// and a little random bits in case we have multiple builds simultaneously
150+
random := rand.New(rand.NewSource(time.Now().UnixNano())).Int31()
151+
id = fmt.Sprintf("kind-build-%d-%d", time.Now().UTC().Unix(), random)
152+
runArgs := []string{
153+
// make the client exit while the container continues to run
154+
"-d",
155+
// run containerd so that the cri command works
156+
"--entrypoint=/usr/local/bin/containerd",
157+
"--name=" + id,
158+
"--platform=" + build.DockerBuildOsAndArch(c.arch),
159+
"--security-opt", "seccomp=unconfined", // ignore seccomp
160+
}
161+
162+
// Pass proxy settings from environment variables to the building container
163+
// to make them work during the building process
164+
for _, name := range []string{httpProxy, httpsProxy, noProxy} {
165+
val := os.Getenv(name)
166+
if val == "" {
167+
val = os.Getenv(strings.ToLower(name))
168+
}
169+
if val != "" {
170+
runArgs = append(runArgs, "--env", name+"="+val)
171+
}
172+
}
173+
174+
// Run it
175+
err = docker.Run(
176+
c.baseImage,
177+
runArgs,
178+
[]string{
179+
"",
180+
},
181+
)
182+
183+
return id, errors.Wrap(err, "failed to create build container")
184+
}

‎pkg/build/nodeimage/internal/container/docker/save.go renamed to ‎pkg/build/addimage/defaults.go

+4-10
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
Copyright 2018 The Kubernetes Authors.
2+
Copyright 2024 The Kubernetes Authors.
33
44
Licensed under the Apache License, Version 2.0 (the "License");
55
you may not use this file except in compliance with the License.
@@ -14,13 +14,7 @@ See the License for the specific language governing permissions and
1414
limitations under the License.
1515
*/
1616

17-
package docker
17+
package addimage
1818

19-
import (
20-
"sigs.k8s.io/kind/pkg/exec"
21-
)
22-
23-
// Save saves image to dest, as in `docker save`
24-
func Save(image, dest string) error {
25-
return exec.Command("docker", "save", "-o", dest, image).Run()
26-
}
19+
// DefaultImage is the default name:tag for the built image
20+
const DefaultImage = "kindest/custom-node:latest"

‎pkg/build/addimage/options.go

+82
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package addimage
18+
19+
import (
20+
"sigs.k8s.io/kind/pkg/log"
21+
)
22+
23+
// Option is a configuration option supplied to Build
24+
type Option interface {
25+
apply(*buildContext) error
26+
}
27+
28+
type optionAdapter func(*buildContext) error
29+
30+
func (c optionAdapter) apply(o *buildContext) error {
31+
return c(o)
32+
}
33+
34+
// WithImage configures a build to tag the built image with `image`
35+
func WithImage(image string) Option {
36+
return optionAdapter(func(b *buildContext) error {
37+
b.image = image
38+
return nil
39+
})
40+
}
41+
42+
// WithBaseImage configures a build to use `image` as the base image
43+
func WithBaseImage(image string) Option {
44+
return optionAdapter(func(b *buildContext) error {
45+
b.baseImage = image
46+
return nil
47+
})
48+
}
49+
50+
// WithAdditionalImages configures a build to add images to the node image
51+
func WithAdditonalImages(images []string) Option {
52+
return optionAdapter(func(b *buildContext) error {
53+
b.additionalImages = images
54+
return nil
55+
})
56+
}
57+
58+
// WithLogger sets the logger
59+
func WithLogger(logger log.Logger) Option {
60+
return optionAdapter(func(b *buildContext) error {
61+
b.logger = logger
62+
return nil
63+
})
64+
}
65+
66+
// WithArch sets the architecture to build for
67+
func WithArch(arch string) Option {
68+
return optionAdapter(func(b *buildContext) error {
69+
if arch != "" {
70+
b.arch = arch
71+
}
72+
return nil
73+
})
74+
}
75+
76+
// WithPullPolicy sets whether to always pull the images
77+
func WithPullPolicy(pull bool) Option {
78+
return optionAdapter(func(b *buildContext) error {
79+
b.alwaysPull = pull
80+
return nil
81+
})
82+
}

‎pkg/build/internal/build/helpers.go

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
/*
2+
Copyright 2020 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package build
18+
19+
// SupportedArch checks whether the requested arch is one that is officially supportedf
20+
// by the project.
21+
func SupportedArch(arch string) bool {
22+
switch arch {
23+
default:
24+
return false
25+
// currently we nominally support building node images for these
26+
case "amd64":
27+
case "arm64":
28+
}
29+
return true
30+
}
31+
32+
func DockerBuildOsAndArch(arch string) string {
33+
return "linux/" + arch
34+
}

0 commit comments

Comments
 (0)