From 1f4c74ce3e3eac2f0ae0c18f983f0ffab114bc12 Mon Sep 17 00:00:00 2001 From: Parthiba-Hazra Date: Wed, 3 Apr 2024 18:36:24 +0530 Subject: [PATCH] Add support to build OCI images from checkpoint archives - With this enhancement, users can now build OCI images from checkpoint archives using the `checkpointctl build` command. This command accepts checkpoint archive and a image name as input and generates an OCI image suitable for use with container runtimes like CRI-O or Podman. Users can inspect the image to get information about runtime, container, pod, namespace, image name etc. Signed-off-by: Parthiba-Hazra --- .github/workflows/tests.yml | 4 +- Makefile | 21 +++- checkpointctl.go | 2 + cmd/build.go | 41 ++++++++ docs/checkpointctl-build.adoc | 25 +++++ internal/image_from_checkpoint_archive.go | 113 ++++++++++++++++++++++ internal/scripts/build_image.sh | 78 +++++++++++++++ lib/annotations.go | 35 +++++++ 8 files changed, 314 insertions(+), 5 deletions(-) create mode 100644 cmd/build.go create mode 100644 docs/checkpointctl-build.adoc create mode 100644 internal/image_from_checkpoint_archive.go create mode 100644 internal/scripts/build_image.sh create mode 100644 lib/annotations.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67012da3..c8155d4b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,8 +12,10 @@ jobs: - uses: actions/checkout@v3 - name: Install tools run: | - sudo dnf -y install ShellCheck bats golang criu asciidoctor iptables iproute kmod jq bash bash-completion zsh fish + sudo dnf -y install ShellCheck shfmt bats golang criu asciidoctor iptables iproute kmod jq bash bash-completion zsh fish sudo modprobe -va ip_tables ip6table_filter nf_conntrack nf_conntrack_netlink + - name: Run make shfmt-lint + run: make shfmt-lint - name: Run make shellcheck run: make shellcheck - name: Run make all diff --git a/Makefile b/Makefile index 24d93da3..6a63c449 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,14 @@ SHELL = /bin/bash PREFIX ?= $(DESTDIR)/usr/local BINDIR ?= $(PREFIX)/bin +SCRIPTDIR ?= $(DESTDIR)/usr/libexec GO ?= go GOPATH := $(shell $(GO) env GOPATH) GOBIN := $(shell $(GO) env GOBIN) GO_SRC = $(shell find . -name \*.go) GO_BUILD = $(GO) build NAME = checkpointctl +SCRIPTNAME = build_image.sh BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions ZSHINSTALLDIR=${PREFIX}/share/zsh/site-functions @@ -52,17 +54,22 @@ release: CGO_ENABLED=0 $(GO_BUILD) -o $(NAME) -ldflags "-X main.name=$(NAME) -X main.version=${VERSION}" .PHONY: install -install: $(NAME) install.completions +install: $(NAME) install.completions install-scripts @echo " INSTALL " $< @mkdir -p $(DESTDIR)$(BINDIR) @install -m0755 $< $(DESTDIR)$(BINDIR) @make -C docs install +.PHONY: install-scripts +install-scripts: + @echo " INSTALL SCRIPTS" + @install -m0755 internal/scripts/build_image.sh $(DESTDIR)$(SCRIPTDIR) + .PHONY: uninstall uninstall: uninstall.completions @make -C docs uninstall @echo " UNINSTALL" $(NAME) - @$(RM) $(addprefix $(DESTDIR)$(BINDIR)/,$(NAME)) + @$(RM) $(addprefix $(DESTDIR)$(BINDIR)/,$(NAME)) $(addprefix $(DESTDIR)$(SCRIPTDIR)/,$(SCRIPTNAME)) .PHONY: clean clean: @@ -77,9 +84,14 @@ golang-lint: .PHONY: shellcheck shellcheck: shellcheck test/*bats + shellcheck internal/scripts/build_image.sh + +.PHONY: shfmt-lint +shfmt-lint: + shfmt -w -d internal/scripts/build_image.sh .PHONY: lint -lint: golang-lint shellcheck +lint: golang-lint shellcheck shfmt-lint .PHONY: test test: $(NAME) @@ -151,9 +163,10 @@ help: @echo " * completions - generate auto-completion files" @echo " * clean - remove artifacts" @echo " * docs - build man pages" - @echo " * lint - verify the source code (shellcheck/golangci-lint)" + @echo " * lint - verify the source code (shellcheck/golangci-lint/shfmt-lint)" @echo " * golang-lint - run golangci-lint" @echo " * shellcheck - run shellcheck" + @echo " * shfmt-lint - run shfmt on selected shell scripts" @echo " * vendor - update go.mod, go.sum, and vendor directory" @echo " * test - run tests" @echo " * test-junit - run tests and create junit output" diff --git a/checkpointctl.go b/checkpointctl.go index 14848af3..3851ba20 100644 --- a/checkpointctl.go +++ b/checkpointctl.go @@ -31,6 +31,8 @@ func main() { rootCommand.AddCommand(cmd.List()) + rootCommand.AddCommand(cmd.BuildCmd()) + rootCommand.Version = version if err := rootCommand.Execute(); err != nil { diff --git a/cmd/build.go b/cmd/build.go new file mode 100644 index 00000000..e0f63146 --- /dev/null +++ b/cmd/build.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "context" + "fmt" + "log" + + "github.com/checkpoint-restore/checkpointctl/internal" + "github.com/spf13/cobra" +) + +func BuildCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "build [checkpoint-path] [image-name]", + Short: "Create an OCI image from a container checkpoint archive", + RunE: convertArchive, + } + + return cmd +} + +func convertArchive(cmd *cobra.Command, args []string) error { + if len(args) != 2 { + return fmt.Errorf("please provide both the checkpoint path and the image name") + } + + checkpointPath := args[0] + imageName := args[1] + + imageCreater := internal.NewImageCreator(imageName, checkpointPath) + + err := imageCreater.CreateImageFromCheckpoint(context.Background()) + if err != nil { + return err + } + + log.Printf("Image '%s' created successfully from checkpoint '%s'\n", imageName, checkpointPath) + return nil +} diff --git a/docs/checkpointctl-build.adoc b/docs/checkpointctl-build.adoc new file mode 100644 index 00000000..8fec3817 --- /dev/null +++ b/docs/checkpointctl-build.adoc @@ -0,0 +1,25 @@ += checkpointctl-build(1) +include::footer.adoc[] + +== Name + +*checkpointctl-build* - Create OCI image from a checkpoint tar file. + +== Synopsis + +*checkpointctl build* CHECKPOINT_PATH IMAGE_NAME + +== Options + +*-h*, *--help*:: + Show help for checkpointctl build + +== Description + +Creates an OCI image from a checkpoint tar file. This command requires `buildah` to be installed on the system. + +Please ensure that `buildah` is installed before running this command. + +== See also + +checkpointctl(1) diff --git a/internal/image_from_checkpoint_archive.go b/internal/image_from_checkpoint_archive.go new file mode 100644 index 00000000..f21de47e --- /dev/null +++ b/internal/image_from_checkpoint_archive.go @@ -0,0 +1,113 @@ +package internal + +import ( + "bytes" + "context" + "fmt" + "log" + "os" + "os/exec" + + metadata "github.com/checkpoint-restore/checkpointctl/lib" +) + +const ( + BUILD_SCRIPT = "/usr/libexec/build_image.sh" + PODMAN_ENGINE = "libpod" +) + +type ImageCreator struct { + imageName string + checkpointPath string +} + +func NewImageCreator(imageName, checkpointPath string) *ImageCreator { + return &ImageCreator{ + imageName: imageName, + checkpointPath: checkpointPath, + } +} + +func (ic *ImageCreator) CreateImageFromCheckpoint(ctx context.Context) error { + tempDir, err := os.MkdirTemp("", "checkpoint_tmp") + if err != nil { + return err + } + defer os.RemoveAll(tempDir) + + annotationsFilePath, err := ic.setCheckpointAnnotations(tempDir) + if err != nil { + return err + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd := exec.Command(BUILD_SCRIPT, "-a", annotationsFilePath, "-c", ic.checkpointPath, "-i", ic.imageName) + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err = cmd.Run() + if err != nil { + return fmt.Errorf("failed to execute script: %v, %v, %w", stdout.String(), stderr.String(), err) + } + + return nil +} + +func writeAnnotationsToFile(tempDir string, annotations map[string]string) (string, error) { + tempFile, err := os.CreateTemp(tempDir, "annotations_*.txt") + if err != nil { + return "", err + } + defer tempFile.Close() + + for key, value := range annotations { + _, err := fmt.Fprintf(tempFile, "%s=%s\n", key, value) + if err != nil { + return "", err + } + } + + return tempFile.Name(), nil +} + +func (ic *ImageCreator) setCheckpointAnnotations(tempDir string) (string, error) { + filesToExtract := []string{"spec.dump", "config.dump"} + if err := UntarFiles(ic.checkpointPath, tempDir, filesToExtract); err != nil { + log.Printf("Error extracting files from archive %s: %v\n", ic.checkpointPath, err) + return "", err + } + + var err error + info := &checkpointInfo{} + info.configDump, _, err = metadata.ReadContainerCheckpointConfigDump(tempDir) + if err != nil { + return "", err + } + + info.specDump, _, err = metadata.ReadContainerCheckpointSpecDump(tempDir) + if err != nil { + return "", err + } + + info.containerInfo, err = getContainerInfo(info.specDump, info.configDump) + if err != nil { + return "", err + } + + checkpointImageAnnotations := map[string]string{} + checkpointImageAnnotations[metadata.CheckpointAnnotationContainerManager] = info.containerInfo.Engine + checkpointImageAnnotations[metadata.CheckpointAnnotationName] = info.containerInfo.Name + checkpointImageAnnotations[metadata.CheckpointAnnotationPod] = info.containerInfo.Pod + checkpointImageAnnotations[metadata.CheckpointAnnotationNamespace] = info.containerInfo.Namespace + checkpointImageAnnotations[metadata.CheckpointAnnotationRootfsImage] = info.configDump.RootfsImage + checkpointImageAnnotations[metadata.CheckpointAnnotationRootfsImageName] = info.configDump.RootfsImageName + checkpointImageAnnotations[metadata.CheckpointAnnotationRootfsImageID] = info.configDump.RootfsImageRef + checkpointImageAnnotations[metadata.CheckpointAnnotationRuntimeName] = info.configDump.OCIRuntime + + annotationsFilePath, err := writeAnnotationsToFile(tempDir, checkpointImageAnnotations) + if err != nil { + return "", err + } + + return annotationsFilePath, nil +} diff --git a/internal/scripts/build_image.sh b/internal/scripts/build_image.sh new file mode 100644 index 00000000..35080a24 --- /dev/null +++ b/internal/scripts/build_image.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +set -euo pipefail + +usage() { + cat </dev/null; then + echo "buildah is not installed. Please install buildah before running 'checkpointctl build' command." + exit 1 +fi + +if [[ ! -f $annotationsFilePath ]]; then + echo "Annotations file not found: $annotationsFilePath" + exit 1 +fi + +if [[ ! -f $checkpointPath ]]; then + echo "Checkpoint file not found: $checkpointPath" + exit 1 +fi + +newcontainer=$(buildah from scratch) + +buildah add "$newcontainer" "$checkpointPath" + +while IFS= read -r line; do + key=$(echo "$line" | cut -d '=' -f 1) + value=$(echo "$line" | cut -d '=' -f 2-) + buildah config --annotation "$key=$value" "$newcontainer" +done <"$annotationsFilePath" + +buildah commit "$newcontainer" "$imageName" + +buildah rm "$newcontainer" + +echo "Checkpoint image created successfully: $imageName" diff --git a/lib/annotations.go b/lib/annotations.go new file mode 100644 index 00000000..85962058 --- /dev/null +++ b/lib/annotations.go @@ -0,0 +1,35 @@ +package metadata + +const ( + // CheckpointAnnotationContainerManager is used when creating an OCI image + // from a checkpoint archive to specify the name of container manager. + CheckpointAnnotationContainerManager = "checkpointctl.annotation.container.manager" + + // CheckpointAnnotationName is used when creating an OCI image + // from a checkpoint archive to specify the name of the checkpoint. + CheckpointAnnotationName = "checkpointctl.annotation.checkpoint.name" + + // CheckpointAnnotationPod is used when creating an OCI image + // from a checkpoint archive to specify the name of the pod associated with the checkpoint. + CheckpointAnnotationPod = "checkpointctl.annotation.checkpoint.pod" + + // CheckpointAnnotationNamespace is used when creating an OCI image + // from a checkpoint archive to specify the namespace of the pod associated with the checkpoint. + CheckpointAnnotationNamespace = "checkpointctl.annotation.checkpoint.namespace" + + // CheckpointAnnotationRootfsImage is used when creating an OCI image + // from a checkpoint archive to specify the root filesystem image associated with the checkpoint. + CheckpointAnnotationRootfsImage = "checkpointctl.annotation.checkpoint.rootfsImage" + + // CheckpointAnnotationRootfsImageID is used when creating an OCI image + // from a checkpoint archive to specify the ID of the root filesystem image associated with the checkpoint. + CheckpointAnnotationRootfsImageID = "checkpointctl.annotation.checkpoint.rootfsImageID" + + // CheckpointAnnotationRootfsImageName is used when creating an OCI image + // from a checkpoint archive to specify the name of the root filesystem image associated with the checkpoint. + CheckpointAnnotationRootfsImageName = "checkpointctl.annotation.checkpoint.rootfsImageName" + + // CheckpointAnnotationRuntimeName is used when creating an OCI image + // from a checkpoint archive to specify the runtime used on the host where the checkpoint was created. + CheckpointAnnotationRuntimeName = "checkpointctl.annotation.checkpoint.runtime.name" +)