From 209aaf58fa30028f31292cd2f1ca477bec9f835a Mon Sep 17 00:00:00 2001 From: Robbie Buxton <138501839+rbpdt@users.noreply.github.com> Date: Fri, 11 Aug 2023 18:05:07 +0100 Subject: [PATCH 1/2] updated CI to work with a forked version of containerd Signed-off-by: Robbie Buxton <138501839+rbpdt@users.noreply.github.com> --- .github/workflows/tests.yml | 16 ++++++++-------- Dockerfile | 8 ++++---- cmd/go.mod | 6 ++++-- cmd/go.sum | 8 ++++---- go.mod | 4 +++- go.sum | 4 ++-- ipfs/go.mod | 2 +- ipfs/go.sum | 4 ++-- script/k3s-argo-workflow/run.sh | 5 ++++- script/k3s/run-k3s.sh | 6 +++++- service/keychain/kubeconfig/kubeconfig.go | 2 +- 11 files changed, 38 insertions(+), 27 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4365409af..873d54fb3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,7 @@ name: Tests on: push: branches: - - main + - v1.7.2-stargz pull_request: env: @@ -52,7 +52,7 @@ jobs: strategy: fail-fast: false matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version + buildargs: ["", "--build-arg=CONTAINERD_VERSION=v1.7.2-stargz"] # released version & v1.7.2-stargz version builtin: ["true", "false"] metadata-store: ["memory", "db"] exclude: @@ -61,7 +61,7 @@ jobs: - metadata-store: "db" builtin: "true" - metadata-store: "db" - buildargs: "--build-arg=CONTAINERD_VERSION=main" + buildargs: "--build-arg=CONTAINERD_VERSION=v1.7.2-stargz" steps: - name: Install htpasswd for setting up private registry run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils @@ -79,7 +79,7 @@ jobs: strategy: fail-fast: false matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version + buildargs: ["", "--build-arg=CONTAINERD_VERSION=v1.7.2-stargz"] # released version & v1.7.2-stargz version steps: - name: Install htpasswd for setting up private registry run: sudo apt-get update -y && sudo apt-get --no-install-recommends install -y apache2-utils @@ -95,7 +95,7 @@ jobs: strategy: fail-fast: false matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version + buildargs: ["", "--build-arg=CONTAINERD_VERSION=v1.7.2-stargz"] # released version & v1.7.2-stargz version builtin: ["true", "false"] exclude: - buildargs: "" @@ -116,7 +116,7 @@ jobs: strategy: fail-fast: false matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version + buildargs: ["", "--build-arg=CONTAINERD_VERSION=v1.7.2-stargz"] # released version & v1.7.2-stargz version builtin: ["true", "false"] exclude: - buildargs: "" @@ -137,7 +137,7 @@ jobs: strategy: fail-fast: false matrix: - buildargs: ["", "--build-arg=CONTAINERD_VERSION=main"] # released version & main version + buildargs: ["", "--build-arg=CONTAINERD_VERSION=v1.7.2-stargz"] # released version & v1.7.2-stargz version builtin: ["true", "false"] metadata-store: ["memory", "db"] exclude: @@ -146,7 +146,7 @@ jobs: - metadata-store: "db" builtin: "true" - metadata-store: "db" - buildargs: "--build-arg=CONTAINERD_VERSION=main" + buildargs: "--build-arg=CONTAINERD_VERSION=v1.7.2-stargz" steps: - uses: actions/checkout@v3 - name: Validate containerd through CRI diff --git a/Dockerfile b/Dockerfile index 6a274590b..9fa7b7e1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG CONTAINERD_VERSION=v1.7.2 +ARG CONTAINERD_VERSION=v1.7.2-stargz ARG RUNC_VERSION=v1.1.7 ARG CNI_PLUGINS_VERSION=v1.3.0 ARG NERDCTL_VERSION=1.4.0 @@ -32,14 +32,14 @@ ARG CRI_TOOLS_VERSION=v1.27.0 # Legacy builder that doesn't support TARGETARCH should set this explicitly using --build-arg. # If TARGETARCH isn't supported by the builder, the default value is "amd64". -FROM golang:1.20.6-bullseye AS golang-base +FROM golang:1.20.7-bullseye AS golang-base # Build containerd FROM golang-base AS containerd-dev ARG CONTAINERD_VERSION RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ git clone -b ${CONTAINERD_VERSION} --depth 1 \ - https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \ + https://github.com/pdtpartners/containerd $GOPATH/src/github.com/containerd/containerd && \ cd $GOPATH/src/github.com/containerd/containerd && \ make && DESTDIR=/out/ PREFIX= make install @@ -49,7 +49,7 @@ ARG CONTAINERD_VERSION COPY . $GOPATH/src/github.com/containerd/stargz-snapshotter RUN apt-get update -y && apt-get install -y libbtrfs-dev libseccomp-dev && \ git clone -b ${CONTAINERD_VERSION} --depth 1 \ - https://github.com/containerd/containerd $GOPATH/src/github.com/containerd/containerd && \ + https://github.com/pdtpartners/containerd $GOPATH/src/github.com/containerd/containerd && \ cd $GOPATH/src/github.com/containerd/containerd && \ echo 'require github.com/containerd/stargz-snapshotter v0.0.0' >> go.mod && \ echo 'replace github.com/containerd/stargz-snapshotter => '$GOPATH'/src/github.com/containerd/stargz-snapshotter' >> go.mod && \ diff --git a/cmd/go.mod b/cmd/go.mod index b33fbfa85..7eece51ce 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -3,7 +3,7 @@ module github.com/containerd/stargz-snapshotter/cmd go 1.19 require ( - github.com/containerd/containerd v1.7.2 + github.com/containerd/containerd v1.7.3 github.com/containerd/go-cni v1.1.9 github.com/containerd/stargz-snapshotter v0.14.3 github.com/containerd/stargz-snapshotter/estargz v0.14.3 @@ -36,7 +36,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cilium/ebpf v0.9.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect - github.com/containerd/cgroups/v3 v3.0.1 // indirect + github.com/containerd/cgroups/v3 v3.0.2 // indirect github.com/containerd/console v1.0.3 // indirect github.com/containerd/continuity v0.4.1 // indirect github.com/containerd/fifo v1.1.0 // indirect @@ -144,3 +144,5 @@ replace ( github.com/containerd/stargz-snapshotter/estargz => ../estargz github.com/containerd/stargz-snapshotter/ipfs => ../ipfs ) + +replace github.com/containerd/containerd => github.com/pdtpartners/containerd v1.7.2-stargz diff --git a/cmd/go.sum b/cmd/go.sum index 7c5dad0f3..a31ad8ed7 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -30,13 +30,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE= -github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= +github.com/containerd/cgroups/v3 v3.0.2 h1:f5WFqIVSgo5IZmtTT3qVBo6TzI1ON6sycSBKkymb9L0= +github.com/containerd/cgroups/v3 v3.0.2/go.mod h1:JUgITrzdFqp42uI2ryGA+ge0ap/nxzYgkGmIcetmErE= github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.7.2 h1:UF2gdONnxO8I6byZXDi5sXWiWvlW3D/sci7dTQimEJo= -github.com/containerd/containerd v1.7.2/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI= github.com/containerd/continuity v0.4.1 h1:wQnVrjIyQ8vhU2sgOiL5T07jo+ouqc2bnKsv5/EqGhU= github.com/containerd/continuity v0.4.1/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= @@ -280,6 +278,8 @@ github.com/opencontainers/runtime-spec v1.1.0-rc.3/go.mod h1:jwyrGlmzljRJv/Fgzds github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/pdtpartners/containerd v1.7.2-stargz h1:b/o+aD51Am8oWD01MRAbfuBWt+oR1OzJaXWIWtAhLis= +github.com/pdtpartners/containerd v1.7.2-stargz/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/go.mod b/go.mod index 4addbf2bd..eab0688c4 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.19 require ( github.com/containerd/console v1.0.3 github.com/containerd/containerd v1.7.2 - github.com/containerd/continuity v0.4.1 github.com/containerd/stargz-snapshotter/estargz v0.14.3 github.com/docker/cli v24.0.4+incompatible github.com/docker/go-metrics v0.0.1 @@ -39,6 +38,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/cgroups/v3 v3.0.1 // indirect + github.com/containerd/continuity v0.4.1 // indirect github.com/containerd/fifo v1.1.0 // indirect github.com/containerd/go-cni v1.1.9 // indirect github.com/containerd/ttrpc v1.2.2 // indirect @@ -116,3 +116,5 @@ require ( // Import local package for estargz. replace github.com/containerd/stargz-snapshotter/estargz => ./estargz + +replace github.com/containerd/containerd => github.com/pdtpartners/containerd v1.7.2-stargz diff --git a/go.sum b/go.sum index d1e44f111..7aa495976 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,6 @@ github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5To github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.7.2 h1:UF2gdONnxO8I6byZXDi5sXWiWvlW3D/sci7dTQimEJo= -github.com/containerd/containerd v1.7.2/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI= github.com/containerd/continuity v0.4.1 h1:wQnVrjIyQ8vhU2sgOiL5T07jo+ouqc2bnKsv5/EqGhU= github.com/containerd/continuity v0.4.1/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= @@ -230,6 +228,8 @@ github.com/opencontainers/runtime-spec v1.1.0-rc.3/go.mod h1:jwyrGlmzljRJv/Fgzds github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= +github.com/pdtpartners/containerd v1.7.2-stargz h1:b/o+aD51Am8oWD01MRAbfuBWt+oR1OzJaXWIWtAhLis= +github.com/pdtpartners/containerd v1.7.2-stargz/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/ipfs/go.mod b/ipfs/go.mod index 3baa9e946..6d6662b9a 100644 --- a/ipfs/go.mod +++ b/ipfs/go.mod @@ -3,7 +3,7 @@ module github.com/containerd/stargz-snapshotter/ipfs go 1.19 require ( - github.com/containerd/containerd v1.7.2 + github.com/containerd/containerd v1.7.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.10.1 github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b diff --git a/ipfs/go.sum b/ipfs/go.sum index 01872d662..52daeb608 100644 --- a/ipfs/go.sum +++ b/ipfs/go.sum @@ -16,8 +16,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= -github.com/containerd/containerd v1.7.2 h1:UF2gdONnxO8I6byZXDi5sXWiWvlW3D/sci7dTQimEJo= -github.com/containerd/containerd v1.7.2/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI= +github.com/containerd/containerd v1.7.3 h1:cKwYKkP1eTj54bP3wCdXXBymmKRQMrWjkLSWZZJDa8o= +github.com/containerd/containerd v1.7.3/go.mod h1:32FOM4/O0RkNg7AjQj3hDzN9cUGtu+HMvaKUNiqCZB8= github.com/containerd/continuity v0.4.1 h1:wQnVrjIyQ8vhU2sgOiL5T07jo+ouqc2bnKsv5/EqGhU= github.com/containerd/continuity v0.4.1/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= diff --git a/script/k3s-argo-workflow/run.sh b/script/k3s-argo-workflow/run.sh index a87d9b9e5..f4201d96f 100755 --- a/script/k3s-argo-workflow/run.sh +++ b/script/k3s-argo-workflow/run.sh @@ -21,7 +21,7 @@ REPO="${CONTEXT}../../" K3S_VERSION=master K3S_REPO=https://github.com/k3s-io/k3s -K3S_CONTAINERD_REPO=https://github.com/k3s-io/containerd +K3S_CONTAINERD_REPO=https://github.com/pdtpartners/containerd K3S_NODE_REPO=ghcr.io/stargz-containers K3S_NODE_IMAGE_NAME=k3s @@ -126,6 +126,8 @@ wget -O "${ORG_ARGOYAML}" https://raw.githubusercontent.com/argoproj/argo-workfl git clone -b ${K3S_VERSION} --depth 1 "${K3S_REPO}" "${TMP_K3S_REPO}" sed -i "s|github.com/k3s-io/stargz-snapshotter .*$|$(realpath ${REPO})|g" "${TMP_K3S_REPO}/go.mod" +sed -i "s|github.com/k3s-io/containerd v1.7.3-k3s1|github.com/pdtpartners/containerd v1.7.2-stargz|g" "${TMP_K3S_REPO}/go.mod" + echo "replace github.com/containerd/stargz-snapshotter/estargz => $(realpath ${REPO})/estargz" >> "${TMP_K3S_REPO}/go.mod" # typeurl version stargz-snapshotter indirectly depends on is incompatible to the one github.com/k3s-io/containerd depends on. @@ -137,6 +139,7 @@ cat "${TMP_K3S_REPO}/go.mod" sed -i -E 's|(ENV DAPPER_RUN_ARGS .*)|\1 -v '"$(realpath ${REPO})":"$(realpath ${REPO})"':ro|g' "${TMP_K3S_REPO}/Dockerfile.dapper" sed -i -E 's|(ENV DAPPER_ENV .*)|\1 DOCKER_BUILDKIT|g' "${TMP_K3S_REPO}/Dockerfile.dapper" +sed -i -E 's|github.com/k3s-io/containerd|github.com/pdtpartners/containerd|g' "${TMP_K3S_REPO}/scripts/download" ( cd "${TMP_K3S_REPO}" && \ git config user.email "dummy@example.com" && \ diff --git a/script/k3s/run-k3s.sh b/script/k3s/run-k3s.sh index 0878115ec..4d79c5c47 100755 --- a/script/k3s/run-k3s.sh +++ b/script/k3s/run-k3s.sh @@ -18,7 +18,7 @@ set -euo pipefail K3S_VERSION=master K3S_REPO=https://github.com/k3s-io/k3s -K3S_CONTAINERD_REPO=https://github.com/k3s-io/containerd +K3S_CONTAINERD_REPO=https://github.com/pdtpartners/containerd REGISTRY_HOST=k3s-private-registry K3S_NODE_REPO=ghcr.io/stargz-containers @@ -55,6 +55,8 @@ trap 'cleanup "$?"' EXIT SIGHUP SIGINT SIGQUIT SIGTERM echo "Preparing node image..." git clone -b ${K3S_VERSION} --depth 1 "${K3S_REPO}" "${TMP_K3S_REPO}" sed -i "s|github.com/k3s-io/stargz-snapshotter .*$|$(realpath ${REPO})|g" "${TMP_K3S_REPO}/go.mod" +sed -i "s|github.com/k3s-io/containerd v1.7.3-k3s1|github.com/pdtpartners/containerd v1.7.2-stargz|g" "${TMP_K3S_REPO}/go.mod" + echo "replace github.com/containerd/stargz-snapshotter/estargz => $(realpath ${REPO})/estargz" >> "${TMP_K3S_REPO}/go.mod" # typeurl version stargz-snapshotter indirectly depends on is incompatible to the one github.com/k3s-io/containerd depends on. @@ -66,6 +68,8 @@ cat "${TMP_K3S_REPO}/go.mod" sed -i -E 's|(ENV DAPPER_RUN_ARGS .*)|\1 -v '"$(realpath ${REPO})":"$(realpath ${REPO})"':ro|g' "${TMP_K3S_REPO}/Dockerfile.dapper" sed -i -E 's|(ENV DAPPER_ENV .*)|\1 DOCKER_BUILDKIT|g' "${TMP_K3S_REPO}/Dockerfile.dapper" +sed -i -E 's|github.com/k3s-io/containerd|github.com/pdtpartners/containerd|g' "${TMP_K3S_REPO}/scripts/download" + ( cd "${TMP_K3S_REPO}" && \ git config user.email "dummy@example.com" && \ diff --git a/service/keychain/kubeconfig/kubeconfig.go b/service/keychain/kubeconfig/kubeconfig.go index 27f8b41d0..f200edb08 100644 --- a/service/keychain/kubeconfig/kubeconfig.go +++ b/service/keychain/kubeconfig/kubeconfig.go @@ -218,7 +218,7 @@ func (kc *keychain) startSyncSecrets(ctx context.Context, client kubernetes.Inte } func (kc *keychain) runWorker() { - for kc.processNextItem() { + for kc.processNextItem() { //nolint:all // continue looping } } From 51a564bb2ff15fc0f3a8c5db1efe2ae07cc44787 Mon Sep 17 00:00:00 2001 From: Robbie Buxton <138501839+rbpdt@users.noreply.github.com> Date: Fri, 11 Aug 2023 18:20:55 +0100 Subject: [PATCH 2/2] embedded the overlay snapshotter and removed duplicated code Signed-off-by: Robbie Buxton <138501839+rbpdt@users.noreply.github.com> --- snapshot/snapshot.go | 343 +++++--------------------------------- snapshot/snapshot_test.go | 10 +- 2 files changed, 46 insertions(+), 307 deletions(-) diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 5bf378411..8d3f7ec6d 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -28,9 +28,8 @@ import ( "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/snapshots" - "github.com/containerd/containerd/snapshots/overlay/overlayutils" + "github.com/containerd/containerd/snapshots/overlay" "github.com/containerd/containerd/snapshots/storage" - "github.com/containerd/continuity/fs" "github.com/moby/sys/mountinfo" "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" @@ -99,13 +98,13 @@ func AllowInvalidMountsOnRestart(config *SnapshotterConfig) error { } type snapshotter struct { + snapshots.Snapshotter root string ms *storage.MetaStore asyncRemove bool // fs is a filesystem that this snapshotter recognizes. fs FileSystem - userxattr bool // whether to enable "userxattr" mount option noRestore bool allowInvalidMountsOnRestart bool } @@ -126,36 +125,26 @@ func NewSnapshotter(ctx context.Context, root string, targetFs FileSystem, opts } } - if err := os.MkdirAll(root, 0700); err != nil { - return nil, err - } - supportsDType, err := fs.SupportsDType(root) - if err != nil { - return nil, err - } - if !supportsDType { - return nil, fmt.Errorf("%s does not support d_type. If the backing filesystem is xfs, please reformat with ftype=1 to enable d_type support", root) - } ms, err := storage.NewMetaStore(filepath.Join(root, "metadata.db")) if err != nil { return nil, err } - if err := os.Mkdir(filepath.Join(root, "snapshots"), 0700); err != nil && !os.IsExist(err) { - return nil, err + overlayOpts := []overlay.Opt{} + overlayOpts = append(overlayOpts, overlay.WithMetaStore(ms)) + if config.asyncRemove { + overlayOpts = append(overlayOpts, overlay.AsynchronousRemove) } - - userxattr, err := overlayutils.NeedsUserXAttr(root) + overlaySnapshotter, err := overlay.NewSnapshotter(root, overlayOpts...) if err != nil { - logrus.WithError(err).Warnf("cannot detect whether \"userxattr\" option needs to be used, assuming to be %v", userxattr) + return nil, err } - o := &snapshotter{ + Snapshotter: overlaySnapshotter, root: root, ms: ms, asyncRemove: config.asyncRemove, fs: targetFs, - userxattr: userxattr, noRestore: config.noRestore, allowInvalidMountsOnRestart: config.allowInvalidMountsOnRestart, } @@ -167,85 +156,11 @@ func NewSnapshotter(ctx context.Context, root string, targetFs FileSystem, opts return o, nil } -// Stat returns the info for an active or committed snapshot by name or -// key. -// -// Should be used for parent resolution, existence checks and to discern -// the kind of snapshot. -func (o *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { - ctx, t, err := o.ms.TransactionContext(ctx, false) - if err != nil { - return snapshots.Info{}, err - } - defer t.Rollback() - _, info, _, err := storage.GetInfo(ctx, key) - if err != nil { - return snapshots.Info{}, err - } - - return info, nil -} - -func (o *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { - ctx, t, err := o.ms.TransactionContext(ctx, true) - if err != nil { - return snapshots.Info{}, err - } - - info, err = storage.UpdateInfo(ctx, info, fieldpaths...) - if err != nil { - t.Rollback() - return snapshots.Info{}, err - } - - if err := t.Commit(); err != nil { - return snapshots.Info{}, err - } - - return info, nil -} - -// Usage returns the resources taken by the snapshot identified by key. -// -// For active snapshots, this will scan the usage of the overlay "diff" (aka -// "upper") directory and may take some time. -// for remote snapshots, no scan will be held and recognise the number of inodes -// and these sizes as "zero". -// -// For committed snapshots, the value is returned from the metadata database. -func (o *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { - ctx, t, err := o.ms.TransactionContext(ctx, false) - if err != nil { - return snapshots.Usage{}, err - } - id, info, usage, err := storage.GetInfo(ctx, key) - t.Rollback() // transaction no longer needed at this point. - - if err != nil { - return snapshots.Usage{}, err - } - - upperPath := o.upperPath(id) - - if info.Kind == snapshots.KindActive { - du, err := fs.DiskUsage(ctx, upperPath) - if err != nil { - // TODO(stevvooe): Consider not reporting an error in this case. - return snapshots.Usage{}, err - } - - usage = snapshots.Usage(du) - } - - return usage, nil -} - func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { - s, err := o.createSnapshot(ctx, snapshots.KindActive, key, parent, opts) + mnts, err := o.Snapshotter.Prepare(ctx, key, parent, opts...) if err != nil { return nil, err } - // Try to prepare the remote snapshot. If succeeded, we commit the snapshot now // and return ErrAlreadyExists. var base snapshots.Info @@ -265,7 +180,7 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s WithError(err).Warn("failed to prepare remote snapshot") } else { base.Labels[remoteLabel] = remoteLabelVal // Mark this snapshot as remote - err := o.commit(ctx, true, target, key, append(opts, snapshots.WithLabels(base.Labels))...) + err := o.remoteCommit(ctx, target, key, append(opts, snapshots.WithLabels(base.Labels))...) if err == nil || errdefs.IsAlreadyExists(err) { // count also AlreadyExists as "success" log.G(lCtx).WithField(remoteSnapshotLogKey, prepareSucceeded).Debug("prepared remote snapshot") @@ -278,39 +193,14 @@ func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...s return nil, err } } - return o.mounts(ctx, s, parent) -} - -func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { - s, err := o.createSnapshot(ctx, snapshots.KindView, key, parent, opts) + err = o.checkKey(ctx, parent) if err != nil { return nil, err } - return o.mounts(ctx, s, parent) + return mnts, nil } -// Mounts returns the mounts for the transaction identified by key. Can be -// called on an read-write or readonly transaction. -// -// This can be used to recover mounts after calling View or Prepare. -func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) { - ctx, t, err := o.ms.TransactionContext(ctx, false) - if err != nil { - return nil, err - } - s, err := storage.GetSnapshot(ctx, key) - t.Rollback() - if err != nil { - return nil, fmt.Errorf("failed to get active mount: %w", err) - } - return o.mounts(ctx, s, key) -} - -func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { - return o.commit(ctx, false, name, key, opts...) -} - -func (o *snapshotter) commit(ctx context.Context, isRemote bool, name, key string, opts ...snapshots.Opt) error { +func (o *snapshotter) remoteCommit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { ctx, t, err := o.ms.TransactionContext(ctx, true) if err != nil { return err @@ -326,19 +216,11 @@ func (o *snapshotter) commit(ctx context.Context, isRemote bool, name, key strin }() // grab the existing id - id, _, usage, err := storage.GetInfo(ctx, key) + _, _, usage, err := storage.GetInfo(ctx, key) if err != nil { return err } - if !isRemote { // skip diskusage for remote snapshots for allowing lazy preparation of nodes - du, err := fs.DiskUsage(ctx, o.upperPath(id)) - if err != nil { - return err - } - usage = snapshots.Usage(du) - } - if _, err = storage.CommitActive(ctx, key, name, usage, opts...); err != nil { return fmt.Errorf("failed to commit snapshot: %w", err) } @@ -347,6 +229,26 @@ func (o *snapshotter) commit(ctx context.Context, isRemote bool, name, key strin return t.Commit() } +func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + err := o.checkKey(ctx, parent) + if err != nil { + return nil, err + } + return o.Snapshotter.View(ctx, key, parent, opts...) +} + +// Mounts returns the mounts for the transaction identified by key. Can be +// called on an read-write or readonly transaction. +// +// This can be used to recover mounts after calling View or Prepare. +func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) { + err := o.checkKey(ctx, key) + if err != nil { + return nil, err + } + return o.Snapshotter.Mounts(ctx, key) +} + // Remove abandons the snapshot identified by key. The snapshot will // immediately become unavailable and unrecoverable. Disk space will // be freed up on the next call to `Cleanup`. @@ -394,20 +296,9 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { return t.Commit() } -// Walk the snapshots. -func (o *snapshotter) Walk(ctx context.Context, fn snapshots.WalkFunc, fs ...string) error { - ctx, t, err := o.ms.TransactionContext(ctx, false) - if err != nil { - return err - } - defer t.Rollback() - return storage.WalkInfo(ctx, fn, fs...) -} - // Cleanup cleans up disk resources from removed or abandoned snapshots func (o *snapshotter) Cleanup(ctx context.Context) error { - const cleanupCommitted = false - return o.cleanup(ctx, cleanupCommitted) + return o.cleanup(ctx, false) } func (o *snapshotter) cleanup(ctx context.Context, cleanupCommitted bool) error { @@ -485,172 +376,18 @@ func (o *snapshotter) cleanupSnapshotDirectory(ctx context.Context, dir string) return nil } -func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) (_ storage.Snapshot, err error) { - ctx, t, err := o.ms.TransactionContext(ctx, true) - if err != nil { - return storage.Snapshot{}, err - } - - var td, path string - defer func() { - if err != nil { - if td != "" { - if err1 := o.cleanupSnapshotDirectory(ctx, td); err1 != nil { - log.G(ctx).WithError(err1).Warn("failed to cleanup temp snapshot directory") - } - } - if path != "" { - if err1 := o.cleanupSnapshotDirectory(ctx, path); err1 != nil { - log.G(ctx).WithError(err1).WithField("path", path).Error("failed to reclaim snapshot directory, directory may need removal") - err = fmt.Errorf("failed to remove path: %v: %w", err1, err) - } - } - } - }() - - snapshotDir := filepath.Join(o.root, "snapshots") - td, err = o.prepareDirectory(ctx, snapshotDir, kind) - if err != nil { - if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") - } - return storage.Snapshot{}, fmt.Errorf("failed to create prepare snapshot dir: %w", err) - } - rollback := true - defer func() { - if rollback { - if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") - } - } - }() - - s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...) - if err != nil { - return storage.Snapshot{}, fmt.Errorf("failed to create snapshot: %w", err) - } - - if len(s.ParentIDs) > 0 { - st, err := os.Stat(o.upperPath(s.ParentIDs[0])) - if err != nil { - return storage.Snapshot{}, fmt.Errorf("failed to stat parent: %w", err) - } - - stat := st.Sys().(*syscall.Stat_t) - - if err := os.Lchown(filepath.Join(td, "fs"), int(stat.Uid), int(stat.Gid)); err != nil { - if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") - } - return storage.Snapshot{}, fmt.Errorf("failed to chown: %w", err) - } - } - - path = filepath.Join(snapshotDir, s.ID) - if err = os.Rename(td, path); err != nil { - return storage.Snapshot{}, fmt.Errorf("failed to rename: %w", err) - } - td = "" - - rollback = false - if err = t.Commit(); err != nil { - return storage.Snapshot{}, fmt.Errorf("commit failed: %w", err) - } - - return s, nil -} - -func (o *snapshotter) prepareDirectory(ctx context.Context, snapshotDir string, kind snapshots.Kind) (string, error) { - td, err := os.MkdirTemp(snapshotDir, "new-") - if err != nil { - return "", fmt.Errorf("failed to create temp dir: %w", err) - } - - if err := os.Mkdir(filepath.Join(td, "fs"), 0755); err != nil { - return td, err - } - - if kind == snapshots.KindActive { - if err := os.Mkdir(filepath.Join(td, "work"), 0711); err != nil { - return td, err - } - } - - return td, nil -} - -func (o *snapshotter) mounts(ctx context.Context, s storage.Snapshot, checkKey string) ([]mount.Mount, error) { +func (o *snapshotter) checkKey(ctx context.Context, checkKey string) error { // Make sure that all layers lower than the target layer are available if checkKey != "" && !o.checkAvailability(ctx, checkKey) { - return nil, fmt.Errorf("layer %q unavailable: %w", s.ID, errdefs.ErrUnavailable) + return fmt.Errorf("layer %q unavailable: %w", checkKey, errdefs.ErrUnavailable) } - - if len(s.ParentIDs) == 0 { - // if we only have one layer/no parents then just return a bind mount as overlay - // will not work - roFlag := "rw" - if s.Kind == snapshots.KindView { - roFlag = "ro" - } - - return []mount.Mount{ - { - Source: o.upperPath(s.ID), - Type: "bind", - Options: []string{ - roFlag, - "rbind", - }, - }, - }, nil - } - var options []string - - if s.Kind == snapshots.KindActive { - options = append(options, - fmt.Sprintf("workdir=%s", o.workPath(s.ID)), - fmt.Sprintf("upperdir=%s", o.upperPath(s.ID)), - ) - } else if len(s.ParentIDs) == 1 { - return []mount.Mount{ - { - Source: o.upperPath(s.ParentIDs[0]), - Type: "bind", - Options: []string{ - "ro", - "rbind", - }, - }, - }, nil - } - - parentPaths := make([]string, len(s.ParentIDs)) - for i := range s.ParentIDs { - parentPaths[i] = o.upperPath(s.ParentIDs[i]) - } - - options = append(options, fmt.Sprintf("lowerdir=%s", strings.Join(parentPaths, ":"))) - if o.userxattr { - options = append(options, "userxattr") - } - return []mount.Mount{ - { - Type: "overlay", - Source: "overlay", - Options: options, - }, - }, nil - + return nil } func (o *snapshotter) upperPath(id string) string { return filepath.Join(o.root, "snapshots", id, "fs") } -func (o *snapshotter) workPath(id string) string { - return filepath.Join(o.root, "snapshots", id, "work") -} - // Close closes the snapshotter func (o *snapshotter) Close() error { // unmount all mounts including Committed @@ -659,7 +396,7 @@ func (o *snapshotter) Close() error { if err := o.cleanup(ctx, cleanupCommitted); err != nil { log.G(ctx).WithError(err).Warn("failed to cleanup") } - return o.ms.Close() + return o.Snapshotter.Close() } // prepareRemoteSnapshot tries to prepare the snapshot as a remote snapshot diff --git a/snapshot/snapshot_test.go b/snapshot/snapshot_test.go index b0476349c..ea8bf6179 100644 --- a/snapshot/snapshot_test.go +++ b/snapshot/snapshot_test.go @@ -142,6 +142,7 @@ func TestRemoteOverlay(t *testing.T) { lower = "lowerdir=" + getParents(ctx, sn, root, pKey)[0] ) for i, v := range []string{ + "index=off", work, upper, lower, @@ -549,6 +550,7 @@ func TestOverlayOverlayMount(t *testing.T) { lower = "lowerdir=" + getParents(ctx, o, root, "/tmp/layer2")[0] ) for i, v := range []string{ + "index=off", work, upper, lower, @@ -709,12 +711,12 @@ func TestOverlayView(t *testing.T) { if m.Source != "overlay" { t.Errorf("mount source should be overlay but received %q", m.Source) } - if len(m.Options) != 1 { - t.Errorf("expected 1 mount option but got %d", len(m.Options)) + if len(m.Options) != 2 { + t.Errorf("expected 2 mount option but got %d", len(m.Options)) } lowers := getParents(ctx, o, root, "/tmp/view2") expected = fmt.Sprintf("lowerdir=%s:%s", lowers[0], lowers[1]) - if m.Options[0] != expected { - t.Errorf("expected option %q but received %q", expected, m.Options[0]) + if m.Options[1] != expected { + t.Errorf("expected option %q but received %q", expected, m.Options[1]) } }