Merge pull request #5775 from fidencio/topic/CC-switch-tests-from-for…

…ked-containerd-to-pulling-on-guest CC | ci | stop testing with the forked containerd ...
kata-containers · Oct 9, 2023 · 61806ee · 61806ee
2 parents 6cf4306 + dca198d
commit 61806ee
Show file tree

Hide file tree

Showing 17 changed files with 277 additions and 61 deletions.
diff --git a/.ci/ci_job_flags.sh b/.ci/ci_job_flags.sh
@@ -40,6 +40,9 @@ init_ci_flags() {
 	# Use the forked version of containerd for Confidential Containers
 	# Valyes: "yes|no"
 	export FORKED_CONTAINERD="no"
+	# Do the pull on the guest using the upstream containerd for Confidential Containers
+	# Values: "yes|no"
+	export IMAGE_OFFLOAD_TO_GUEST="no"
 	# Hypervisor to use
 	export KATA_HYPERVISOR=""
 	# Install k8s
@@ -122,7 +125,8 @@ case "${CI_JOB}" in
 		"CC_CRI_CONTAINERD"|"CC_CRI_CONTAINERD_K8S")
 			# Export any CC specific environment variables
 			export KATA_BUILD_CC="yes"
-			export FORKED_CONTAINERD="yes"
+			export FORKED_CONTAINERD="no"
+			export IMAGE_OFFLOAD_TO_GUEST="yes"
 			export MEASURED_ROOTFS="yes"
 			export AA_KBC="offline_fs_kbc"
 			if [[ "${CI_JOB}" =~ K8S ]]; then
@@ -137,7 +141,8 @@ case "${CI_JOB}" in
 	export CRI_RUNTIME="containerd"
 	export KATA_HYPERVISOR="qemu"
 	export KATA_BUILD_CC="yes"
-	export FORKED_CONTAINERD="yes"
+	export FORKED_CONTAINERD="no"
+	export IMAGE_OFFLOAD_TO_GUEST="yes"
 	export AA_KBC="offline_fs_kbc"
 	export TEST_INITRD="yes"
 	if [[ "${CI_JOB}" =~ K8S ]]; then
@@ -160,7 +165,8 @@ case "${CI_JOB}" in
 	export CRI_RUNTIME="containerd"
 	export KATA_HYPERVISOR="qemu"
 	export KATA_BUILD_CC="yes"
-	export FORKED_CONTAINERD="yes"
+	export FORKED_CONTAINERD="no"
+	export IMAGE_OFFLOAD_TO_GUEST="yes"
 	export MEASURED_ROOTFS="yes"
 	export AA_KBC="cc_kbc"
 	export TEE_TYPE="tdx"
@@ -180,7 +186,8 @@ case "${CI_JOB}" in
 	export KUBERNETES="yes"
 	export AA_KBC="offline_fs_kbc"
 	export KATA_BUILD_CC="yes"
-	export FORKED_CONTAINERD="yes"
+	export FORKED_CONTAINERD="no"
+	export IMAGE_OFFLOAD_TO_GUEST="yes"
 	export MEASURED_ROOTFS="yes"
 	if [[ "${CI_JOB}" =~ TDX ]]; then
 		export TEE_TYPE="tdx"
@@ -206,7 +213,8 @@ case "${CI_JOB}" in
 	export KATA_HYPERVISOR="cloud-hypervisor"
 	# Export any CC specific environment variables
 	export KATA_BUILD_CC="yes"
-	export FORKED_CONTAINERD="yes"
+	export FORKED_CONTAINERD="no"
+	export IMAGE_OFFLOAD_TO_GUEST="yes"
 	export MEASURED_ROOTFS="yes"
 	export AA_KBC="offline_fs_kbc"
 	;;

diff --git a/.ci/containerd_nydus_setup.sh b/.ci/containerd_nydus_setup.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cidir=$(dirname "$0")
+source "${cidir}/../lib/common.bash"
+
+# Nydus related configurations
+NYDUS_SNAPSHOTTER_BINARY="/usr/local/bin/containerd-nydus-grpc"
+NYDUS_SNAPSHOTTER_TARFS_CONFIG="/usr/local/share/nydus-snapshotter/config-coco-host-sharing.toml"
+NYDUS_SNAPSHOTTER_GUEST_CONFIG="/usr/local/share/nydus-snapshotter/config-coco-guest-pulling.toml"
+NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_CONFIG:-${NYDUS_SNAPSHOTTER_TARFS_CONFIG}}"
+NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE="${PULL_ON_HOST_EXPORT_MODE:-image_block}"
+
+echo "Configure nydus snapshotter"
+if [ "${IMAGE_OFFLOAD_TO_GUEST:-"no"}" == "yes" ]; then
+	echo "Pulling image on the guest"
+	NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_GUEST_CONFIG}"
+else
+	echo "Pulling image on the host | export_mode = ${NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE}"
+	NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_TARFS_CONFIG}"
+	sudo sed -i "s/export_mode = .*/export_mode = \"${NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE}\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
+fi
+
+echo "Start nydus snapshotter"
+sudo "${NYDUS_SNAPSHOTTER_BINARY}" --config "${NYDUS_SNAPSHOTTER_CONFIG}" >/dev/stdout 2>&1 &
+
+echo "Configure containerd to use the nydus snapshotter"
+
+containerd_config_file="/etc/containerd/config.toml"
+
+snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock"
+proxy_config="  [proxy_plugins.nydus]\n    type = \"snapshot\"\n    address = \"${snapshotter_socket}\""
+snapshotter_config="      disable_snapshot_annotations = false\n      snapshotter = \"nydus\""
+
+echo -e "[proxy_plugins]" | sudo tee -a "${containerd_config_file}"
+echo -e "${proxy_config}" | sudo tee -a "${containerd_config_file}"
+
+sudo sed -i '/\[plugins.cri.containerd\]/a\'"${snapshotter_config}" "${containerd_config_file}"
+sudo systemctl restart containerd
+
+# SNP & SEV tests seem to need time for containerd and snapshotter to be running
+# In future fix this to make it check if it's running rather than sleep
+sleep 30
diff --git a/.ci/install_nydus_snapshotter.sh b/.ci/install_nydus_snapshotter.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+target_dir="/usr/local/"
+
+nydus_snapshotter_repo=$(get_version "externals.nydus-snapshotter.url")
+nydus_snapshotter_version=$(get_version "externals.nydus-snapshotter.version")
+nydus_snapshotter_repo_dir="${GOPATH}/src/github.com/containerd/nydus-snapshotter"
+nydus_snapshotter_binary_target_dir="${target_dir}/bin"
+nydus_snapshotter_config_target_dir="${target_dir}/share/nydus-snapshotter"
+
+nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
+nydus_version=${nydus_version:-"v2.2.3"}
+
+arch="$(uname -m)"
+
+clone_nydus_snapshotter_repo() {
+    add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}"
+
+    if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
+        mkdir -p "${nydus_snapshotter_repo_dir}"
+        git clone ${nydus_snapshotter_repo} "${nydus_snapshotter_repo_dir}" || true
+        pushd "${nydus_snapshotter_repo_dir}"
+        git checkout "${nydus_snapshotter_version}"
+        popd
+    fi
+}
+
+build_nydus_snapshotter() {
+    pushd "${nydus_snapshotter_repo_dir}"
+    if [ "${arch}" = "s390x" ]; then
+        export GOARCH=${arch}
+    fi
+    make
+
+    sudo install -D -m 755 "bin/containerd-nydus-grpc" "${nydus_snapshotter_binary_target_dir}/containerd-nydus-grpc"
+    sudo install -D -m 755 "bin/nydus-overlayfs" "${nydus_snapshotter_binary_target_dir}/nydus-overlayfs"
+    rm -rf "${nydus_snapshotter_repo_dir}/bin"
+    popd >/dev/null
+}
+
+download_nydus_snapshotter_config() {
+    tmp_dir=$(mktemp -d -t install-nydus-snapshotter-config-tmp.XXXXXXXXXX)
+    curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/${nydus_snapshotter_version}/misc/snapshotter/config-coco-guest-pulling.toml -o "${tmp_dir}/config-coco-guest-pulling.toml"
+    curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/${nydus_snapshotter_version}/misc/snapshotter/config-coco-host-sharing.toml -o "${tmp_dir}/config-coco-host-sharing.toml"
+    sudo install -D -m 644 "${tmp_dir}/config-coco-guest-pulling.toml" "${nydus_snapshotter_config_target_dir}/config-coco-guest-pulling.toml"
+    sudo install -D -m 644 "${tmp_dir}/config-coco-host-sharing.toml" "${nydus_snapshotter_config_target_dir}/config-coco-host-sharing.toml"
+
+}
+
+download_nydus_from_tarball() {
+    if [ "${arch}" = "s390x" ]; then
+        echo "Skip to download nydus for ${arch}, it doesn't work for ${arch} now."
+        return
+    fi
+    local goarch="$(${cidir}/kata-arch.sh --golang)"
+    local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-${goarch}.tgz"
+    echo "Download tarball from ${tarball_url}"
+    tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
+    curl -Ls "$tarball_url" | sudo tar xfz - -C ${tmp_dir} --strip-components=1
+    sudo install -D -m 755 "${tmp_dir}/nydus-image" "${target_dir}/bin/"
+}
+
+download_nydus_from_tarball
+clone_nydus_snapshotter_repo
+build_nydus_snapshotter
+download_nydus_snapshotter_config
+echo "install nydus-snapshotter successful"
diff --git a/.ci/run.sh b/.ci/run.sh
@@ -73,19 +73,19 @@ case "${CI_JOB}" in
 		;;
 	"CC_CRI_CONTAINERD"|"CC_CRI_CONTAINERD_CLOUD_HYPERVISOR"|"CC_CRI_CONTAINERD_TDX_QEMU"|"CC_CRI_CONTAINERD_TDX_CLOUD_HYPERVISOR")
 		echo "INFO: Running Confidential Container tests"
-		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-containerd"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="yes" bash -c "make cc-containerd"
 		;;
 	"CC_SEV_CRI_CONTAINERD_K8S")
 		info "Running Confidential Containers tests for AMD SEV"
-		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-sev-kubernetes"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="yes" bash -c "make cc-sev-kubernetes"
 		;;
 	"CC_SNP_CRI_CONTAINERD_K8S")
 		info "Running Confidential Containers tests for AMD SEV-SNP"
-		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-snp-kubernetes"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="yes" bash -c "make cc-snp-kubernetes"
 		;;
 	"CC_CRI_CONTAINERD_K8S"|"CC_CRI_CONTAINERD_K8S_TDX_QEMU"|"CC_CRI_CONTAINERD_K8S_SE_QEMU"|"CC_CRI_CONTAINERD_K8S_TDX_CLOUD_HYPERVISOR")
 		info "Running Confidential Container tests"
-		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-kubernetes"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="yes" bash -c "make cc-kubernetes"
 		;;
 	"CRIO_K8S")
 		echo "INFO: Running kubernetes tests"

diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh
@@ -209,13 +209,6 @@ configure_cc_containerd() {
 	waitForProcess 30 5 "sudo crictl info >/dev/null"
 
 	# Ensure the cc CRI handler is set.
-	local cri_handler=$(sudo crictl info | \
-		jq '.config.containerd.runtimes.kata.cri_handler')
-	if [[ ! "$cri_handler" =~ cc ]]; then
-		sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
-			"$containerd_conf_file"
-	fi
-
 	if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
 		echo "    [plugins.cri.cni]
 		  # conf_dir is the directory in which the admin places a CNI conf.
@@ -357,6 +350,11 @@ setup_credentials_files() {
 	auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
 	CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
 	cp_to_guest_img "etc" "${dest_file}"
+
+	# With the change in behaviour in CDH, the `/etc/aa-offline_fs_kbc-keys.json` file has to exist, so create a blank one
+	dest_file=${dest_dir}/aa-offline_fs_kbc-keys.json
+	echo "{}" >  "${dest_file}"
+	cp_to_guest_img "etc" "${dest_file}"
 }
 
 ###############################################################################

diff --git a/integration/containerd/confidential/agent_image.bats b/integration/containerd/confidential/agent_image.bats
@@ -42,7 +42,8 @@ setup() {
 	local pod_id=$(crictl pods --name "$sandbox_name" -q)
 	rootfs=($(find /run/kata-containers/shared/sandboxes/${pod_id}/shared \
 		-name rootfs))
-	[ ${#rootfs[@]} -eq 1 ]
+
+	[ ${#rootfs[@]} -eq 0 ]
 }
 
 @test "$test_tag Test can pull a unencrypted signed image from a protected registry" {
@@ -56,6 +57,9 @@ setup() {
 }
 
 @test "$test_tag Test cannot pull an unencrypted unsigned image from a protected registry" {
+	# Ensure that the signed image that currently has the same sha256 as unsigned is not used by the snapshotter
+	sudo crictl rmi quay.io/kata-containers/confidential-containers:signed
+
 	local container_config="${FIXTURES_DIR}/container-config_unsigned-protected.yaml"
 
 	setup_signature_files
@@ -109,33 +113,65 @@ setup() {
 }
 
 @test "$test_tag Test pull an unencrypted unsigned image from an authenticated registry with correct credentials" {
+	# Docker config doesn't seem to be read by nydus0snapshotter despite documentation
+# 	mkdir -p ~/.docker
+# 	cat << EOF | tee ~/.docker/config.json
+# {
+# 	"quay.io/kata-containers/confidential-containers-auth": {
+# 		"quay.io": {
+# 			"auth": "$REGISTRY_CREDENTIAL_ENCODED"
+# 		}
+# 	}
+# }
+# EOF
 	local container_config="${FIXTURES_DIR}/container-config_authenticated.yaml"
 
 	setup_credentials_files "quay.io/kata-containers/confidential-containers-auth" 
 
 	create_test_pod
-	
+
 	assert_container "${container_config}"
+	# rm ~/.docker/config.json
 }
 
 @test "$test_tag Test cannot pull an image from an authenticated registry with incorrect credentials" {
+# Docker config doesn't seem to be read by nydus0snapshotter despite documentation
+# 	mkdir -p ~/.docker
+# 	cat << EOF | tee ~/.docker/config.json
+# {
+#   "auths": {
+#     "quay.io/kata-containers/confidential-containers-auth": {
+#       "auth": "incorrectCredentials",
+#       "email": ""
+#     }
+#   }
+# }
+# EOF
+
 	local container_config="${FIXTURES_DIR}/container-config_authenticated.yaml"
 
-	REGISTRY_CREDENTIAL_ENCODED="QXJhbmRvbXF1YXl0ZXN0YWNjb3VudHRoYXRkb2VzbnRleGlzdDpwYXNzd29yZAo=" setup_credentials_files "quay.io/kata-containers/confidential-containers-auth"
+	# Set up incorrect credentials
+	# Note - because we are currently exporting them here to be used in `crictl_create_cc_container` if this test runs before
+	# the correct credentials one it will override the secret we pass in and fail
+	export REGISTRY_CREDENTIAL_ENCODED="QXJhbmRvbXF1YXl0ZXN0YWNjb3VudHRoYXRkb2VzbnRleGlzdDpwYXNzd29yZAo="
+	setup_credentials_files "quay.io/kata-containers/confidential-containers-auth"
 
 	create_test_pod
 
 	assert_container_fail "$container_config"
-	assert_logs_contain 'failed to pull manifest Authentication failure'
+	assert_logs_contain 'failed to resolve reference \\"quay.io/kata-containers/confidential-containers-auth:test\\": failed to authorize: failed to fetch oauth token: unexpected status: 401 UNAUTHORIZED'
+	# rm ~/.docker/config.json
 }
 
 @test "$test_tag Test cannot pull an image from an authenticated registry without credentials" {
 	local container_config="${FIXTURES_DIR}/container-config_authenticated.yaml"
 
+	# Set no credentials in the `crictl_create_cc_container`
+	export REGISTRY_CREDENTIAL_ENCODED=""
 	create_test_pod
 
 	assert_container_fail "$container_config"
-	assert_logs_contain 'failed to pull manifest Not authorized'
+    assert_logs_contain 'failed to resolve reference \\"quay.io/kata-containers/confidential-containers-auth:test\\": pulling from host quay.io failed with status code \[manifests test\]: 401 UNAUTHORIZED'
 }
 
 teardown() {
@@ -145,4 +181,9 @@ teardown() {
 	echo "-- Kata logs:"
 	# Note - with image-rs we hit more that the default 1000 lines of logs
 	sudo journalctl -xe -t kata --since "$test_start_time" -n 100000 
+
+		# Print the logs and cleanup resources.
+	echo "-- containerd logs:"
+	# Note - with image-rs we hit more that the default 1000 lines of logs
+	sudo journalctl -xe -t containerd --since "$test_start_time" -n 100000 
 }
diff --git a/integration/containerd/confidential/lib.sh b/integration/containerd/confidential/lib.sh
@@ -141,8 +141,13 @@ crictl_create_cc_container() {
 		return 1
 	fi
 
+	local auth_string=""
+	if [ -n "$REGISTRY_CREDENTIAL_ENCODED" ]; then
+		auth_string="--auth $REGISTRY_CREDENTIAL_ENCODED"
+	fi
+
 	pod_id=$(sudo crictl pods --name ${pod_name} -q)
-	container_id=$(sudo crictl create -with-pull "${pod_id}" \
+	container_id=$(sudo crictl create ${auth_string} --with-pull "${pod_id}" \
 		"${container_config}" "${pod_config}")
 
 	if [ -z "$container_id" ]; then

diff --git a/integration/containerd/confidential/run_tests.sh b/integration/containerd/confidential/run_tests.sh
@@ -12,6 +12,13 @@ cidir="${script_dir}/../../../.ci"
 source "${cidir}/lib.sh"
 
 main() {
+	# Ensure nydus is installed and configured
+	if [ "${IMAGE_OFFLOAD_TO_GUEST}" = "yes" ]; then
+		info "Install nydus-snapshotter"
+		bash -f "${cidir}/install_nydus_snapshotter.sh"
+		bash -f "${cidir}/containerd_nydus_setup.sh"
+	fi
+
 	# Ensure bats is installed.
 	${cidir}/install_bats.sh >/dev/null
 	bats ${script_dir}/agent_image.bats \

diff --git a/integration/containerd/confidential/tests_common.sh b/integration/containerd/confidential/tests_common.sh
@@ -32,9 +32,6 @@ setup_common() {
 	saved_kernel_params=$(get_kernel_params)
 	export saved_kernel_params
 
-	echo "Enable image service offload"
-	switch_image_service_offload on
-
 	# On CI mode we only want to enable the agent debug for the case of
 	# the test failure to obtain logs.
 	if [ "${CI:-}" == "true" ]; then
@@ -84,7 +81,6 @@ teardown_common() {
 	[ -n "$saved_kernel_params" ] && \
 		add_kernel_params "$saved_kernel_params"
 
-	switch_image_service_offload off
 	disable_full_debug
 
 	# Restore containerd to pre-test state.