Skip to content

Commit

Permalink
ci: Add test cases for CoCo image pulling without forked containerd
Browse files Browse the repository at this point in the history 
Add test cases for CoCo image pulling without forked containerd,
including:
1) sharing image on the host with dm-verity
2) pulling image in the guest

Fixes: kata-containers#5763

Depends-on:
github.com/kata-containers/kata-containers#7676
github.com/kata-containers/kata-containers#7688

Signed-off-by: ChengyuZhu6 <[email protected]>
  • Loading branch information
ChengyuZhu6 committed Sep 6, 2023
1 parent 6ab2a91 commit cbe3231
Show file tree
Hide file tree
Showing 2 changed files with 163 additions and 101 deletions.
239 changes: 142 additions & 97 deletions integration/confidential/lib.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
NYDUS_SNAPSHOTTER_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config.toml"
CONTAINERD_CONFIG="/etc/containerd/config.toml"

# Toggle between true and false the service_offload configuration of
# the Kata agent.
Expand All @@ -29,18 +31,18 @@ switch_image_service_offload() {
load_runtime_config_path

case "$1" in
"on")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
"$RUNTIME_CONFIG_PATH"
;;
"off")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
"$RUNTIME_CONFIG_PATH"

;;
*)
die "Unknown option '$1'"
;;
"on")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
"$RUNTIME_CONFIG_PATH"
;;
"off")
sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
"$RUNTIME_CONFIG_PATH"

;;
*)
die "Unknown option '$1'"
;;
esac
}

Expand All @@ -66,13 +68,13 @@ switch_measured_rootfs_verity_scheme() {
load_runtime_config_path

case "$1" in
"dm-verity"|"none")
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
"$RUNTIME_CONFIG_PATH"
;;
*)
die "Unknown option '$1'"
;;
"dm-verity" | "none")
sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
"$RUNTIME_CONFIG_PATH"
;;
*)
die "Unknown option '$1'"
;;
esac
}

Expand Down Expand Up @@ -110,8 +112,8 @@ add_kernel_params() {
get_kernel_params() {
load_runtime_config_path

local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
"$RUNTIME_CONFIG_PATH")
local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
"$RUNTIME_CONFIG_PATH")
echo "$kernel_params"
}

Expand Down Expand Up @@ -203,13 +205,13 @@ configure_cc_containerd() {
# restart containerd because it might be in an inconsistent state here.
sudo systemctl stop containerd
sleep 5
[ -n "$saved_containerd_conf_file" ] && \
[ -n "$saved_containerd_conf_file" ] &&
sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
sudo systemctl start containerd
waitForProcess 30 5 "sudo crictl info >/dev/null"

# Ensure the cc CRI handler is set.
local cri_handler=$(sudo crictl info | \
local cri_handler=$(sudo crictl info |
jq '.config.containerd.runtimes.kata.cri_handler')
if [[ ! "$cri_handler" =~ cc ]]; then
sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
Expand All @@ -219,8 +221,8 @@ configure_cc_containerd() {
if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
echo " [plugins.cri.cni]
# conf_dir is the directory in which the admin places a CNI conf.
conf_dir = \"/etc/cni/net.d\"" | \
sudo tee -a "$containerd_conf_file"
conf_dir = \"/etc/cni/net.d\"" |
sudo tee -a "$containerd_conf_file"
fi

sudo systemctl restart containerd
Expand Down Expand Up @@ -286,38 +288,36 @@ setup_cosign_signatures_files() {

# Set-up required files in guest image
case "${AA_KBC:-}" in
"offline_fs_kbc")
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
;;
"cc_kbc")
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
# by the service, as well as the one configured in the Kata Containers rootfs.

CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
;;
*)
;;
"offline_fs_kbc")
add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
;;
"cc_kbc")
# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
# by the service, as well as the one configured in the Kata Containers rootfs.

CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
;;
*) ;;
esac
}

setup_signature_files() {
case "${AA_KBC:-}" in
"offline_fs_kbc")
setup_offline_fs_kbc_signature_files_in_guest
;;
"cc_kbc")
setup_cc_kbc_signature_files_in_guest
;;
*)
;;
"offline_fs_kbc")
setup_offline_fs_kbc_signature_files_in_guest
;;
"cc_kbc")
setup_cc_kbc_signature_files_in_guest
;;
*) ;;
esac
}

# In case the tests run behind a firewall where images needed to be fetched
# through a proxy.
# through a proxy.
# Note: With measured rootfs enabled, we can not set proxy through
# agent config file.
setup_proxy() {
Expand Down Expand Up @@ -349,8 +349,8 @@ setup_credentials_files() {

dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)"
dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}"
cp_to_guest_img "etc" "${dest_file}"
}

Expand All @@ -366,77 +366,122 @@ KBS_DB="${KBS_DB:-simple_kbs}"

# Run the simple-kbs
simple_kbs_run() {
# Retrieve simple-kbs repo and tag from versions.yaml
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")

# Cleanup and create installation directory
esudo rm -rf "${SIMPLE_KBS_DIR}"
mkdir -p "${SIMPLE_KBS_DIR}"
pushd "${SIMPLE_KBS_DIR}"

# Clone and run
git clone "${simple_kbs_url}" --branch main
pushd simple-kbs

# Checkout, build and start
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
esudo docker-compose build
esudo docker-compose up -d

# Wait for simple-kbs to start
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
popd
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

# Confirm connection to the database is possible
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
popd
# Retrieve simple-kbs repo and tag from versions.yaml
local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")

# Cleanup and create installation directory
esudo rm -rf "${SIMPLE_KBS_DIR}"
mkdir -p "${SIMPLE_KBS_DIR}"
pushd "${SIMPLE_KBS_DIR}"

# Clone and run
git clone "${simple_kbs_url}" --branch main
pushd simple-kbs

# Checkout, build and start
git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
esudo docker-compose build
esudo docker-compose up -d

# Wait for simple-kbs to start
waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
popd

# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

# Confirm connection to the database is possible
waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
popd
}

# Stop simple-kbs and database containers
simple_kbs_stop() {
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
}

# Delete all test inserted data in the simple-kbs
simple_kbs_delete_data() {
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

# Delete all data with 'id = 10'
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
# Delete all data with 'id = 10'
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
DELETE FROM secrets WHERE id = 10;
DELETE FROM policy WHERE id = 10;
EOF
}

# Get the ip of the simple-kbs database docker container
simple_kbs_get_db_ip() {
esudo docker network inspect simple-kbs_default \
| jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \
| sed "s|/.*$||g"
esudo docker network inspect simple-kbs_default |
jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' |
sed "s|/.*$||g"
}

# Add key and keyset to database
# If measurement is provided, add policy with measurement to database
simple_kbs_add_key_to_db() {
local encryption_key="${1}"
local measurement="${2}"
# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

if [ -n "${measurement}" ]; then
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
local encryption_key="${1}"
local measurement="${2}"

# Get simple-kbs database container ip
local kbs_db_host=$(simple_kbs_get_db_ip)

if [ -n "${measurement}" ]; then
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', 10);
INSERT INTO policy VALUES (10, '["${measurement}"]', '[]', 0, 0, '[]', now(), NULL, 1);
EOF
else
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
else
mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', NULL);
EOF
fi
fi
}

###############################################################################

# remote-snapshotter

configure_remote_snapshotter() {
case "${SNAPSHOTTER:-}" in
"nydus")
configure_nydus_snapshotter
;;
*) ;;

esac
}
check_containerd_version() {
containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
if echo $containerd_version | grep -q "^v1.7"; then
return 1
else
return 0
fi
}
configure_containerd_for_nydus_snapshotter() {
if check_containerd_version; then
sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n disable_snapshot_annotations = false' "$CONTAINERD_CONFIG"
else
sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g; s/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
fi
}
remove_nydus_snapshotter_from_containerd() {
if check_containerd_version; then
sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;/disable_snapshot_annotations = false/d;}' "$CONTAINERD_CONFIG"
else
sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g; s/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
fi
}
restart_nydus_snapshotter() {
echo "Restart nydus snapshotter"
}

configure_nydus_snapshotter() {
echo "Configure nydus snapshotter"
sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" $NYDUS_SNAPSHOTTER_CONFIG
restart_nydus_snapshotter
}
25 changes: 21 additions & 4 deletions integration/kubernetes/confidential/agent_image.bats
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ if [ "$(uname -m)" != "x86_64" ]; then
fi

# Images used on the tests.
## Cosign
## Cosign
image_cosigned="quay.io/kata-containers/confidential-containers:cosign-signed${tag_suffix}"
image_cosigned_other="quay.io/kata-containers/confidential-containers:cosign-signed-key2"

Expand Down Expand Up @@ -70,7 +70,7 @@ new_pod_config() {
local image="$1"

local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config"
IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst <"$base_config" >"$new_config"
echo "$new_config"
}

Expand All @@ -90,7 +90,7 @@ setup() {
switch_image_service_offload on
clear_kernel_params
add_kernel_params "${original_kernel_params}"

setup_proxy
switch_measured_rootfs_verity_scheme none
}
Expand Down Expand Up @@ -183,7 +183,6 @@ assert_logs_contain() {
assert_logs_contain 'Validate image failed: \[PublicKeyVerifier { key: ECDSA_P256_SHA256_ASN1'
}


@test "$test_tag Test pull an unencrypted unsigned image from an authenticated registry with correct credentials" {
if [ "${AA_KBC}" = "offline_fs_kbc" ]; then
setup_credentials_files "quay.io/kata-containers/confidential-containers-auth"
Expand Down Expand Up @@ -223,6 +222,24 @@ assert_logs_contain() {
assert_logs_contain 'failed to pull manifest Not authorized'
}

@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
if [ "${SNAPSHOTTER}" = "nydus" ]; then
EXPORT_MODE="image_block_with_verity" configure_remote_snapshotter
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
echo $pod_config
create_test_pod
fi
}

@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
if [ "${SNAPSHOTTER}" = "nydus" ]; then
EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter
pod_config="$(new_pod_config "$image_unsigned_unprotected")"
echo $pod_config
create_test_pod
fi
}

teardown() {
# Print the logs and cleanup resources.
echo "-- Kata logs:"
Expand Down

0 comments on commit cbe3231

Please sign in to comment.