Skip to content

Commit 4328f87

Browse files
adodon2goadodon2go
committed
ci(nightly): add prometheus kind test
Signed-off-by: Alexei Dodon <[email protected]>
1 parent 7ce5a74 commit 4328f87

File tree

3 files changed

+156
-0
lines changed

3 files changed

+156
-0
lines changed

‎.github/workflows/kind.yaml

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: 'Kind setup'
2+
on:
3+
push:
4+
branches:
5+
- main
6+
pull_request:
7+
branches:
8+
- main
9+
10+
permissions: read-all
11+
12+
jobs:
13+
kind-setup:
14+
name: Prometheus setup
15+
runs-on: ubuntu-latest-16-cores
16+
steps:
17+
- uses: actions/checkout@v4
18+
- uses: ./.github/actions/clean-runner
19+
- uses: actions/setup-go@v3
20+
with:
21+
go-version: 1.20.x
22+
- name: Install dependencies
23+
run: |
24+
cd $GITHUB_WORKSPACE
25+
go mod download
26+
sudo apt-get update
27+
sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
28+
# install skopeo
29+
git clone -b v1.12.0 https://github.com/containers/skopeo.git
30+
cd skopeo
31+
make bin/skopeo
32+
sudo cp bin/skopeo /usr/bin
33+
skopeo -v
34+
- name: Log in to GitHub Docker Registry
35+
uses: docker/login-action@v3
36+
with:
37+
registry: ghcr.io
38+
username: ${{ github.actor }}
39+
password: ${{ github.token }}
40+
- name: Run tests
41+
run: |
42+
./examples/kind/kind-ci.sh
43+

‎examples/kind/kind-ci.sh

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/sh
2+
set -o errexit
3+
4+
# Reference: https://kind.sigs.k8s.io/docs/user/local-registry/
5+
6+
# set no_proxy if applicable
7+
if [ ! -z "${no_proxy}" ]; then
8+
echo "Updating no_proxy env var";
9+
export no_proxy=${no_proxy},kind-registry;
10+
export NO_PROXY=${no_proxy};
11+
fi
12+
13+
# create registry container unless it already exists
14+
reg_name='kind-registry'
15+
reg_port='5001'
16+
if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
17+
docker run \
18+
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
19+
ghcr.io/project-zot/zot-minimal-linux-amd64:latest
20+
fi
21+
22+
CLUSTER_NAME=kind
23+
## Delete the cluster if it already exist
24+
kind get clusters | grep ${CLUSTER_NAME} && kind delete cluster --name ${CLUSTER_NAME}
25+
26+
# create a cluster with the local registry enabled in containerd
27+
cat <<EOF | kind create cluster --config=-
28+
kind: Cluster
29+
apiVersion: kind.x-k8s.io/v1alpha4
30+
containerdConfigPatches:
31+
- |-
32+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
33+
endpoint = ["http://${reg_name}:5000"]
34+
EOF
35+
36+
# connect the registry to the cluster network if not already connected
37+
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
38+
docker network connect "kind" "${reg_name}"
39+
fi
40+
41+
# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
42+
#
43+
# document the local registry
44+
cat <<EOF | kubectl apply -f -
45+
apiVersion: v1
46+
kind: ConfigMap
47+
metadata:
48+
name: local-registry-hosting
49+
namespace: kube-public
50+
data:
51+
localRegistryHosting.v1: |
52+
host: "localhost:${reg_port}"
53+
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
54+
EOF
55+
56+
## Deploy prometheus operator
57+
kubectl create -f examples/metrics/kubernetes/prometheus/bundle.yaml
58+
59+
## Deploy the Kubernetes objects for RBAC, prometheus CRD and deploy the service
60+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_rbac.yaml
61+
kubectl apply -f examples/metrics/kubernetes/prometheus/prometheus.yaml
62+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_service.yaml
63+
64+
make oci-image
65+
# copy the image
66+
skopeo copy --format=oci --dest-tls-verify=false oci:oci docker://localhost:5001/zot-build:latest
67+
68+
# deploy the image
69+
kubectl apply -f examples/metrics/kubernetes/zot-extended/deployment.yaml
70+
kubectl patch deployment/zot-extended --patch-file examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
71+
kubectl apply -f examples/metrics/kubernetes/zot-extended/service.yaml
72+
kubectl apply -f examples/metrics/kubernetes/zot-extended/servicemonitor.yaml
73+
74+
# check for availability
75+
echo "Waiting for deployment/zot-extended to be ready ..."
76+
kubectl wait deployment -n default zot-extended --for condition=Available=True --timeout=90s
77+
kubectl wait deployment -n default prometheus-operator --for condition=Available=True --timeout=90s
78+
79+
kubectl port-forward svc/prometheus 9090 --address='0.0.0.0' &
80+
echo "Kind cluster status before sleep:"
81+
kubectl get pods -A
82+
# Put enough amount of time for prometheus scraping take place
83+
sleep 40
84+
echo "Kind cluster status:"
85+
kubectl get pods -A
86+
echo "zot-extended logs:"
87+
kubectl logs -l app=zot-extended --tail=-1
88+
89+
containername=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].metric.container'`
90+
echo "containername=${containername}"
91+
if [ "${containername}" != '"zot-extended"' ]; then
92+
exit 1
93+
fi
94+
95+
containerup=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].value[1]'`
96+
echo "containerup=${containerup}"
97+
if [ "${containerup}" != '"1"' ]; then
98+
exit 1
99+
fi
100+
101+
zotinfo=`curl -s http://localhost:9090/api/v1/query?query=zot_info | jq '.data.result[].value[1]'`
102+
echo "zotinfo=${zotinfo}"
103+
if [ "${zotinfo}" != '"0"' ]; then
104+
exit 1
105+
fi

‎examples/metrics/kubernetes/zot-extended/patch-deployment.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
spec:
2+
template:
3+
spec:
4+
containers:
5+
- name: zot-extended
6+
image: localhost:5001/zot-build:latest
7+
command: ["/usr/local/bin/zot-linux-amd64"]
8+

0 commit comments

Comments
 (0)