Skip to content

Commit 263ab4c

Browse files
adodon2goadodon2go
committed
ci(nightly): add prometheus kind test
Signed-off-by: Alexei Dodon <[email protected]>
1 parent ff16e4c commit 263ab4c

File tree

5 files changed

+189
-4
lines changed

5 files changed

+189
-4
lines changed

‎.github/workflows/nightly.yaml

Lines changed: 35 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ permissions: read-all
1414
jobs:
1515
dedupe:
1616
name: Dedupe/restore blobs
17-
runs-on: ubuntu-latest-16-cores
17+
runs-on: ubuntu-latest-8-cores
1818
steps:
1919
- uses: actions/checkout@v4
2020
- uses: ./.github/actions/clean-runner
@@ -62,7 +62,7 @@ jobs:
6262

6363
sync:
6464
name: Sync harness
65-
runs-on: ubuntu-latest-16-cores
65+
runs-on: ubuntu-latest-8-cores
6666
steps:
6767
- name: Check out source code
6868
uses: actions/checkout@v4
@@ -92,6 +92,7 @@ jobs:
9292
- uses: ./.github/actions/setup-localstack
9393

9494
- name: Run zb
95+
timeout-minutes: 240
9596
id: bench
9697
run: |
9798
make binary
@@ -129,6 +130,7 @@ jobs:
129130
- uses: ./.github/actions/setup-localstack
130131

131132
- name: Run zb
133+
timeout-minutes: 240
132134
id: bench
133135
run: |
134136
make binary
@@ -155,7 +157,7 @@ jobs:
155157

156158
docker-image:
157159
name: Build docker image (for users still using Docker environments)
158-
runs-on: ubuntu-latest-16-cores
160+
runs-on: ubuntu-latest-4-cores
159161
steps:
160162
- uses: actions/checkout@v4
161163
- uses: ./.github/actions/clean-runner
@@ -164,3 +166,33 @@ jobs:
164166
- name: Build image
165167
run: |
166168
make docker-image
169+
170+
kind-setup:
171+
name: Prometheus setup
172+
runs-on: ubuntu-latest-8-cores
173+
steps:
174+
- uses: actions/checkout@v4
175+
- uses: actions/setup-go@v3
176+
with:
177+
go-version: 1.20.x
178+
- name: Install dependencies
179+
run: |
180+
cd $GITHUB_WORKSPACE
181+
go mod download
182+
sudo apt-get update
183+
sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
184+
# install skopeo
185+
git clone -b v1.12.0 https://github.com/containers/skopeo.git
186+
cd skopeo
187+
make bin/skopeo
188+
sudo cp bin/skopeo /usr/bin
189+
skopeo -v
190+
- name: Log in to GitHub Docker Registry
191+
uses: docker/login-action@v3
192+
with:
193+
registry: ghcr.io
194+
username: ${{ github.actor }}
195+
password: ${{ github.token }}
196+
- name: Run tests
197+
run: |
198+
./examples/kind/kind-ci.sh

‎examples/kind/kind-ci.sh

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/sh
2+
set -o errexit
3+
4+
# Reference: https://kind.sigs.k8s.io/docs/user/local-registry/
5+
6+
# set no_proxy if applicable
7+
if [ ! -z "${no_proxy}" ]; then
8+
echo "Updating no_proxy env var";
9+
export no_proxy=${no_proxy},kind-registry;
10+
export NO_PROXY=${no_proxy};
11+
fi
12+
13+
# create registry container unless it already exists
14+
reg_name='kind-registry'
15+
reg_port='5001'
16+
if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
17+
docker run \
18+
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
19+
ghcr.io/project-zot/zot-minimal-linux-amd64:latest
20+
fi
21+
22+
CLUSTER_NAME=kind
23+
## Delete the cluster if it already exist
24+
kind get clusters | grep ${CLUSTER_NAME} && kind delete cluster --name ${CLUSTER_NAME}
25+
26+
# create a cluster with the local registry enabled in containerd
27+
cat <<EOF | kind create cluster --config=-
28+
kind: Cluster
29+
apiVersion: kind.x-k8s.io/v1alpha4
30+
containerdConfigPatches:
31+
- |-
32+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
33+
endpoint = ["http://${reg_name}:5000"]
34+
EOF
35+
36+
# connect the registry to the cluster network if not already connected
37+
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
38+
docker network connect "kind" "${reg_name}"
39+
fi
40+
41+
# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
42+
#
43+
# document the local registry
44+
cat <<EOF | kubectl apply -f -
45+
apiVersion: v1
46+
kind: ConfigMap
47+
metadata:
48+
name: local-registry-hosting
49+
namespace: kube-public
50+
data:
51+
localRegistryHosting.v1: |
52+
host: "localhost:${reg_port}"
53+
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
54+
EOF
55+
56+
## Deploy prometheus operator
57+
kubectl create -f examples/metrics/kubernetes/prometheus/bundle.yaml
58+
59+
## Deploy the Kubernetes objects for RBAC, prometheus CRD and deploy the service
60+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_rbac.yaml
61+
kubectl apply -f examples/metrics/kubernetes/prometheus/prometheus.yaml
62+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_service.yaml
63+
64+
make oci-image
65+
# copy the image
66+
skopeo copy --format=oci --dest-tls-verify=false oci:oci docker://localhost:5001/zot-build:latest
67+
68+
# deploy the image
69+
kubectl apply -f examples/metrics/kubernetes/zot-extended/deployment.yaml
70+
kubectl patch deployment/zot-extended --patch-file examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
71+
kubectl apply -f examples/metrics/kubernetes/zot-extended/service.yaml
72+
kubectl apply -f examples/metrics/kubernetes/zot-extended/servicemonitor.yaml
73+
74+
# check for availability
75+
echo "Waiting for deployment/zot-extended to be ready ..."
76+
kubectl wait deployment -n default zot-extended --for condition=Available=True --timeout=90s
77+
kubectl wait deployment -n default prometheus-operator --for condition=Available=True --timeout=90s
78+
79+
kubectl port-forward svc/prometheus 9090 --address='0.0.0.0' &
80+
echo "Kind cluster status before sleep:"
81+
kubectl get pods -A
82+
# Put enough amount of time for prometheus scraping take place
83+
sleep 90
84+
echo "Kind cluster status:"
85+
kubectl get pods -A
86+
echo "zot-extended logs:"
87+
kubectl logs -l app=zot-extended --tail=-1
88+
89+
containername=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].metric.container'`
90+
echo "containername=${containername}"
91+
if [ "${containername}" != '"zot-extended"' ]; then
92+
exit 1
93+
fi
94+
95+
containerup=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].value[1]'`
96+
echo "containerup=${containerup}"
97+
if [ "${containerup}" != '"1"' ]; then
98+
exit 1
99+
fi
100+
101+
zotinfo=`curl -s http://localhost:9090/api/v1/query?query=zot_info | jq '.data.result[].value[1]'`
102+
echo "zotinfo=${zotinfo}"
103+
if [ "${zotinfo}" != '"0"' ]; then
104+
exit 1
105+
fi

‎examples/metrics/kubernetes/zot-extended/deployment.yaml

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ spec:
3535
items:
3636
- key: zot_config.json
3737
path: config.json
38+
- key: htpasswd
39+
path: htpasswd
3840
---
3941
apiVersion: v1
4042
kind: ConfigMap
@@ -49,7 +51,25 @@ data:
4951
},
5052
"http": {
5153
"address": "0.0.0.0",
52-
"port": "5000"
54+
"port": "5000",
55+
"auth": {
56+
"htpasswd": {
57+
"path": "/zot-config/htpasswd"
58+
}
59+
},
60+
"accessControl": {
61+
"metrics":{
62+
"users": ["metrics"]
63+
},
64+
"repositories": {
65+
"**": {
66+
"anonymousPolicy": [
67+
"read"
68+
],
69+
"defaultPolicy": ["read","create"]
70+
}
71+
}
72+
}
5373
},
5474
"log": {
5575
"level": "debug"
@@ -91,3 +111,7 @@ data:
91111
}
92112
}
93113
}
114+
# Example htpasswd with 'test:test' & 'metrics:metrics' user:pass pairs
115+
htpasswd: |-
116+
test:$2y$05$.jSWenVyzEK3em/Gfr0AG.WRSAIdi4nxqW9h27xK7WCw24wmQH/1m
117+
metrics:$2y$05$4yBka/ZTKgXhvCMb48BnyOZqj/DrKT1sGPZLAg5RbobQ0CQCJHmTO

‎examples/metrics/kubernetes/zot-extended/patch-deployment.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
spec:
2+
template:
3+
spec:
4+
containers:
5+
- name: zot-extended
6+
image: localhost:5001/zot-build:latest
7+
command: ["/usr/local/bin/zot-linux-amd64"]
8+

‎examples/metrics/kubernetes/zot-extended/servicemonitor.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,23 @@ spec:
99
- interval: 10s
1010
port: zot-extended
1111
scrapeTimeout: 5s
12+
basicAuth:
13+
password:
14+
name: basic-auth
15+
key: password
16+
username:
17+
name: basic-auth
18+
key: user
1219
selector:
1320
matchLabels:
1421
app: zot-extended
22+
---
23+
apiVersion: v1
24+
kind: Secret
25+
metadata:
26+
name: basic-auth
27+
data:
28+
password: bWV0cmljcw== # metrics
29+
user: bWV0cmljcw== # metrics
30+
type: Opaque
1531

0 commit comments

Comments
 (0)