Skip to content

Commit 87f3c62

Browse files
adodon2goadodon2go
committed
ci(nightly): add prometheus kind test
Signed-off-by: Alexei Dodon <[email protected]>
1 parent b2a9239 commit 87f3c62

File tree

5 files changed

+191
-9
lines changed

5 files changed

+191
-9
lines changed

‎.github/workflows/nightly.yaml

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,9 @@ permissions: read-all
1414
jobs:
1515
dedupe:
1616
name: Dedupe/restore blobs
17-
runs-on: ubuntu-latest-16-cores
17+
runs-on: ubuntu-latest-8-cores
1818
steps:
1919
- uses: actions/checkout@v4
20-
- uses: ./.github/actions/clean-runner
2120
- uses: actions/setup-go@v3
2221
with:
2322
go-version: 1.20.x
@@ -62,11 +61,10 @@ jobs:
6261

6362
sync:
6463
name: Sync harness
65-
runs-on: ubuntu-latest-16-cores
64+
runs-on: ubuntu-latest-8-cores
6665
steps:
6766
- name: Check out source code
6867
uses: actions/checkout@v4
69-
- uses: ./.github/actions/clean-runner
7068
- uses: actions/setup-go@v3
7169
with:
7270
go-version: 1.20.x
@@ -84,14 +82,14 @@ jobs:
8482
runs-on: ubuntu-latest-16-cores
8583
steps:
8684
- uses: actions/checkout@v4
87-
- uses: ./.github/actions/clean-runner
8885
- uses: actions/setup-go@v4
8986
with:
9087
cache: false
9188
go-version: 1.20.x
9289
- uses: ./.github/actions/setup-localstack
9390

9491
- name: Run zb
92+
timeout-minutes: 240
9593
id: bench
9694
run: |
9795
make binary
@@ -121,14 +119,14 @@ jobs:
121119
runs-on: ubuntu-latest-16-cores
122120
steps:
123121
- uses: actions/checkout@v4
124-
- uses: ./.github/actions/clean-runner
125122
- uses: actions/setup-go@v4
126123
with:
127124
cache: false
128125
go-version: 1.20.x
129126
- uses: ./.github/actions/setup-localstack
130127

131128
- name: Run zb
129+
timeout-minutes: 240
132130
id: bench
133131
run: |
134132
make binary
@@ -155,12 +153,41 @@ jobs:
155153

156154
docker-image:
157155
name: Build docker image (for users still using Docker environments)
158-
runs-on: ubuntu-latest-16-cores
156+
runs-on: ubuntu-latest-4-cores
159157
steps:
160158
- uses: actions/checkout@v4
161-
- uses: ./.github/actions/clean-runner
162159
- name: Check out source code
163160
uses: actions/checkout@v4
164161
- name: Build image
165162
run: |
166163
make docker-image
164+
165+
kind-setup:
166+
name: Prometheus setup
167+
runs-on: ubuntu-latest-8-cores
168+
steps:
169+
- uses: actions/checkout@v4
170+
- uses: actions/setup-go@v3
171+
with:
172+
go-version: 1.20.x
173+
- name: Install dependencies
174+
run: |
175+
cd $GITHUB_WORKSPACE
176+
go mod download
177+
sudo apt-get update
178+
sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
179+
# install skopeo
180+
git clone -b v1.12.0 https://github.com/containers/skopeo.git
181+
cd skopeo
182+
make bin/skopeo
183+
sudo cp bin/skopeo /usr/bin
184+
skopeo -v
185+
- name: Log in to GitHub Docker Registry
186+
uses: docker/login-action@v3
187+
with:
188+
registry: ghcr.io
189+
username: ${{ github.actor }}
190+
password: ${{ github.token }}
191+
- name: Run tests
192+
run: |
193+
./examples/kind/kind-ci.sh

‎examples/kind/kind-ci.sh

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/bin/sh
2+
set -o errexit
3+
4+
# Reference: https://kind.sigs.k8s.io/docs/user/local-registry/
5+
6+
# set no_proxy if applicable
7+
if [ ! -z "${no_proxy}" ]; then
8+
echo "Updating no_proxy env var";
9+
export no_proxy=${no_proxy},kind-registry;
10+
export NO_PROXY=${no_proxy};
11+
fi
12+
13+
# create registry container unless it already exists
14+
reg_name='kind-registry'
15+
reg_port='5001'
16+
if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
17+
docker run \
18+
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
19+
ghcr.io/project-zot/zot-minimal-linux-amd64:latest
20+
fi
21+
22+
CLUSTER_NAME=kind
23+
## Delete the cluster if it already exist
24+
kind get clusters | grep ${CLUSTER_NAME} && kind delete cluster --name ${CLUSTER_NAME}
25+
26+
# create a cluster with the local registry enabled in containerd
27+
cat <<EOF | kind create cluster --config=-
28+
kind: Cluster
29+
apiVersion: kind.x-k8s.io/v1alpha4
30+
containerdConfigPatches:
31+
- |-
32+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
33+
endpoint = ["http://${reg_name}:5000"]
34+
EOF
35+
36+
# connect the registry to the cluster network if not already connected
37+
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
38+
docker network connect "kind" "${reg_name}"
39+
fi
40+
41+
# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
42+
#
43+
# document the local registry
44+
cat <<EOF | kubectl apply -f -
45+
apiVersion: v1
46+
kind: ConfigMap
47+
metadata:
48+
name: local-registry-hosting
49+
namespace: kube-public
50+
data:
51+
localRegistryHosting.v1: |
52+
host: "localhost:${reg_port}"
53+
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
54+
EOF
55+
56+
## Deploy prometheus operator
57+
kubectl create -f examples/metrics/kubernetes/prometheus/bundle.yaml
58+
59+
## Deploy the Kubernetes objects for RBAC, prometheus CRD and deploy the service
60+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_rbac.yaml
61+
kubectl apply -f examples/metrics/kubernetes/prometheus/prometheus.yaml
62+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_service.yaml
63+
64+
make oci-image
65+
# copy the image
66+
COMMIT_HASH=$(git describe --always --tags --long)
67+
echo "deploy zot-build:${COMMIT_HASH} image to local registry"
68+
skopeo copy --format=oci --dest-tls-verify=false oci:oci docker://localhost:5001/zot-build:${COMMIT_HASH}
69+
70+
# deploy the image
71+
kubectl apply -f examples/metrics/kubernetes/zot-extended/deployment.yaml
72+
kubectl patch deployment/zot-extended --patch-file examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
73+
kubectl set image deployment/zot-extended zot-extended=localhost:5001/zot-build:${COMMIT_HASH}
74+
kubectl apply -f examples/metrics/kubernetes/zot-extended/service.yaml
75+
kubectl apply -f examples/metrics/kubernetes/zot-extended/servicemonitor.yaml
76+
77+
# check for availability
78+
echo "Waiting for deployment/zot-extended to be ready ..."
79+
kubectl wait deployment -n default zot-extended --for condition=Available=True --timeout=90s
80+
kubectl wait deployment -n default prometheus-operator --for condition=Available=True --timeout=90s
81+
82+
kubectl port-forward svc/prometheus 9090 --address='0.0.0.0' &
83+
echo "Kind cluster status before sleep:"
84+
kubectl get pods -A
85+
# Put enough amount of time for prometheus scraping take place
86+
sleep 90
87+
echo "Kind cluster status:"
88+
kubectl get pods -A
89+
echo "zot-extended logs:"
90+
kubectl logs -l app=zot-extended --tail=-1
91+
92+
containername=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].metric.container'`
93+
echo "containername=${containername}"
94+
if [ "${containername}" != '"zot-extended"' ]; then
95+
exit 1
96+
fi
97+
98+
containerup=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].value[1]'`
99+
echo "containerup=${containerup}"
100+
if [ "${containerup}" != '"1"' ]; then
101+
exit 1
102+
fi
103+
104+
zotinfo=`curl -s http://localhost:9090/api/v1/query?query=zot_info | jq '.data.result[].value[1]'`
105+
echo "zotinfo=${zotinfo}"
106+
if [ "${zotinfo}" != '"0"' ]; then
107+
exit 1
108+
fi

‎examples/metrics/kubernetes/zot-extended/deployment.yaml

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ spec:
3535
items:
3636
- key: zot_config.json
3737
path: config.json
38+
- key: htpasswd
39+
path: htpasswd
3840
---
3941
apiVersion: v1
4042
kind: ConfigMap
@@ -49,7 +51,25 @@ data:
4951
},
5052
"http": {
5153
"address": "0.0.0.0",
52-
"port": "5000"
54+
"port": "5000",
55+
"auth": {
56+
"htpasswd": {
57+
"path": "/zot-config/htpasswd"
58+
}
59+
},
60+
"accessControl": {
61+
"metrics":{
62+
"users": ["metrics"]
63+
},
64+
"repositories": {
65+
"**": {
66+
"anonymousPolicy": [
67+
"read"
68+
],
69+
"defaultPolicy": ["read","create"]
70+
}
71+
}
72+
}
5373
},
5474
"log": {
5575
"level": "debug"
@@ -91,3 +111,7 @@ data:
91111
}
92112
}
93113
}
114+
# Example htpasswd with 'test:test' & 'metrics:metrics' user:pass pairs
115+
htpasswd: |-
116+
test:$2y$05$.jSWenVyzEK3em/Gfr0AG.WRSAIdi4nxqW9h27xK7WCw24wmQH/1m
117+
metrics:$2y$05$4yBka/ZTKgXhvCMb48BnyOZqj/DrKT1sGPZLAg5RbobQ0CQCJHmTO

‎examples/metrics/kubernetes/zot-extended/patch-deployment.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
spec:
2+
template:
3+
spec:
4+
containers:
5+
- name: zot-extended
6+
command: ["/usr/local/bin/zot-linux-amd64"]
7+

‎examples/metrics/kubernetes/zot-extended/servicemonitor.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,23 @@ spec:
99
- interval: 10s
1010
port: zot-extended
1111
scrapeTimeout: 5s
12+
basicAuth:
13+
password:
14+
name: basic-auth
15+
key: password
16+
username:
17+
name: basic-auth
18+
key: user
1219
selector:
1320
matchLabels:
1421
app: zot-extended
22+
---
23+
apiVersion: v1
24+
kind: Secret
25+
metadata:
26+
name: basic-auth
27+
data:
28+
password: bWV0cmljcw== # metrics
29+
user: bWV0cmljcw== # metrics
30+
type: Opaque
1531

0 commit comments

Comments
 (0)