Skip to content

Commit 372f80b

Browse files
adodon2goadodon2go
committed
ci(nightly): add prometheus kind test
Signed-off-by: Alexei Dodon <[email protected]>
1 parent 7f6534a commit 372f80b

File tree

5 files changed

+127
-129
lines changed

5 files changed

+127
-129
lines changed

‎.github/workflows/nightly.yaml

Lines changed: 5 additions & 129 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: 'Nightly jobs'
22
on:
3-
schedule:
4-
- cron: '30 1 * * *'
3+
pull_request:
4+
branches: [main]
55
workflow_dispatch:
66

77
permissions: read-all
@@ -13,7 +13,7 @@ permissions: read-all
1313
# task scheduler will start a restore all blobs process at zot startup, after it finishes all blobs should be restored to their original state (have content)
1414
jobs:
1515
dedupe:
16-
name: Dedupe/restore blobs
16+
name: Kind Prometheus test
1717
runs-on: ubuntu-latest
1818
steps:
1919
- uses: actions/checkout@v4
@@ -24,8 +24,6 @@ jobs:
2424
- name: Install dependencies
2525
run: |
2626
cd $GITHUB_WORKSPACE
27-
go install github.com/swaggo/swag/cmd/[email protected]
28-
go mod download
2927
sudo apt-get update
3028
sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
3129
# install skopeo
@@ -40,127 +38,5 @@ jobs:
4038
registry: ghcr.io
4139
username: ${{ github.actor }}
4240
password: ${{ github.token }}
43-
- name: Install localstack
44-
run: |
45-
pip install --upgrade pyopenssl
46-
pip install localstack==2.3.1 awscli-local[ver1] # install LocalStack cli and awslocal
47-
docker pull ghcr.io/project-zot/ci-images/localstack:2.3.1 # Make sure to pull the latest version of the image
48-
localstack start -d # Start LocalStack in the background
49-
50-
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
51-
localstack wait -t 30 # to become ready before timing out
52-
echo "Startup complete"
53-
- name: Run blackbox nightly dedupe tests
54-
run: |
55-
# test restoring s3 blobs after cache is deleted
56-
# test deduping filesystem blobs after switching dedupe to enable
57-
make run-blackbox-dedupe-nightly
58-
env:
59-
AWS_ACCESS_KEY_ID: fake
60-
AWS_SECRET_ACCESS_KEY: fake
61-
- uses: ./.github/actions/teardown-localstack
62-
63-
sync:
64-
name: Sync harness
65-
runs-on: ubuntu-latest
66-
steps:
67-
- name: Check out source code
68-
uses: actions/checkout@v4
69-
- uses: ./.github/actions/clean-runner
70-
- uses: actions/setup-go@v3
71-
with:
72-
go-version: 1.20.x
73-
- name: Install dependencies
74-
run: |
75-
cd $GITHUB_WORKSPACE
76-
go install github.com/swaggo/swag/cmd/[email protected]
77-
go mod download
78-
- name: Run sync harness
79-
run: |
80-
make run-blackbox-sync-nightly
81-
82-
gc-referrers-stress-s3:
83-
name: GC(with referrers) on S3(localstack) with short interval
84-
runs-on: ubuntu-latest
85-
steps:
86-
- uses: actions/checkout@v4
87-
- uses: ./.github/actions/clean-runner
88-
- uses: actions/setup-go@v4
89-
with:
90-
cache: false
91-
go-version: 1.20.x
92-
- uses: ./.github/actions/setup-localstack
93-
94-
- name: Run zb
95-
id: bench
96-
run: |
97-
make binary
98-
make bench
99-
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-localstack.json &
100-
sleep 10
101-
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup
102-
103-
killall -r zot-*
104-
105-
# clean zot storage
106-
sudo rm -rf /tmp/zot
107-
env:
108-
AWS_ACCESS_KEY_ID: fake
109-
AWS_SECRET_ACCESS_KEY: fake
110-
continue-on-error: true
111-
112-
- name: Check on failures
113-
if: steps.bench.outcome != 'success'
114-
run: |
115-
cat /tmp/gc-referrers-bench-s3.log
116-
exit 1
117-
- uses: ./.github/actions/teardown-localstack
118-
119-
gc-stress-s3:
120-
name: GC(without referrers) on S3(localstack) with short interval
121-
runs-on: ubuntu-latest
122-
steps:
123-
- uses: actions/checkout@v4
124-
- uses: ./.github/actions/clean-runner
125-
- uses: actions/setup-go@v4
126-
with:
127-
cache: false
128-
go-version: 1.20.x
129-
- uses: ./.github/actions/setup-localstack
130-
131-
- name: Run zb
132-
id: bench
133-
run: |
134-
make binary
135-
make bench
136-
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-localstack.json &
137-
sleep 10
138-
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup
139-
140-
killall -r zot-*
141-
142-
# clean zot storage
143-
sudo rm -rf /tmp/zot
144-
env:
145-
AWS_ACCESS_KEY_ID: fake
146-
AWS_SECRET_ACCESS_KEY: fake
147-
continue-on-error: true
148-
149-
- name: Check on failures
150-
if: steps.bench.outcome != 'success'
151-
run: |
152-
cat /tmp/gc-bench-s3.log
153-
exit 1
154-
- uses: ./.github/actions/teardown-localstack
155-
156-
docker-image:
157-
name: Build docker image (for users still using Docker environments)
158-
runs-on: ubuntu-latest
159-
steps:
160-
- uses: actions/checkout@v4
161-
- uses: ./.github/actions/clean-runner
162-
- name: Check out source code
163-
uses: actions/checkout@v4
164-
- name: Build image
165-
run: |
166-
make docker-image
41+
- name: Run tests
42+
run: ./examples/kind/kind-ci.sh

‎.github/workflows/test.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ jobs:
3232
key: ${{ runner.os }}-go-mod-${{ hashFiles('**/go.sum') }}
3333
restore-keys: |
3434
${{ runner.os }}-go-mod-
35+
- name: Install RAMDISK
36+
run: |
37+
lscpu
38+
free --giga -h -t
39+
sudo mkdir -p /tmp/ramdisk
40+
sudo chmod 777 /tmp/ramdisk
41+
sudo mount -t tmpfs -o size=6G tmpfs /tmp/ramdisk
3542
- name: Install go dependencies
3643
if: steps.cache-go-dependencies.outputs.cache-hit != 'true'
3744
run: |
@@ -44,6 +51,7 @@ jobs:
4451
cd $GITHUB_WORKSPACE
4552
make test-minimal
4653
env:
54+
TMPDIR: /tmp/ramdisk
4755
S3MOCK_ENDPOINT: localhost:4566
4856
DYNAMODBMOCK_ENDPOINT: http://localhost:4566
4957
AWS_ACCESS_KEY_ID: fake

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ test-extended: test-prereq
136136

137137
.PHONY: test-minimal
138138
test-minimal: test-prereq
139+
echo "TMPDIR=$(TMPDIR)"
139140
go test -failfast -tags containers_image_openpgp -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./...
140141
rm -rf /tmp/getter*; rm -rf /tmp/trivy*
141142

‎examples/kind/kind-ci.sh

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#!/bin/sh
2+
set -o errexit
3+
4+
# Reference: https://kind.sigs.k8s.io/docs/user/local-registry/
5+
6+
# set no_proxy if applicable
7+
if [ ! -z "${no_proxy}" ]; then
8+
echo "Updating no_proxy env var";
9+
export no_proxy=${no_proxy},kind-registry;
10+
export NO_PROXY=${no_proxy};
11+
fi
12+
13+
# create registry container unless it already exists
14+
reg_name='kind-registry'
15+
reg_port='5001'
16+
if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
17+
docker run \
18+
-d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
19+
ghcr.io/project-zot/zot-minimal-linux-amd64:latest
20+
fi
21+
22+
CLUSTER_NAME=kind
23+
## Delete the cluster if it already exist
24+
kind get clusters | grep ${CLUSTER_NAME} && kind delete cluster --name ${CLUSTER_NAME}
25+
26+
# create a cluster with the local registry enabled in containerd
27+
cat <<EOF | kind create cluster --config=-
28+
kind: Cluster
29+
apiVersion: kind.x-k8s.io/v1alpha4
30+
containerdConfigPatches:
31+
- |-
32+
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
33+
endpoint = ["http://${reg_name}:5000"]
34+
EOF
35+
36+
# connect the registry to the cluster network if not already connected
37+
if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
38+
docker network connect "kind" "${reg_name}"
39+
fi
40+
41+
# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
42+
#
43+
# document the local registry
44+
cat <<EOF | kubectl apply -f -
45+
apiVersion: v1
46+
kind: ConfigMap
47+
metadata:
48+
name: local-registry-hosting
49+
namespace: kube-public
50+
data:
51+
localRegistryHosting.v1: |
52+
host: "localhost:${reg_port}"
53+
help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
54+
EOF
55+
56+
## Deploy prometheus operator
57+
kubectl create -f examples/metrics/kubernetes/prometheus/bundle.yaml
58+
59+
## Deploy the Kubernetes objects for RBAC, prometheus CRD and deploy the service
60+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_rbac.yaml
61+
kubectl apply -f examples/metrics/kubernetes/prometheus/prometheus.yaml
62+
kubectl apply -f examples/metrics/kubernetes/prometheus/prom_service.yaml
63+
64+
make oci-image
65+
# copy the image
66+
skopeo copy --format=oci --dest-tls-verify=false oci:oci docker://localhost:5001/zot-build:latest
67+
68+
# deploy the image
69+
kubectl apply -f examples/metrics/kubernetes/zot-extended/deployment.yaml
70+
kubectl patch deployment/zot-extended --patch-file examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
71+
kubectl apply -f examples/metrics/kubernetes/zot-extended/service.yaml
72+
kubectl apply -f examples/metrics/kubernetes/zot-extended/servicemonitor.yaml
73+
74+
# check for availability
75+
echo "Waiting for deployment/zot-extended to be ready ..."
76+
kubectl wait deployment -n default zot-extended --for condition=Available=True --timeout=90s
77+
kubectl wait deployment -n default prometheus-operator --for condition=Available=True --timeout=90s
78+
79+
kubectl port-forward svc/prometheus 9090 --address='0.0.0.0' &
80+
echo "Kind cluster status before sleep:"
81+
kubectl get pods -A
82+
sleep 20
83+
echo "Kind cluster status:"
84+
kubectl get pods -A
85+
echo "zot-extended logs:"
86+
kubectl logs -l app=zot-extended --tail=-1
87+
88+
89+
containername=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].metric.container'`
90+
echo "containername=${containername}"
91+
if [ "${containername}" != '"zot-extended"' ]; then
92+
exit 1
93+
fi
94+
95+
containerup=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].value[1]'`
96+
echo "containerup=${containerup}"
97+
if [ "${containerup}" != '"1"' ]; then
98+
exit 1
99+
fi
100+
101+
zotinfo=`curl -s http://localhost:9090/api/v1/query?query=zot_info | jq '.data.result[].value[1]'`
102+
echo "zotinfo=${zotinfo}"
103+
if [ "${zotinfo}" != '"0"' ]; then
104+
exit 1
105+
fi

‎examples/metrics/kubernetes/zot-extended/patch-deployment.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
spec:
2+
template:
3+
spec:
4+
containers:
5+
- name: zot-extended
6+
image: localhost:5001/zot-build:latest
7+
command: ["/usr/local/bin/zot-linux-amd64"]
8+

0 commit comments

Comments
 (0)