ci(nightly): add prometheus kind test

adodon2go · adodon2go · commit 7f2b8d60d617 · 2023-10-20T14:53:48.000+03:00
Signed-off-by: Alexei Dodon &lt;adodon@cisco.com&gt;
diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml
@@ -3,9 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [main]
 
 permissions: read-all
 
diff --git a/.github/workflows/branch-cov.yaml b/.github/workflows/branch-cov.yaml
@@ -3,9 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [main]
 
 permissions: read-all
 
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -2,12 +2,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches:
-      - main
-  release:
-    types:
-      - published
 name: build
 
 permissions: read-all
diff --git a/.github/workflows/cluster.yaml b/.github/workflows/cluster.yaml
@@ -3,11 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches: [main]
-  release:
-    types:
-      - published
 
 permissions: read-all
 
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -13,12 +13,8 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ main ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ main ]
-  schedule:
-    - cron: '17 11 * * 0'
+    branches:
+      - main
 
 permissions: read-all
 
diff --git a/.github/workflows/compare-binary-size.yaml b/.github/workflows/compare-binary-size.yaml
@@ -1,7 +1,8 @@
 name: "Check binary size increase"
 on:
-  pull_request:
-    branches: [main]
+  push:
+    branches:
+      - main
 
 permissions: read-all
 
diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml
@@ -3,11 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches: [main]
-  release:
-    types:
-      - published
 
 permissions: read-all
 
diff --git a/.github/workflows/gc-stress-test.yaml b/.github/workflows/gc-stress-test.yaml
@@ -3,11 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches: [main]
-  release:
-    types:
-      - published
 
 permissions: read-all
 
diff --git a/.github/workflows/gqlgen.yaml b/.github/workflows/gqlgen.yaml
@@ -8,11 +8,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches: [main]
-  release:
-    types:
-      - published
 
 permissions: read-all
 
diff --git a/.github/workflows/kind.yaml b/.github/workflows/kind.yaml
@@ -0,0 +1,43 @@
+name: 'Kind setup'
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+permissions: read-all
+
+jobs:
+  kind-setup:
+    name: Prometheus setup
+    runs-on: ubuntu-latest-16-cores
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/clean-runner
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+      - name: Install dependencies
+        run: |
+          cd $GITHUB_WORKSPACE
+          go mod download
+          sudo apt-get update
+          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
+          # install skopeo
+          git clone -b v1.12.0 https://github.com/containers/skopeo.git
+          cd skopeo
+          make bin/skopeo
+          sudo cp bin/skopeo /usr/bin
+          skopeo -v
+      - name: Log in to GitHub Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Run tests
+        run: |
+            ./examples/kind/kind-ci.sh
+
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -3,11 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches: [main]
-  release:
-    types:
-      - published
 
 permissions: read-all
 
diff --git a/.github/workflows/tls.yaml b/.github/workflows/tls.yaml
@@ -3,9 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [main]
 
 permissions: read-all
 
diff --git a/.github/workflows/web-scan.yaml b/.github/workflows/web-scan.yaml
@@ -3,12 +3,6 @@ on:
   push:
     branches:
       - main
-  pull_request:
-    branches:
-      - main
-  release:
-    types:
-      - published
 
 permissions:
   contents: read
diff --git a/examples/kind/kind-ci.sh b/examples/kind/kind-ci.sh
@@ -0,0 +1,106 @@
+#!/bin/sh
+set -o errexit
+
+# Reference: https://kind.sigs.k8s.io/docs/user/local-registry/
+
+# set no_proxy if applicable
+if [ ! -z "${no_proxy}" ]; then
+  echo "Updating no_proxy env var";
+  export no_proxy=${no_proxy},kind-registry;
+  export NO_PROXY=${no_proxy};
+fi
+
+# create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    ghcr.io/project-zot/zot-minimal-linux-amd64:latest
+fi
+
+CLUSTER_NAME=kind
+## Delete the cluster if it already exist
+kind get clusters | grep ${CLUSTER_NAME} &&  kind delete cluster --name ${CLUSTER_NAME}
+
+# create a cluster with the local registry enabled in containerd
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_name}:5000"]
+EOF
+
+# connect the registry to the cluster network if not already connected
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+#
+# document the local registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+## Deploy prometheus operator
+kubectl create -f examples/metrics/kubernetes/prometheus/bundle.yaml
+
+## Deploy the Kubernetes objects for RBAC, prometheus CRD and deploy the service
+kubectl apply -f examples/metrics/kubernetes/prometheus/prom_rbac.yaml
+kubectl apply -f examples/metrics/kubernetes/prometheus/prometheus.yaml
+kubectl apply -f examples/metrics/kubernetes/prometheus/prom_service.yaml
+
+make oci-image
+# copy the image
+skopeo copy --format=oci --dest-tls-verify=false oci:oci docker://localhost:5001/zot-build:latest
+
+# deploy the image
+kubectl apply -f examples/metrics/kubernetes/zot-extended/deployment.yaml
+kubectl patch deployment/zot-extended --patch-file examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
+kubectl apply -f examples/metrics/kubernetes/zot-extended/service.yaml
+kubectl apply -f examples/metrics/kubernetes/zot-extended/servicemonitor.yaml
+
+# check for availability
+echo "Waiting for deployment/zot-extended to be ready ..."
+kubectl wait deployment -n default zot-extended --for condition=Available=True --timeout=90s
+kubectl wait deployment -n default prometheus-operator --for condition=Available=True --timeout=90s
+
+kubectl port-forward svc/prometheus 9090 --address='0.0.0.0' &
+echo "Kind cluster status before sleep:"
+kubectl get pods -A
+# Put enough amount of time for prometheus scraping take place
+sleep 40
+echo "Kind cluster status:"
+kubectl get pods -A
+echo "zot-extended logs:"
+kubectl logs -l app=zot-extended --tail=-1
+
+
+containername=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].metric.container'`
+echo "containername=${containername}"
+if [ "${containername}" != '"zot-extended"' ]; then
+    exit 1
+fi
+
+containerup=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].value[1]'`
+echo "containerup=${containerup}"
+if [ "${containerup}" != '"1"' ]; then
+    exit 1
+fi
+
+zotinfo=`curl -s http://localhost:9090/api/v1/query?query=zot_info | jq '.data.result[].value[1]'`
+echo "zotinfo=${zotinfo}"
+if [ "${zotinfo}" != '"0"' ]; then
+    exit 1
+fi
diff --git a/examples/metrics/kubernetes/zot-extended/patch-deployment.yaml b/examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
@@ -0,0 +1,8 @@
+spec:
+  template:
+    spec:
+      containers:
+      - name: zot-extended
+        image: localhost:5001/zot-build:latest
+        command: ["/usr/local/bin/zot-linux-amd64"]
+
