debug docker and k8s deployment tests

qrkourier · qrkourier · commit be554c09bb5a · 2025-03-12T16:43:19.000-04:00
diff --git a/.github/workflows/test-deployments.yml b/.github/workflows/test-deployments.yml
@@ -20,7 +20,7 @@ env:
 jobs:
   build-linux-packages:
     name: Build ${{ matrix.package_name }} ${{ matrix.arch.gox }} ${{ matrix.packager }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         package_name:
@@ -77,7 +77,7 @@ jobs:
   dry-run-linux-packages:
     needs: build-linux-packages
     name: Dry Run ${{ format('{0}:{1}', matrix.distro.name, matrix.distro.version) }} ${{ matrix.arch.gox }}
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     env:
       ZITI_PWD: ziggypw
       ZITI_CTRL_ADVERTISED_ADDRESS: linux-controller.127.21.71.0.sslip.io
@@ -136,7 +136,10 @@ jobs:
       - name: Install Packages
         shell: bash
         run: |
-          set -x
+
+          set -o pipefail
+          set -o xtrace
+
           ls -lR ./release/
           case "${{ matrix.distro.type }}" in
             rpm)
@@ -158,7 +161,10 @@ jobs:
         env:
           DEBUG: 1
         run: |
-          set -x
+
+          set -o pipefail
+          set -o xtrace
+
           /opt/openziti/etc/controller/bootstrap.bash <<CTRL
           ZITI_CTRL_ADVERTISED_ADDRESS=${ZITI_CTRL_ADVERTISED_ADDRESS}
           ZITI_CTRL_ADVERTISED_PORT=${ZITI_CTRL_ADVERTISED_PORT}
@@ -176,9 +182,9 @@ jobs:
           ZITI_ARGS="--verbose"
           ROUTER
 
-  test-linux-services:
-    name: Test Debian Linux Services
-    runs-on: ubuntu-latest
+  linux-deployments:
+    name: Test Linux Deployments
+    runs-on: ubuntu-24.04
     steps:
       - name: Shallow checkout
         uses: actions/checkout@v4
@@ -192,9 +198,13 @@ jobs:
       - name: Install nfpm
         shell: bash
         run: |
+
+          set -o pipefail
+          set -o xtrace
+
           echo ~/.local/bin >> $GITHUB_PATH
           mkdir -p ~/.local/bin
-          wget -qO- https://github.com/goreleaser/nfpm/releases/download/v${{ env.NFPM_VERSION }}/nfpm_${{ env.NFPM_VERSION }}_Linux_x86_64.tar.gz | tar --directory ~/.local/bin -xz nfpm
+          wget -qO- https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz | tar --directory ~/.local/bin -xz nfpm
           nfpm --version
 
       - name: Bootstrap & Run
@@ -205,8 +215,10 @@ jobs:
         if: always()
         shell: bash
         run: |
-          set -x
-          set +e
+
+          set -o xtrace
+          set +o errexit
+
           sudo ss -lntp | grep -E ":(${ZITI_CTRL_ADVERTISED_PORT}|${ZITI_ROUTER_PORT})"
           sudo journalctl --no-pager -o cat -u ziti-controller.service
           sudo journalctl --no-pager -o cat -u ziti-router.service
@@ -215,7 +227,7 @@ jobs:
 
   docker-deployments:
     name: Test Docker Deployments
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     env:
       ZIGGY_UID: 1001  # let container EUID run-as GHA "runner" user to share cache, etc.
     steps:
@@ -228,15 +240,15 @@ jobs:
         with:
           go-version-file: ./go.mod
 
-      - name: Run the Compose Test Script
+      - name: Run the Docker Test Script
         shell: bash
-        run: dist/docker-images/compose.test.bash
+        run: dist/docker-images/docker.test.bash
         env:
           ZITI_GO_VERSION: ${{ steps.setup-go.outputs.go-version }}
 
   kubernetes-deployments:
     name: Test Kubernetes Deployments
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     env:
       ZITI_NAMESPACE: zititest
     steps:
@@ -257,4 +269,3 @@ jobs:
       - name: Run the Kubernetes Test Script
         shell: bash
         run: ./dist/docker-images/k8s.test.bash --cpus=2  # GitHub runners have 4 vCPUs
-
diff --git a/dist/dist-packages/linux/linux.test.bash b/dist/dist-packages/linux/linux.test.bash
@@ -182,18 +182,26 @@ else
     exit 1
 fi
 
+export \
 ZITI_CTRL_EDGE_ADVERTISED_ADDRESS=${ZITI_CTRL_ADVERTISED_ADDRESS} \
-ZITI_CTRL_EDGE_ADVERTISED_PORT=${ZITI_CTRL_ADVERTISED_PORT} \
-go test -v -count=1 -tags="quickstart manual" ./ziti/cmd/edge/...
+ZITI_CTRL_EDGE_ADVERTISED_PORT=${ZITI_CTRL_ADVERTISED_PORT}
+
+_test_result=$(go test -v -count=1 -tags="quickstart manual" ./ziti/run/...)
+
+# check for failure modes that don't result in an error exit code
+if [[ "${_test_result}" =~ "no tests to run" ]]
+then
+    echo "ERROR: test failed because no tests to run"
+    exit 1
+fi
 
 ATTEMPTS=5
 DELAY=3
 
 # verify console is available
 curl_cmd="curl -skSfw '%{http_code}\t%{url}\n' -o/dev/null \"https://${ZITI_CTRL_ADVERTISED_ADDRESS}:${ZITI_CTRL_ADVERTISED_PORT}/zac/\""
-until ! ((ATTEMPTS)) || eval "${curl_cmd}" &> /dev/null
+until ! (( ATTEMPTS-- )) || eval "${curl_cmd}" &> /dev/null
 do
-    (( ATTEMPTS-- ))
     echo "Waiting for zac"
     sleep ${DELAY}
 done
diff --git a/dist/docker-images/docker.test.bash b/dist/docker-images/docker.test.bash
@@ -124,18 +124,26 @@ ziti edge create edge-router "${ZITI_ROUTER_NAME}" -to ~ziggy/.config/ziti/"${ZI
 docker compose up ziti-router --detach
 
 unset GOOS
+export \
 ZITI_CTRL_EDGE_ADVERTISED_ADDRESS=${ZITI_CTRL_ADVERTISED_ADDRESS} \
-ZITI_CTRL_EDGE_ADVERTISED_PORT=${ZITI_CTRL_ADVERTISED_PORT} \
-go test -v -count=1 -tags="quickstart manual" ./ziti/cmd/edge/...
+ZITI_CTRL_EDGE_ADVERTISED_PORT=${ZITI_CTRL_ADVERTISED_PORT}
+
+_test_result=$(go test -v -count=1 -tags="quickstart manual" ./ziti/run/...)
+
+# check for failure modes that don't result in an error exit code
+if [[ "${_test_result}" =~ "no tests to run" ]]
+then
+    echo "ERROR: test failed because no tests to run"
+    exit 1
+fi
 
 ATTEMPTS=5
 DELAY=3
 
 # verify console is available
 curl_cmd="curl -skSfw '%{http_code}\t%{url}\n' -o/dev/null \"https://${ZITI_CTRL_ADVERTISED_ADDRESS}:${ZITI_CTRL_ADVERTISED_PORT}/zac/\""
-until ! ((ATTEMPTS)) || eval "${curl_cmd}" &> /dev/null
+until ! (( ATTEMPTS-- )) || eval "${curl_cmd}" &> /dev/null
 do
-    (( ATTEMPTS-- ))
     echo "Waiting for zac"
     sleep ${DELAY}
 done
diff --git a/dist/docker-images/k8s.test.bash b/dist/docker-images/k8s.test.bash
@@ -122,34 +122,40 @@ image:
     pullPolicy: Never
 ROUTER
 
-./quickstart/kubernetes/miniziti.bash start \
+bash -x ./quickstart/kubernetes/miniziti.bash start \
 --profile "${ZITI_NAMESPACE}" \
 --no-hosts \
---debug \
 --values-dir "${EXTRA_VALUES_DIR}"
 
 MINIKUBE_IP="$(minikube --profile "${ZITI_NAMESPACE}" ip)"
+ZITI_CTRL_ADVERTISED_ADDRESS="miniziti-controller.${MINIKUBE_IP}.sslip.io"
 
 # verify console is available
-curl -skSfw '%{http_code}\t%{url}\n' -o/dev/null "https://miniziti-controller.${MINIKUBE_IP}.sslip.io/zac/"
+curl -skSfw '%{http_code}\t%{url}\n' -o/dev/null "https://${ZITI_CTRL_ADVERTISED_ADDRESS}:${ZITI_CTRL_ADVERTISED_PORT}/zac/"
 
 ZITI_PWD=$(
-minikube kubectl --profile "${ZITI_NAMESPACE}" -- \
---context "${ZITI_NAMESPACE}" \
-get secrets "ziti-controller-admin-secret" \
---namespace "${ZITI_NAMESPACE}" \
---output go-template='{{index .data "admin-password" | base64decode }}'
+    minikube kubectl --profile "${ZITI_NAMESPACE}" -- \
+        --context "${ZITI_NAMESPACE}" \
+        get secrets "ziti-controller-admin-secret" \
+        --namespace "${ZITI_NAMESPACE}" \
+        --output go-template='{{index .data "admin-password" | base64decode }}'
 )
 
 
 export \
 ZITI_PWD \
 ZITI_ROUTER_NAME="miniziti-router" \
-ZITI_CTRL_ADVERTISED_ADDRESS="miniziti-controller.${MINIKUBE_IP}.sslip.io"
-
-ZITI_CTRL_EDGE_ADVERTISED_ADDRESS=${ZITI_CTRL_ADVERTISED_ADDRESS} \
-ZITI_CTRL_EDGE_ADVERTISED_PORT=${ZITI_CTRL_ADVERTISED_PORT} \
-ZITI_TEST_BIND_ADDRESS="ziti-controller-client.${ZITI_NAMESPACE}.svc.cluster.local" \
-go test -v -count=1 -tags="quickstart manual" ./ziti/cmd/edge/...
+ZITI_CTRL_EDGE_ADVERTISED_ADDRESS="${ZITI_CTRL_ADVERTISED_ADDRESS}" \
+ZITI_CTRL_EDGE_ADVERTISED_PORT="${ZITI_CTRL_ADVERTISED_PORT}" \
+ZITI_TEST_BIND_ADDRESS="ziti-controller-client.${ZITI_NAMESPACE}.svc.cluster.local"
+
+_test_result=$(go test -v -count=1 -tags="quickstart manual" ./ziti/run/...)
+
+# check for failure modes that don't result in an error exit code
+if [[ "${_test_result}" =~ "no tests to run" ]]
+then
+    echo "ERROR: test failed because no tests to run"
+    exit 1
+fi
 
 cleanup
