Interface with EWMS [major]

.github/workflows/wipac-cicd.yml

@@ -3,13 +3,25 @@ name: wipac ci/cd
 on: [ push ]
 
 env:
-  CI_TEST: 'yes'
-  THIS_IMAGE_WITH_TAG: 'ghcr.io/wipacrepo/skydriver:latest'
+  THIS_IMAGE_WITH_TAG: 'ghcr.io/wipacrepo/skydriver:vX.Y.Z'
   EWMS_PILOT_TASK_TIMEOUT: 999
   SCAN_BACKLOG_RUNNER_SHORT_DELAY: 1
   SCAN_BACKLOG_RUNNER_DELAY: 1
   SCAN_BACKLOG_PENDING_ENTRY_TTL_REVIVE: 200
   LOG_LEVEL: debug
+  # mandatory env vars...
+  EWMS_ADDRESS: http://localhost:8081
+  EWMS_TOKEN_URL: 65f3b929
+  EWMS_CLIENT_ID: b75a974d
+  EWMS_CLIENT_SECRET: 411b16fe
+  S3_URL: a4f92304
+  S3_ACCESS_KEY_ID: 36c5c849
+  S3_ACCESS_KEY_ID__K8S_SECRET_KEY: 230ec9dc
+  S3_SECRET_KEY: 8dea68a1
+  S3_SECRET_KEY__K8S_SECRET_KEY: cdf7c60b
+  S3_BUCKET: 72017610
+  K8S_SECRET_NAME: super-secrets
+  MIN_SKYMAP_SCANNER_TAG: "v3.1.2" # TODO: remove once skyscan v4 is out (that's the real min)
 
 
 jobs:
@@ -36,7 +48,7 @@ jobs:
       - uses: actions/setup-python@v5
       - uses: WIPACrepo/[email protected]
         with:
-          max-complexity: 20  # decrease this to `10` in the future
+          max-complexity: 10
 
   mypy:
     needs: [ py-versions ]
@@ -69,6 +81,7 @@ jobs:
         #   & don't run non-branch triggers (like tags)
         #   & we don't want to trigger an update on PR's merge to main/master/default (which is a branch)
         run: |
+          set -euo pipefail
           if [[ \
               ${{github.actor}} != 'dependabot[bot]' && \
               ${{github.ref_type}} == 'branch' && \
@@ -135,16 +148,19 @@ jobs:
 
       - name: pip install
         run: |
+          set -euo pipefail
           pip install --upgrade pip wheel setuptools
           pip install .[tests]
 
       - name: test
         run: |
-          pytest -vvv tests/unit
+          set -euo pipefail
+          pytest -vvv tests/unit --exitfirst
 
       - name: Dump logs
         if: always()
         run: |
+          set -euo pipefail
           cat pytest.logs || true
 
   integration-tests:
@@ -161,6 +177,10 @@ jobs:
           - 27017:27017
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.py3 }}
+
       - uses: docker/setup-buildx-action@v2
       - uses: docker/build-push-action@v3
         with:
@@ -173,6 +193,11 @@ jobs:
 
       - name: test
         run: |
+          set -euo pipefail
+
+          pip install .[tests]
+          python tests/integration/dummy_ewms.py &> ./dummy_ewms.out &
+
           export LATEST_TAG=$( \
               curl -I https://github.com/icecube/skymap_scanner/releases/latest \
               | awk -F '/' '/^location/ {print  substr($NF, 1, length($NF)-1)}' \
@@ -193,23 +218,32 @@ jobs:
           docker run --network="host" --rm -i --name test \
               --env LATEST_TAG=$LATEST_TAG \
               --env THIS_IMAGE_WITH_TAG=$THIS_IMAGE_WITH_TAG \
+              --env K8S_SECRET_NAME=$K8S_SECRET_NAME \
               $(env | grep '^SKYSCAN_' | awk '$0="--env "$0') \
               $(env | grep '^EWMS_' | awk '$0="--env "$0') \
-              $(env | grep '^CLIENTMANAGER_' | awk '$0="--env "$0') \
+              $(env | grep '^S3_' | awk '$0="--env "$0') \
+              $(env | grep '^CI' | awk '$0="--env "$0') \
               $(env | grep '^CI_' | awk '$0="--env "$0') \
               $(env | grep '^SCAN_' | awk '$0="--env "$0') \
+              $(env | grep '^MIN_SKYMAP_SCANNER_TAG' | awk '$0="--env "$0') \
               --mount type=bind,source=$(realpath $DIR),target=/local/$DIR \
               wipac/skydriver:local \
               /local/$DIR/test-script.sh
-
+      #
       - name: dump test logs
         if: always()
         run: |
+          set -euo pipefail
           docker logs test || true
-
+      - name: dump dummy-ewms logs
+        if: always()
+        run: |
+          set -euo pipefail
+          cat ./dummy_ewms.out
       - name: dump mongo logs
         if: always()
         run: |
+          set -euo pipefail
           docker logs "${{ job.services.mongo.id }}" || true
 
   test-build-docker:

.gitignore

@@ -143,3 +143,4 @@ gke-cluster-config.yaml
 .idea/
 
 test-suit-sandbox*
+resources/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]/