CORE-11514: Added CI test to check images availability

Author: skoryk-oleksandr

.semaphore/semaphore.yml

@@ -1,6 +1,7 @@
 global_job_config:
   secrets:
     - name: docker-hub
+    - name: oss-release-secrets
   prologue:
     commands:
       - checkout
@@ -10,6 +11,8 @@ global_job_config:
       # unshallow it for GIT_VERSION:=$(shell git describe --tags --dirty --always)
       - retry git fetch --unshallow
       - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
+      - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/gcr-credentials.json
+      - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
   epilogue:
     commands:
       - cd "$REPO_DIR"

.semaphore/semaphore.yml.d/02-global_job_config.yml

@@ -5,3 +5,10 @@
       - name: Pre-flight checks
         commands:
           - make ci-preflight-checks
+    secrets:
+      - name: oss-release-secrets
+    prologue:
+      commands:
+        - export GOOGLE_APPLICATION_CREDENTIALS=$HOME/secrets/gcr-credentials.json
+        - echo "🔐 Authenticating to GCR using GOOGLE_APPLICATION_CREDENTIALS"
+        - gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}

.semaphore/semaphore.yml.d/blocks/10-prerequisites.yml

@@ -50,6 +50,7 @@ clean:
 
 ci-preflight-checks:
 	$(MAKE) check-go-mod
+	$(MAKE) check-images-availability
 	$(MAKE) verify-go-mods
 	$(MAKE) check-dockerfiles
 	$(MAKE) check-language
@@ -71,6 +72,11 @@ go-vet:
 check-dockerfiles:
 	./hack/check-dockerfiles.sh
 
+check-images-availability: bin/yq bin/crane
+	cd ./hack && \
+		OPERATOR_VERSION=$(OPERATOR_VERSION) \
+		./check-images-availability.sh
+
 check-language:
 	./hack/check-language.sh
 

Makefile

@@ -0,0 +1,100 @@
+#!/bin/bash
+set -euo pipefail
+
+# Resolve script directory
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Use provided YQ or fallback to ../bin/yq (relative to hack/)
+YQ="${YQ:-../bin/yq}"
+CRANE="${CRANE:-../bin/crane}"
+
+echo "🔍 Available files in \$HOME/secrets:"
+ls -l "$HOME/secrets" || echo "❌ ~/secrets not found"
+
+find "$HOME/secrets" 2>/dev/null || echo "❌ Cannot traverse ~/secrets"
+
+if [ ! -x "$YQ" ]; then
+  echo "❌ Error: yq not found or not executable at: $YQ" >&2
+  echo "🔍 Resolved path: $(realpath "$YQ" 2>/dev/null || echo '<unresolvable>')" >&2
+  exit 1
+fi
+
+# Operator version from values.yaml
+defaultOperatorVersion=$("$YQ" .tigeraOperator.version < "${SCRIPT_DIR}/../charts/tigera-operator/values.yaml")
+OPERATOR_VERSION="${OPERATOR_VERSION:-$defaultOperatorVersion}"
+IMAGE_SOURCE="quay.io/tigera/operator:${OPERATOR_VERSION}"
+
+echo "🔍 Checking images for OPERATOR_VERSION=${OPERATOR_VERSION}"
+echo "🔍 Fetching image list from ${IMAGE_SOURCE}..."
+
+#########################################
+# Step 1: Fetch image list from operator
+#########################################
+operator_images=$(
+  docker run --rm "${IMAGE_SOURCE}" --print-images=list 2>/dev/null \
+  | grep -E '^[a-z0-9.-]+\.[a-z0-9.-]+/[a-z0-9._/-]+:[a-zA-Z0-9._-]+' \
+  | grep -v -- '-fips' \
+  | sort -u
+)
+
+#########################################
+# Step 2: Extract from manifests
+#########################################
+manifest_dir="${SCRIPT_DIR}/../manifests"
+manifest_images=$(
+  grep -rhoE 'image:\s*["'\''"]?[a-z0-9.-]+\.[a-z0-9.-]+/[a-z0-9._/-]+:[a-zA-Z0-9._-]+' "$manifest_dir" \
+  | sed -E 's/image:\s*["'\''"]?//' \
+  | grep -v -- '-fips' \
+  | sort -u
+)
+
+#########################################
+# Step 3: Combine and deduplicate
+#########################################
+all_images=$(echo -e "${operator_images}\n${manifest_images}" | sort -u)
+count=$(echo "$all_images" | wc -l)
+
+echo "📦 Total unique images to check (excluding -fips): ${count}"
+
+#########################################
+# Step 4: Check availability with retries
+#########################################
+FAILED=0
+FAILED_IMAGES=()
+
+while IFS= read -r image; do
+  success=0
+  for attempt in 1 2 3; do
+    if "$CRANE" digest "$image" >/dev/null 2>&1; then
+      echo "✅ Available: $image"
+      success=1
+      break
+    else
+      echo "⚠️  Attempt $attempt failed for: $image"
+      if [ "$attempt" -eq 3 ]; then
+        echo "🔍 Used crane at: $(realpath "$CRANE" 2>/dev/null || echo '<unresolvable>')"
+      fi
+      sleep 3
+    fi
+  done
+
+  if [ "$success" -ne 1 ]; then
+    echo "❌ NOT FOUND after 3 attempts: $image"
+    FAILED=1
+    FAILED_IMAGES+=("$image")
+  fi
+done <<< "$all_images"
+
+#########################################
+# Step 5: Final result
+#########################################
+if [ "$FAILED" -eq 1 ]; then
+  echo ""
+  echo "❗ Some images are missing or invalid:"
+  for img in "${FAILED_IMAGES[@]}"; do
+    echo "   ❌ $img"
+  done
+  exit 1
+else
+  echo "All images are available!"
+fi

hack/check-images-availability.sh

@@ -1249,6 +1249,26 @@ bin/yq:
 	tar -zxvf $(TMP)/yq4.tar.gz -C $(TMP)
 	mv $(TMP)/yq_linux_$(BUILDARCH) bin/yq
 
+# This setup is used to download and install the 'crane' binary into the local bin/ directory.
+# The binary will be placed at: ./bin/crane
+# Normalize architecture for go-containerregistry filenames
+CRANE_BUILDARCH := $(shell uname -m | sed 's/amd64/x86_64/;s/x86_64/x86_64/;s/aarch64/arm64/')
+ifeq ($(CRANE_BUILDARCH),)
+  $(error Unsupported or unknown architecture: $(shell uname -m))
+endif
+CRANE_VERSION := v0.20.6
+CRANE_FILENAME := go-containerregistry_Linux_$(CRANE_BUILDARCH).tar.gz
+CRANE_URL := https://github.com/google/go-containerregistry/releases/download/$(CRANE_VERSION)/$(CRANE_FILENAME)
+
+# Install crane binary into bin/
+bin/crane:
+	mkdir -p bin
+	$(eval TMP := $(shell mktemp -d))
+	curl -sSfL --retry 5 -o $(TMP)/crane.tar.gz $(CRANE_URL)
+	tar -xzf $(TMP)/crane.tar.gz -C $(TMP) crane
+	mv $(TMP)/crane bin/crane
+	chmod +x bin/crane
+
 ###############################################################################
 # Common functions for launching a local Kubernetes control plane.
 ###############################################################################