Add Observability Check for the Prometheus Adapter & the K8s Metrics …

…Server cncf/cnf-conformance#343

spec/platform/observability_spec.cr

@@ -16,5 +16,18 @@ describe "Observability" do
       (/(PASSED){1}.*(Your platform is using the){1}.*(release for the node exporter){1}/ =~ response_s).should_not be_nil
   end
 
+  it "'prometheus_adapter' should detect the named release of the installed prometheus_adapter", tags: "platform:prometheus_adapter" do
+      response_s = `./cnf-conformance platform:prometheus_adapter poc`
+      LOGGING.info response_s
+      (/(PASSED){1}.*(Your platform is using the){1}.*(release for the prometheus adapter){1}/ =~ response_s).should_not be_nil
+  end
+
+  it "'metrics_server' should detect the named release of the installed metrics_server", tags: "platform:metrics_server" do
+      response_s = `./cnf-conformance platform:metrics_server poc`
+      LOGGING.info response_s
+      (/(PASSED){1}.*(Your platform is using the){1}.*(release for the metrics server){1}/ =~ response_s).should_not be_nil
+  end
+
+
 end
 

src/tasks/platform/observability.cr

@@ -5,7 +5,7 @@ require "../utils/utils.cr"
 
 namespace "platform" do
   desc "The CNF conformance suite checks to see if the Platform has Observability support."
-  task "observability", ["kube_state_metrics", "node_exporter"] do |t, args|
+  task "observability", ["kube_state_metrics", "node_exporter", "prometheus_adapter", "metrics_server"] do |t, args|
     VERBOSE_LOGGING.info "resilience" if check_verbose(args)
     VERBOSE_LOGGING.debug "resilience args.raw: #{args.raw}" if check_verbose(args)
     VERBOSE_LOGGING.debug "resilience args.named: #{args.named}" if check_verbose(args)
@@ -131,6 +131,126 @@ namespace "platform" do
   end
 end
 
+
+  desc "Does the Platform have the prometheus adapter installed"
+  task "prometheus_adapter" do |_, args|
+    unless check_poc(args)
+      LOGGING.info "skipping prometheus_adapter: not in poc mode"
+      puts "Skipped".colorize(:yellow)
+      next
+    end
+    LOGGING.info "Running POC: prometheus_adapter"
+    task_response = task_runner(args) do |args|
+      # Fetch image id sha256sums available for all upstream prometheus_adapter releases
+      prometheus_adapter_releases = `curl -L -s 'https://registry.hub.docker.com/v2/repositories/directxman12/k8s-prometheus-adapter-amd64/tags?page_size=1024'`
+      sha_list = named_sha_list(prometheus_adapter_releases)
+      LOGGING.debug "sha_list: #{sha_list}"
+
+      # TODO find hash for image
+      imageids = KubectlClient::Get.all_container_repo_digests
+      LOGGING.debug "imageids: #{imageids}"
+      found = false
+      release_name = ""
+      sha_list.each do |x|
+        if imageids.find{|i| i.includes?(x["manifest_digest"])}
+          found = true
+          release_name = x["name"]
+        end
+      end
+
+      if found
+        emoji_prometheus_adapter="📶☠️"
+        upsert_passed_task("prometheus_adapter","✔️  PASSED: Your platform is using the #{release_name} release for the prometheus adapter #{emoji_prometheus_adapter}")
+      else
+        emoji_prometheus_adapter="📶☠️"
+        upsert_failed_task("prometheus_adapter", "✖️  FAILURE: Your platform does not have the prometheus adapter installed #{emoji_prometheus_adapter}")
+      end
+    end
+  end
+
+  desc "Does the Platform have the K8s Metrics Server installed"
+  task "metrics_server" do |_, args|
+    unless check_poc(args)
+      LOGGING.info "skipping metrics_server: not in poc mode"
+      puts "Skipped".colorize(:yellow)
+      next
+    end
+    LOGGING.info "Running POC: metrics_server"
+    task_response = task_runner(args) do |args|
+
+      #Select the first node that isn't a master and is also schedulable
+      #worker_nodes = `kubectl get nodes --selector='!node-role.kubernetes.io/master' -o 'go-template={{range .items}}{{$taints:=""}}{{range .spec.taints}}{{if eq .effect "NoSchedule"}}{{$taints = print $taints .key ","}}{{end}}{{end}}{{if not $taints}}{{.metadata.name}}{{ "\\n"}}{{end}}{{end}}'`
+      #worker_node = worker_nodes.split("\n")[0]
+
+      # Install and find CRI Tools name
+      File.write("cri_tools.yml", CRI_TOOLS)
+      install_cri_tools = `kubectl create -f cri_tools.yml`
+      cri_tools_pod = CNFManager.pod_status("cri-tools").split(",")[0]
+      #, "--field-selector spec.nodeName=#{worker_node}")
+      LOGGING.debug "cri_tools_pod: #{cri_tools_pod}"
+
+      # Fetch id sha256 sums for all repo_digests https://github.com/docker/distribution/issues/1662
+      repo_digest_list = KubectlClient::Get.all_container_repo_digests
+      LOGGING.info "container_repo_digests: #{repo_digest_list}"
+      id_sha256_list = repo_digest_list.reduce([] of String) do |acc, repo_digest|
+        LOGGING.debug "repo_digest: #{repo_digest}"
+        cricti = `kubectl exec -ti #{cri_tools_pod} crictl inspecti #{repo_digest}`
+        LOGGING.debug "cricti: #{cricti}"
+        parsed_json = JSON.parse(cricti)
+        acc << parsed_json["status"]["id"].as_s
+      end
+      LOGGING.info "id_sha256_list: #{id_sha256_list}"
+
+
+      # Fetch image id sha256sums available for all upstream node-exporter releases
+      metrics_server_releases = `curl -L -s 'https://registry.hub.docker.com/v2/repositories/bitnami/metrics-server/tags?page=1'`
+      tag_list = named_sha_list(metrics_server_releases)
+      LOGGING.info "tag_list: #{tag_list}"
+      if ENV["DOCKERHUB_USERNAME"]? && ENV["DOCKERHUB_PASSWORD"]?
+        target_ns_repo = "bitnami/metrics-server"
+        params = "service=registry.docker.io&scope=repository:#{target_ns_repo}:pull"
+        token = `curl --user "#{ENV["DOCKERHUB_USERNAME"]}:#{ENV["DOCKERHUB_PASSWORD"]}" "https://auth.docker.io/token?#{params}"`
+        parsed_token = JSON.parse(token)
+        release_id_list = tag_list.reduce([] of Hash(String, String)) do |acc, tag|
+          LOGGING.debug "tag: #{tag}"
+          tag = tag["name"]
+
+          image_id = `curl --header "Accept: application/vnd.docker.distribution.manifest.v2+json" "https://registry-1.docker.io/v2/#{target_ns_repo}/manifests/#{tag}" -H "Authorization:Bearer #{parsed_token["token"].as_s}"`
+          parsed_image = JSON.parse(image_id)
+
+          LOGGING.debug "parsed_image config digest #{parsed_image["config"]["digest"]}"
+          if parsed_image["config"]["digest"]?
+              acc << {"name" => tag, "digest"=> parsed_image["config"]["digest"].as_s}
+          else
+            acc
+          end
+        end
+      else
+        puts "DOCKERHUB_USERNAME & DOCKERHUB_PASSWORD Must be set."
+        exit 1
+      end
+      LOGGING.info "Release id sha256sum list: #{release_id_list}"
+
+      found = false
+      release_name = ""
+      release_id_list.each do |x|
+        if id_sha256_list.find{|i| i.includes?(x["digest"])}
+          found = true
+          release_name = x["name"]
+        end
+      end
+      if found
+        emoji_metrics_server="📶☠️"
+        upsert_passed_task("metrics_server","✔️  PASSED: Your platform is using the #{release_name} release for the metrics server #{emoji_metrics_server}")
+      else
+        emoji_metrics_server="📶☠️"
+        upsert_failed_task("metrics_server", "✖️  FAILURE: Your platform does not have the metrics server installed #{emoji_metrics_server}")
+      end
+    end
+  end
+
+
+
 def named_sha_list(resp_json)
   LOGGING.debug "sha_list resp_json: #{resp_json}"
   parsed_json = JSON.parse(resp_json)