From 7730bc51bfdcdce7391469b37202e1453bf4b2de Mon Sep 17 00:00:00 2001 From: denverwilliams Date: Mon, 21 Sep 2020 16:18:22 -0400 Subject: [PATCH] Add CNF Setups for POC Observability spec tests cncf/cnf-conformance#343 --- spec/fixtures/metrics-server.yaml | 150 ++++++++++++++++++++++++++++ spec/platform/observability_spec.cr | 53 +++++++++- 2 files changed, 201 insertions(+), 2 deletions(-) create mode 100644 spec/fixtures/metrics-server.yaml diff --git a/spec/fixtures/metrics-server.yaml b/spec/fixtures/metrics-server.yaml new file mode 100644 index 000000000..33f541718 --- /dev/null +++ b/spec/fixtures/metrics-server.yaml @@ -0,0 +1,150 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:aggregated-metrics-reader + labels: + rbac.authorization.k8s.io/aggregate-to-view: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-admin: "true" +rules: +- apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-server:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metrics-server-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system +--- +apiVersion: apiregistration.k8s.io/v1beta1 +kind: APIService +metadata: + name: v1beta1.metrics.k8s.io +spec: + service: + name: metrics-server + namespace: kube-system + group: metrics.k8s.io + version: v1beta1 + insecureSkipTLSVerify: true + groupPriorityMinimum: 100 + versionPriority: 100 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metrics-server + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metrics-server + namespace: kube-system + labels: + k8s-app: metrics-server +spec: + selector: + matchLabels: + k8s-app: metrics-server + template: + metadata: + name: metrics-server + labels: + k8s-app: metrics-server + spec: + serviceAccountName: metrics-server + volumes: + # mount in tmp so we can safely use from-scratch images and/or read-only containers + - name: tmp-dir + emptyDir: {} + containers: + - name: metrics-server + image: bitnami/metrics-server:latest + imagePullPolicy: IfNotPresent + args: + - --cert-dir=/tmp + - --secure-port=4443 + ports: + - name: main-port + containerPort: 4443 + protocol: TCP + securityContext: + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 + volumeMounts: + - name: tmp-dir + mountPath: /tmp + nodeSelector: + kubernetes.io/os: linux +--- +apiVersion: v1 +kind: Service +metadata: + name: metrics-server + namespace: kube-system + labels: + kubernetes.io/name: "Metrics-server" + kubernetes.io/cluster-service: "true" +spec: + selector: + k8s-app: metrics-server + ports: + - port: 443 + protocol: TCP + targetPort: main-port +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:metrics-server +rules: +- apiGroups: + - "" + resources: + - pods + - nodes + - nodes/stats + - namespaces + - configmaps + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:metrics-server +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:metrics-server +subjects: +- kind: ServiceAccount + name: metrics-server + namespace: kube-system diff --git a/spec/platform/observability_spec.cr b/spec/platform/observability_spec.cr index bef3244e0..73e589572 100644 --- a/spec/platform/observability_spec.cr +++ b/spec/platform/observability_spec.cr @@ -3,6 +3,49 @@ require "colorize" require "./../../src/tasks/utils/utils.cr" describe "Observability" do + before_all do + current_dir = FileUtils.pwd + LOGGING.info current_dir + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + LOGGING.info "Installing kube_state_metrics" + resp = `#{helm} install kube-state-metrics stable/kube-state-metrics` + LOGGING.info resp + CNFManager.wait_for_install("kube-state-metrics") + + LOGGING.info "Installing prometheus-node-exporter" + resp = `#{helm} install node-exporter stable/prometheus-node-exporter` + LOGGING.info resp + + LOGGING.info "Installing prometheus-adapter" + resp = `#{helm} install prometheus-adapter stable/prometheus-adapter` + LOGGING.info resp + CNFManager.wait_for_install("prometheus-adapter") + + LOGGING.info "Installing metrics_server" + resp = `kubectl create -f spec/fixtures/metrics-server.yaml` + LOGGING.info resp + CNFManager.wait_for_install(deployment_name: "metrics-server", namespace:"kube-system") + # The next line seems to avoid: "Error running at_exit handler" "Invalid Int32" + 0 + end + + after_all do + current_dir = FileUtils.pwd + LOGGING.info current_dir + helm = "#{current_dir}/#{TOOLS_DIR}/helm/linux-amd64/helm" + resp = `#{helm} delete kube-state-metrics` + LOGGING.info resp + $?.success?.should be_true + resp = `#{helm} delete node-exporter` + LOGGING.info resp + $?.success?.should be_true + resp = `#{helm} delete prometheus-adapter` + LOGGING.info resp + $?.success?.should be_true + resp = `kubectl delete -f spec/fixtures/metrics-server.yaml` + LOGGING.info resp + $?.success?.should be_true + end it "'kube_state_metrics' should return some json", tags: "platform:kube_state_metrics" do response_s = `./cnf-conformance platform:kube_state_metrics poc` @@ -11,6 +54,14 @@ describe "Observability" do end it "'node_exporter' should detect the named release of the installed node_exporter", tags: "platform:node_exporter" do + pod_ready = "" + pod_ready_timeout = 45 + until (pod_ready == "true" || pod_ready_timeout == 0) + pod_ready = CNFManager.pod_status("node-exporter-prometheus").split(",")[2] + puts "Pod Ready Status: #{pod_ready}" + sleep 1 + pod_ready_timeout = pod_ready_timeout - 1 + end response_s = `./cnf-conformance platform:node_exporter poc` LOGGING.info response_s (/(PASSED){1}.*(Your platform is using the){1}.*(release for the node exporter){1}/ =~ response_s).should_not be_nil @@ -27,7 +78,5 @@ describe "Observability" do LOGGING.info response_s (/(PASSED){1}.*(Your platform is using the){1}.*(release for the metrics server){1}/ =~ response_s).should_not be_nil end - - end