Fix monitoring known issues described in ansible-playbooks/#392

[PR#392](#392) added
Kubernetes monitoring deployment play. Kubernetes monitoring is
intended to monitor the k3s instance control plane components out of
the box.
PR#392 observes several issues that are mentioned in the
PR's descriptioni, which are related to the k3s distribution.

These issues are addressed within this commit.

Signed-off-by: Matej Feder <[email protected]>

playbooks/infrastructure/kubernetes-monitoring.yml

@@ -20,9 +20,57 @@
           dnation-kubernetes-monitoring:
             grafanaDashboards:
               isLoki: false
-          # Reconfigure node exporter default port
+          grafanaDatasourcesAsConfigMap:
+            cluster-logs: null
           kube-prometheus-stack:
+            # Reconfigure node exporter default port
             prometheus-node-exporter:
               service:
                 port: 9111
                 targetPort: 9111
+            # K3s exposes all metrics combined (apiserver, kubelet, kube-proxy, kube-scheduler, kube-controller)
+            # on each metrics endpoint. Hence, we keep only relevant metrics for each control plane component endpoint.
+            # see https://github.com/k3s-io/k3s/issues/3619#issuecomment-1425852034
+            # FIXME: The strategy to keep only metrics used in particular component dashboards could be insufficient
+            #  when a deeper investigation is needed, using metrics that have been dropped. Improve this logic or check
+            # the current state of k3s monitoring by the kube-prometheus-stack.
+            kubeApiServer:
+              serviceMonitor:
+                metricRelabelings:
+                  - sourceLabels: [ "__name__" ]
+                    regex: "(apiserver|workqueue|rest_client|process|go)_(.+)"
+                    action: keep
+            kubelet:
+              serviceMonitor:
+                metricRelabelings:
+                  - sourceLabels: [ "__name__" ]
+                    regex: "(kubelet|storage|rest_client|process|go)_(.+)"
+                    action: keep
+            kubeEtcd:
+              endpoints: "{{ groups[hosts_k3s_master | default('k3s_master')] | map('extract', hostvars, ['ansible_host']) | list }}"
+              serviceMonitor:
+                metricRelabelings:
+                  - sourceLabels: [ "__name__" ]
+                    regex: "(etcd|grpc|rest_client|process|go)_(.+)"
+                    action: keep
+            kubeProxy:
+              endpoints: "{{ groups[hosts_k3s_master | default('k3s_master')] | map('extract', hostvars, ['ansible_host']) | list }}"
+              serviceMonitor:
+                metricRelabelings:
+                  - sourceLabels: [ "__name__" ]
+                    regex: "(kubeproxy|rest_client|process|go)_(.+)"
+                    action: keep
+            kubeControllerManager:
+              endpoints: "{{ groups[hosts_k3s_master | default('k3s_master')] | map('extract', hostvars, ['ansible_host']) | list }}"
+              serviceMonitor:
+                metricRelabelings:
+                  - sourceLabels: [ "__name__" ]
+                    regex: "(workqueue|rest_client|process|go)_(.+)"
+                    action: keep
+            kubeScheduler:
+              endpoints: "{{ groups[hosts_k3s_master | default('k3s_master')] | map('extract', hostvars, ['ansible_host']) | list }}"
+              serviceMonitor:
+                metricRelabelings:
+                  - sourceLabels: ["__name__"]
+                    regex: "(scheduler|rest_client|process|go)_(.+)"
+                    action: keep