prometheus-configmap.yaml

# Useful examples on how to configure Prometheus
# * https://www.weave.works/prometheus-and-kubernetes-monitoring-your-applications/
# * https://grafana.net/dashboards/162
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-configmap
data:
  prometheus.yml: |-
    # A scrape configuration for running Prometheus on a Kubernetes cluster.
    # This uses separate scrape configs for cluster components (i.e. API server, node)
    # and services to allow each to use different authentication configs.
    #
    # Kubernetes labels will be added as Prometheus labels on metrics via the
    # `labelmap` relabeling action.

    rule_files:
      - "/etc/prometheus-rules/*.rules"

    # Scrape config for cluster components.
    scrape_configs:
    # etcd is living outside of our cluster and we configure
    # it directly.
    #- job_name: 'etcd'
    #  static_configs:
    #  - targets:
    #{% for host in groups['etcd'] %}
    #    - {{ hostvars[host]['ansible_' + etcd_interface].ipv4.address }}:{{ etcd_client_port }}
    #{% endfor %}
    #  tls_config:
    #    ca_file: /etc/etcd/ssl/ca.pem
    #    cert_file: /etc/etcd/ssl/client.pem
    #    key_file: /etc/etcd/ssl/client-key.pem
    #  scheme: https
    # Dynamic etcd rule, which helps to avoid ansible jinja2 templates
    # In order to make it work as expected it is necessary to add kubelet into all etcd nodes (and set them unschedulable if necessary)
    # Also you have to mount proper TLS keypairs in case when your etcd cluster is protected by TLS auth
    - job_name: 'etcd'
      scheme: https
      tls_config:
        ca_file: /etc/etcd/ssl/ca.pem
        cert_file: /etc/etcd/ssl/client.pem
        key_file: /etc/etcd/ssl/client-key.pem
      kubernetes_sd_configs:
      - api_servers:
        - https://kubernetes.default.svc
        in_cluster: true
        role: node
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:2379'
        target_label: __address__

    - job_name: 'kubernetes-cluster'
      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: https
      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration (`in_cluster` below) because discovery & scraping are two
      # separate concerns in Prometheus.
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        # If your node certificates are self-signed or use a different CA to the
        # master CA, then disable certificate verification below. Note that
        # certificate verification is an integral part of a secure infrastructure
        # so this should only be disabled in a controlled environment. You can
        # disable certificate verification by uncommenting the line below.
        #
        # insecure_skip_verify: true
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:
      - api_servers:
        - https://kubernetes.default.svc
        in_cluster: true
        role: apiserver

    - job_name: 'kubernetes-cadvisor'

      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: http

      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration (`in_cluster` below) because discovery & scraping are two
      # separate concerns in Prometheus.
      #tls_config:
      #  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        # If your node certificates are self-signed or use a different CA to the
        # master CA, then disable certificate verification below. Note that
        # certificate verification is an integral part of a secure infrastructure
        # so this should only be disabled in a controlled environment. You can
        # disable certificate verification by uncommenting the line below.
        #
        # insecure_skip_verify: true
      #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:
      - api_servers:
        - https://kubernetes.default.svc
        in_cluster: true
        role: node

      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      # Use cadvisor 4194 port
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:4194'
        target_label: __address__

      # Backward compatibility for Kubernetes 1.3 dashboards
      metric_relabel_configs:
      - source_labels: [io_kubernetes_container_name,container_name]
        action: replace
        regex: (.*);(.*)
        replacement: '${1}${2}'
        target_label: io_kubernetes_container_name
      - source_labels: [kubernetes_pod_name,pod_name]
        action: replace
        regex: (.*);(.*)
        replacement: '${1}${2}'
        target_label: kubernetes_pod_name
      - source_labels: [kubernetes_pod_name]
        action: replace
        target_label: io_kubernetes_pod_name

    - job_name: 'kubernetes-nodes'

      # Default to scraping over https. If required, just disable this or change to
      # `http`.
      scheme: http

      # This TLS & bearer token file config is used to connect to the actual scrape
      # endpoints for cluster components. This is separate to discovery auth
      # configuration (`in_cluster` below) because discovery & scraping are two
      # separate concerns in Prometheus.
      #tls_config:
      #  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
        # If your node certificates are self-signed or use a different CA to the
        # master CA, then disable certificate verification below. Note that
        # certificate verification is an integral part of a secure infrastructure
        # so this should only be disabled in a controlled environment. You can
        # disable certificate verification by uncommenting the line below.
        #
        # insecure_skip_verify: true
      #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

      kubernetes_sd_configs:
      - api_servers:
        - https://kubernetes.default.svc
        in_cluster: true
        role: node

      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      # Use insecure read-only HTTP 10255 port
      # More info is here: https://github.com/kayrus/kubelet-exploit
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:10255'
        target_label: __address__

      # Backward compatibility for Kubernetes 1.3 dashboards
      metric_relabel_configs:
      - source_labels: [io_kubernetes_container_name,container_name]
        action: replace
        regex: (.*);(.*)
        replacement: '${1}${2}'
        target_label: io_kubernetes_container_name
      - source_labels: [kubernetes_pod_name,pod_name]
        action: replace
        regex: (.*);(.*)
        replacement: '${1}${2}'
        target_label: kubernetes_pod_name
      - source_labels: [kubernetes_pod_name]
        action: replace
        target_label: io_kubernetes_pod_name

    - job_name: 'kubernetes-node-exporter'
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      - api_servers:
        - https://kubernetes.default.svc
        in_cluster: true
        role: node

      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - source_labels: [__meta_kubernetes_role]
        action: replace
        target_label: kubernetes_role
      - source_labels: [__address__]
        regex: '(.*):10250'
        replacement: '${1}:9100'
        target_label: __address__
      - source_labels: [__meta_kubernetes_node_label_kubernetes_io_hostname]
        target_label: __instance__
      # set "name" value to "job"
      - source_labels: [job]
        regex: 'kubernetes-(.*)'
        replacement: '${1}'
        target_label: name

    # Scrape config for service endpoints.
    #
    # The relabeling allows the actual service scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/scrape`: Only scrape services that have a value of `true`
    # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
    # to set this to `https` & most likely set the `tls_config` of the scrape config.
    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
    # * `prometheus.io/port`: If the metrics are exposed on a different port to the
    # service then set this appropriately.
    - job_name: 'kubernetes-service-endpoints'

      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: endpoint

      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

    # Example scrape config for probing services via the Blackbox Exporter.
    #
    # The relabeling allows the actual service scrape endpoint to be configured
    # via the following annotations:
    #
    # * `prometheus.io/probe`: Only probe services that have a value of `true`
    - job_name: 'kubernetes-services'

      metrics_path: /probe
      params:
        module: [http_2xx]

      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: service

      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_service_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name

    # Example scrape config for pods
    #
    # The relabeling allows the actual pod scrape endpoint to be configured via the
    # following annotations:
    #
    # * `prometheus.io/scrape`: Only scrape pods that have a value of `true`
    # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
    # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the default of `9102`.
    - job_name: 'kubernetes-pods'

      kubernetes_sd_configs:
      - api_servers:
        - 'https://kubernetes.default.svc'
        in_cluster: true
        role: pod

      relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: (.+):(?:\d+);(\d+)
        replacement: ${1}:${2}
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_pod_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name