support all used cloudalchemy.prometheus config for containerised pro…

…metheus

ansible/.gitignore

@@ -55,3 +55,5 @@ roles/*
 !roles/persist_hostkeys/
 !roles/persist_hostkeys/**
 !roles/requirements.yml
+!roles/prometheus/
+!roles/prometheus/**

ansible/roles/prometheus/README.md

@@ -0,0 +1,15 @@
+# prometheus
+Creates a systemd service `prometheus` which uses the `podman` user to run a containerised [Prometheus](https://github.com/prometheus/prometheus) monitoring system.
+
+Note this contains two task books:
+    - `install.yml`: This is safe to run during a Packer build. It pulls the container and creates the systemd unit file.
+    - `runtime.yml`: This cannot be run during a Packer build. It templates out config and restarts/starts the service as required.
+
+## TODO
+ - Check upgrading clusters from cloudalchemy.prometheus role works OK. The state dir will be owned by `prometheus` user, not `podman`, but this role should fix that.
+
+## Role Variables
+
+See `defaults/main.yml`. All variables can be updated by running `runtime.yml`, except the below which require `install.yml` to be run to update:
+- `prometheus_storage_retention_size`
+- `prometheus_storage_retention`

ansible/roles/prometheus/defaults/main.yml

@@ -1,3 +1,26 @@
 prometheus_container_image: "prom/prometheus"
 prometheus_version:  "v2.36.2"
-podman_service_user: "podman"
+
+# Variables below are compatible with the deprecated, non-containerised [cloudalchemy.prometheus](https://github.com/cloudalchemy/ansible-prometheus) role
+prometheus_alert_rules: []
+prometheus_alert_rules_files:
+  - prometheus/rules/*.rules
+prometheus_config_dir: /etc/prometheus
+prometheus_db_dir: /var/lib/prometheus
+prometheus_external_labels:
+  environment: "{{ ansible_fqdn | default(ansible_host) | default(inventory_hostname) }}"
+prometheus_global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+prometheus_remote_read: []
+prometheus_remote_write: []
+prometheus_scrape_configs: {}
+prometheus_storage_retention_size: "0GB"
+prometheus_storage_retention: "30d"
+prometheus_targets: []
+# See https://github.com/prometheus/exporter-toolkit/blob/master/docs/web-configuration.md
+prometheus_web_config:
+  tls_server_config: {}
+  http_server_config: {}
+  basic_auth_users: {}

ansible/roles/prometheus/handlers/main.yml

@@ -0,0 +1,5 @@
+- name: Restart Prometheus
+  systemd:
+    name: prometheus
+    enabled: true
+    state: restarted

ansible/roles/prometheus/tasks/install.yml

@@ -1,32 +1,31 @@
 ---
 
-- name: Ensure Prometheus config directory exists
-  file:
-    path: /etc/prometheus/
-    state: directory
-
 - name: Pull image for Prometheus
   containers.podman.podman_image:
     name: "{{ prometheus_container_image }}:{{ prometheus_version }}"
   become: yes
-  become_user: "{{ podman_service_user }}"
+  become_user: podman
 
 - name: Install systemd unit for Prometheus
   include_role:
     name: podman
     tasks_from: systemd-unit.yml
   vars:
-    podman_service_name: "prometheus"
+    podman_service_name: prometheus
     podman_service_type: container
-    podman_service_pod: "monitoring"
-    podman_service_wants: ["node_exporter"]
     podman_service_image: "{{ prometheus_container_image }}"
     podman_service_volumes:
-      - /etc/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - "{{ prometheus_data_directory }}:/prometheus:U"
-    podman_service_command: |
+      - "{{ prometheus_config_dir }}:/etc/prometheus/:ro"
+      - "{{ prometheus_db_dir }}:/prometheus:U"
+    podman_service_command: | # TODO: make retention time/size runtime configurable (can't be specified in config yml)
       --config.file=/etc/prometheus/prometheus.yml \
       --storage.tsdb.path=/prometheus \
       --web.console.libraries=/usr/share/prometheus/console_libraries \
       --web.console.templates=/usr/share/prometheus/consoles \
-      --storage.tsdb.retention.size=1GB
+      --storage.tsdb.retention.size={{ prometheus_storage_retention_size }} \
+      --storage.tsdb.retention.time={{ prometheus_storage_retention }}
+    podman_service_network: host
+
+- name: Reload Prometheus unit file
+  command: systemctl daemon-reload
+  when: podman_systemd_unit.changed

ansible/roles/prometheus/tasks/runtime.yml

@@ -0,0 +1,76 @@
+---
+# Based on cloudalchemy.prometheus/tasks/configure.yml so to support same configurations
+# Main changes for containerisation are:
+# - user/group of files should be podman
+# - cannot reload service
+
+- name: Ensure Prometheus host directories exists
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: podman
+    group: podman
+    recurse: true
+  loop:
+    - "{{ prometheus_config_dir }}"
+    - "{{ prometheus_config_dir }}/rules"
+    - "{{ prometheus_config_dir }}/file_sd"
+    - "{{ prometheus_db_dir }}" # this should be on a volume, so can't do during install==image build
+
+- name: Configure alerting rules file
+  template:
+    src: "alert.rules.j2"
+    dest: "{{ prometheus_config_dir }}/rules/ansible_managed.rules"
+    owner: podman
+    group: podman
+    mode: 0644
+    # validate: "{{ _prometheus_binary_install_dir }}/promtool check rules %s" # TODO: is in container, could use it
+  when: prometheus_alert_rules != []
+  notify: Restart Prometheus
+
+- name: Copy custom alerting rule files
+  copy:
+    src: "{{ item }}"
+    dest: "{{ prometheus_config_dir }}/rules/"
+    owner: podman
+    group: podman
+    mode: 0644
+    # validate: "{{ _prometheus_binary_install_dir }}/promtool check rules %s" # see above
+  with_fileglob: "{{ prometheus_alert_rules_files }}"
+  notify: Restart Prometheus
+
+- name: Configure Prometheus
+  template:
+    src: prometheus.yml.j2
+    dest: "{{ prometheus_config_dir }}/prometheus.yml"
+    #validate: "{{ _prometheus_binary_install_dir }}/promtool check config %s" see above
+  notify: Restart Prometheus
+
+- name: Configure Prometheus web
+  copy:
+    content: "{{ prometheus_web_config | to_nice_yaml(indent=2) }}"
+    dest: "{{ prometheus_config_dir }}/web.yml"
+    owner: podman
+    group: podman
+    mode: 0644
+
+- name: Configure prometheus static targets
+  copy:
+    content: |
+      #jinja2: lstrip_blocks: True
+      {{ item.value | to_nice_yaml(indent=2) }}
+    dest: "{{ prometheus_config_dir }}/file_sd/{{ item.key }}.yml"
+    force: true
+    owner: podman
+    group: podman
+    mode: 0644
+  with_dict: "{{ prometheus_targets }}"
+  when: prometheus_targets != {}
+
+- meta: flush_handlers # handles restarts due to config changes
+
+- name: Ensure Prometheus service state
+  systemd:
+    name: prometheus
+    enabled: true
+    state: "{{ 'restarted' if (podman_systemd_unit.changed | default(false)) else 'started' }}"

ansible/roles/prometheus/templates/prometheus.yml.j2

@@ -0,0 +1,34 @@
+#jinja2: trim_blocks: True, lstrip_blocks: True
+{{ ansible_managed | comment }}
+# http://prometheus.io/docs/operating/configuration/
+
+global:
+  {{ prometheus_global | to_nice_yaml(indent=2) | indent(2, False) }}
+  external_labels:
+    {{ prometheus_external_labels | to_nice_yaml(indent=2) | indent(4, False) }}
+
+{% if prometheus_remote_write != [] %}
+remote_write:
+  {{ prometheus_remote_write | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif %}
+
+{% if prometheus_remote_read != [] %}
+remote_read:
+  {{ prometheus_remote_read | to_nice_yaml(indent=2) | indent(2, False) }}
+{% endif %}
+
+rule_files:
+  - {{ prometheus_config_dir }}/rules/*.rules
+
+{% if prometheus_alertmanager_config | length > 0 %}
+alerting:
+  alertmanagers:
+  {{ prometheus_alertmanager_config | to_nice_yaml(indent=2) | indent(2,False) }}
+  {% if prometheus_alert_relabel_configs | length > 0 %}
+  alert_relabel_configs:
+  {{ prometheus_alert_relabel_configs | to_nice_yaml(indent=2) | indent(2,False) }}
+  {% endif %}
+{% endif %}
+
+scrape_configs:
+  {{ prometheus_scrape_configs | to_nice_yaml(indent=2) | indent(2,False) }}

environments/common/inventory/group_vars/all/prometheus.yml

@@ -3,7 +3,7 @@
 # See: https://github.com/cloudalchemy/ansible-prometheus
 # for variable definitions
 
-prometheus_version: 2.27.0 # default from ansible/roles/cloudalchemy.prometheus/defaults/main.yml
+prometheus_version: "v2.36.2"
 prometheus_web_external_url: "http://{{ prometheus_address }}:9090"
 prometheus_storage_retention: "31d"
 prometheus_storage_retention_size: "100GB"