Merge pull request #197 from stackhpc/fix/elasticsearch

Replace opendistro

ansible/.gitignore

@@ -4,8 +4,8 @@ roles/*
 # Whitelist roles that are checked into this repository.
 !roles/filebeat/
 !roles/filebeat/**
-!roles/opendistro/
-!roles/opendistro/**
+!roles/opensearch/
+!roles/opensearch/**
 !roles/podman/
 !roles/podman/**
 !roles/grafana-dashboards/

ansible/monitoring.yml

@@ -1,19 +1,13 @@
 # ---
 # # NOTE: Requires slurmdbd
 
-- name: Setup elasticsearch
-  hosts: opendistro
-  tags: opendistro
+- name: Setup OpenSearch
+  hosts: opensearch
+  tags: opensearch
   tasks:
     - import_role:
-        name: opendistro
-        tasks_from: config.yml
-      tags: config
-
-    - import_role:
-        name: opendistro
-        tasks_from: deploy.yml
-      tags: deploy
+        name: opensearch
+      become: true
 
 - name: Setup slurm stats
   hosts: slurm_stats

ansible/roles/filebeat/defaults/main.yml

@@ -2,3 +2,4 @@
 
 #filebeat_config_path: undefined # REQUIRED. Path to filebeat.yml configuration file template
 filebeat_podman_user: "{{ ansible_user }}" # User that runs the filebeat container
+filebeat_version: 7.12.1 # latest usable with opensearch - see https://opensearch.org/docs/2.4/tools/index/#compatibility-matrix-for-beats

ansible/roles/filebeat/templates/filebeat.service.j2

@@ -12,7 +12,19 @@ After=network-online.target
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
-ExecStart=/usr/bin/podman run --network slirp4netns:cidr={{ podman_cidr }} --sdnotify=conmon --cgroups=no-conmon --replace --name filebeat --user root --restart=always --security-opt label=disable --volume /var/log/:/logs:ro --volume /etc/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro --detach=True docker.elastic.co/beats/filebeat-oss:7.9.3 -e -strict.perms=false -d "*"
+ExecStart=/usr/bin/podman run \
+    --network=host \
+    --sdnotify=conmon \
+    --cgroups=no-conmon \
+    --replace \
+    --name filebeat \
+    --user root \
+    --restart=always \
+    --security-opt label=disable \
+    --volume /var/log/:/logs:ro \
+    --volume /etc/filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro \
+    --detach=True docker.elastic.co/beats/filebeat-oss:{{ filebeat_version }} \
+    -e -strict.perms=false -d "*"
 ExecStop=/usr/bin/podman stop --ignore filebeat -t 10
 ExecStopPost=/usr/bin/podman rm --ignore -f filebeat
 KillMode=none

ansible/roles/mysql/templates/mysql.service.j2

@@ -15,13 +15,17 @@ EnvironmentFile=/etc/sysconfig/mysqld
 ExecStartPre=+install -d -o {{ mysql_podman_user }} -g {{ mysql_podman_user }} -Z container_file_t {{ mysql_datadir }}
 ExecStartPre=+chown -R {{ mysql_podman_user }}:{{ mysql_podman_user }} {{ mysql_datadir }}
 ExecStart=/usr/bin/podman run \
-    --network slirp4netns:cidr={{ podman_cidr }} \
-    --sdnotify=conmon --cgroups=no-conmon \
-    --detach --replace --name mysql --restart=no \
+    --network=host \
+    --sdnotify=conmon \
+    --cgroups=no-conmon \
+    --detach \
+    --replace \
+    --name mysql \
+    --restart=no \
     --user mysql \
     --volume {{ mysql_datadir }}:/var/lib/mysql:U \
     --publish 3306:3306 \
-    -e MYSQL_ROOT_PASSWORD=${MYSQL_INITIAL_ROOT_PASSWORD} \
+    --env MYSQL_ROOT_PASSWORD=${MYSQL_INITIAL_ROOT_PASSWORD} \
     mysql:{{ mysql_tag }}{%- for opt in mysql_mysqld_options %} \
     --{{ opt }}{% endfor %}
 

ansible/roles/opensearch/defaults/main.yml

@@ -0,0 +1,11 @@
+---
+# Used to set passwords
+#opensearch_internal_users_path:
+
+opensearch_podman_user: "{{ ansible_user }}"
+opensearch_version: '2.4.0' # https://hub.docker.com/r/opensearchproject/opensearch/tags
+opensearch_config_path: /usr/share/opensearch/config
+opensearch_data_path: /usr/share/opensearch/data
+opensearch_state: started # will be restarted if required
+opensearch_systemd_service_enabled: true
+opensearch_certs_duration: "{{ 365 * 10 }}" # days validity for self-signed certs

ansible/roles/opensearch/handlers/main.yml

@@ -0,0 +1,9 @@
+---
+
+- name: Restart opensearch service
+  systemd:
+    name: opensearch.service
+    state: "{{ 'restarted' if 'started' in opensearch_state else opensearch_state }}"
+    enabled: "{{ opensearch_systemd_service_enabled }}"
+    daemon_reload: "{{ 'started' in opensearch_state }}"
+  become: true

ansible/roles/opensearch/tasks/certs.yml

@@ -0,0 +1,70 @@
+- name: Ensure host certs directory exists
+  file:
+    path: "{{ opensearch_config_path }}/certs"
+    state: directory
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    mode: ug=rwx,o=
+
+# Cert generation based on https://opensearch.org/docs/latest/security-plugin/configuration/generate-certificates/
+
+- name: Generate root private key
+  community.crypto.openssl_privatekey:
+    path: "{{ opensearch_config_path }}/certs/root-ca-key.pem"
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    mode: ug=rw,o=
+    return_content: false
+
+- name: Generate root CSR
+  # Required as cert has to have a subject
+  community.crypto.openssl_csr_pipe:
+    privatekey_path: "{{ opensearch_config_path }}/certs/root-ca-key.pem"
+    country_name: XX
+  register: _opensearch_root_csr
+
+- name: Generate root cert
+  community.crypto.x509_certificate:
+    provider: selfsigned
+    selfsigned_not_after: "+{{ opensearch_certs_duration }}d"
+    ignore_timestamps: false # so will be regenerated when run if necessary
+    privatekey_path: "{{ opensearch_config_path }}/certs/root-ca-key.pem"
+    path: "{{ opensearch_config_path }}/certs/root-ca.pem"
+    csr_content: "{{ _opensearch_root_csr.csr }}"
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    mode: ug=rw,o=
+    return_content: false
+
+- name: Generate node private key in pkcs8 format
+  community.crypto.openssl_privatekey:
+    path: "{{ opensearch_config_path }}/certs/esnode-key.pem"
+    format: pkcs8
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    mode: ug=rw,o=
+    return_content: false
+
+- name: Generate node CSR
+  # Required as cert has to have a subject and subject_alt_name
+  # Actual subject_alt_name is irrelevant as using enforce_hostname_verification=false
+  community.crypto.openssl_csr_pipe:
+    privatekey_path: "{{ opensearch_config_path }}/certs/esnode-key.pem"
+    country_name: XX
+    subject_alt_name:
+      - "DNS:esnode.dns.a-record"
+  register: _opensearch_node_csr
+
+- name: Generate node cert
+  community.crypto.x509_certificate:
+    provider: ownca
+    ownca_not_after: "+{{ opensearch_certs_duration }}d"
+    ignore_timestamps: false # so will be regenerated when run if necessary
+    ownca_path: "{{ opensearch_config_path }}/certs/root-ca.pem"
+    ownca_privatekey_path: "{{ opensearch_config_path }}/certs/root-ca-key.pem"
+    path: "{{ opensearch_config_path }}/certs/esnode.pem"
+    csr_content: "{{ _opensearch_node_csr.csr }}"
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    mode: ug=rw,o=
+    return_content: false

ansible/roles/opensearch/tasks/main.yml

@@ -0,0 +1,85 @@
+---
+
+- name: Check for existing opendistro service
+  stat:
+    path: /etc/systemd/system/opendistro.service
+  register: _opensearch_opendistro_service
+
+- name: Migrate opendistro data
+  import_tasks:
+    file: migrate-opendistro.yml
+  when: _opensearch_opendistro_service.stat.exists
+
+- name: Remove opendistro service
+  file:
+    path: /etc/systemd/system/opendistro.service
+    state: absent
+
+- name: Increase maximum number of virtual memory maps
+  # see https://opensearch.org/docs/2.0/opensearch/install/important-settings/
+  ansible.posix.sysctl:
+    name: vm.max_map_count
+    value: '262144'
+    state: present
+    reload: yes
+  become: true
+
+- name: Ensure required opensearch host directories exist
+  file:
+    state: directory
+    path: "{{ item }}"
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    mode: 0770
+  become: true
+  loop:
+    - "{{ opensearch_config_path }}"
+    - "{{ opensearch_data_path }}"
+  when: "'started' in opensearch_state" # don't run during image build
+
+- name: Create certs
+  import_tasks: certs.yml
+  when: "'started' in opensearch_state" # don't run during image build
+
+- name: Template general configuration
+  ansible.builtin.template:
+    src: opensearch.yml.j2
+    dest: "{{ opensearch_config_path }}/opensearch.yml"
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    # NOTE: root user in container maps to user on host, so this will appear as
+    # owned by root in the container.
+    mode: 0660
+  notify: Restart opensearch service
+  become: true
+  when: "'started' in opensearch_state" # don't run during image build
+
+- name: Template internal user configuration
+  template:
+      src: "{{ opensearch_internal_users_path }}"
+      dest: "{{ opensearch_config_path }}/internal_users.yml"
+      owner: "{{ opensearch_podman_user }}"
+      group: "{{ opensearch_podman_user }}"
+      # NOTE: root user in container maps to user on host, so this will appear as
+      # owned by root in the container.
+      mode: 0660
+  notify: Restart opensearch service
+  become: true
+  when: "'started' in opensearch_state" # don't run during image build
+
+- name: Create systemd unit file
+  template:
+    dest: /etc/systemd/system/opensearch.service
+    src: opensearch.service.j2
+  become: true
+  notify: Restart opensearch service
+
+- name: Flush handlers
+  meta: flush_handlers
+
+- name: Ensure opensearch service state
+  systemd:
+    name: opensearch.service
+    state: "{{ opensearch_state }}"
+    enabled: "{{ opensearch_systemd_service_enabled }}"
+  become: true

ansible/roles/opensearch/tasks/migrate-opendistro.yml

@@ -0,0 +1,28 @@
+# Migrate data from existing containerised opendistro v1.12.0 to containerised opensearch 2.1.0.
+#
+# This relies on:
+# - Both opendistro and opensearch using host directories for data. See `_default_opendistro_data_path` below
+# - Pre-upgrade group `opendistro` and current group `opensearch` containing the same host.
+#
+# NB: If `opendistro_data_path` was set to something non-default it MUST be set again in the `opensearch` group_vars,
+# as the `opendistro` group will not exist in the groups.
+
+# NB: This deliberately does not remove the opendistro data - this could be done manually if required.
+
+- name: Stop opendistro
+  ansible.builtin.systemd:
+    name: opendistro.service
+    state: stopped
+    enabled: false
+
+- name: Copy opendistro data directory
+  ansible.builtin.copy:
+    remote_src: true
+    src: "{{ opendistro_data_path | default(_default_opendistro_data_path) }}"
+    dest: "{{ opensearch_data_path | dirname }}/" # copying a directory, so need to specify the parent for destination
+    owner: "{{ opensearch_podman_user }}"
+    group: "{{ opensearch_podman_user }}"
+    mode: 0770
+  vars:
+    # from environments/common/inventory/group_vars/all/opendistro.yml:
+    _default_opendistro_data_path: "{{ appliances_state_dir | default('/usr/share') }}/elasticsearch/data"