Merge branch 'master' into alexy_graphing

.ansible-lint

@@ -6,6 +6,7 @@ skip_list:
   - command-instead-of-module  # Using command rather than module
   - galaxy
   - meta-no-info  # meta/main.yml should contain relevant info
+  - name[template]  # Jinja templates should only be at the end of 'name'
   - risky-shell-pipe  # Shells that use pipes should set the pipefail option
   - unnamed-task  # All tasks should be named
   - var-naming[no-role-prefix]  # Variables names from within roles should use role_name_ as a prefix

.zuul.yaml

@@ -10,6 +10,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
     required-projects: &required_projects
@@ -89,6 +90,7 @@
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling.yml"
         - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-metric-verification-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-power-monitoring.yml"
     roles:
       - zuul: github.com/openstack-k8s-operators/ci-framework
     required-projects: *required_projects
@@ -165,7 +167,7 @@
             override-checkout: main
             irrelevant-files: *irrelevant_files
         - functional-tests-on-osp18:
-            voting: false
+            voting: true
         - functional-logging-tests-osp18
         - functional-graphing-tests-osp18
         - functional-metric-verification-tests-osp18

ci/logging_tests_computes.yml

@@ -11,7 +11,14 @@
     journal_list:
       - ceilometer_agent_compute
       - nova_compute
+    common_file_test_id: "RHOSO-12754"
+    common_file_list:
+      - /etc/rsyslog.d/10-telemetry.conf
   tasks:
     - name: "Run Telemetry Logging tests"
       ansible.builtin.import_role:
         name: telemetry_logging
+
+    - name: "Run the tests from the common role"
+      ansible.builtin.import_role:
+        name: common

ci/logging_tests_controller.yml

@@ -7,7 +7,7 @@
     KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
     PATH: "{{ cifmw_path }}"
   vars:
-    common_pod_test_id: "RHOS0-12672"
+    common_pod_test_id: "RHOSO-12672"
     common_pod_status_str: "Running"
     common_pod_nspace: openstack-operators
     common_pod_list:
@@ -42,9 +42,6 @@
       - logging-loki-query-frontend-grpc
       - logging-loki-query-frontend-http
       - openstack-logging
-    common_file_test_id: "RHOSO-12754"
-    common_file_list:
-      - /etc/rsyslog.d/10-telemetry.conf
   tasks:
     - name: "Verify logging infrastructure components"
       ansible.builtin.import_role:

ci/run_autoscaling_osp18.yml

@@ -20,26 +20,6 @@
           tags:
             - setup
 
-        - name: Patch the openstackversions to use the master containers for aodh and heat
-          ansible.builtin.shell:
-            cmd: |
-              oc patch openstackversions controlplane --type merge --patch-file ci/patch-openstack-versions.yaml
-            chdir: "{{ fvt_dir }}"
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
-        - name: Redeploy the dataplane
-          ansible.builtin.shell:
-            cmd: |
-              oc get osdpd edpm-deployment -oyaml > /tmp/osdpd.yaml
-              oc delete osdpd edpm-deployment
-              sleep 10
-              oc apply -f /tmp/osdpd.yaml
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
         - name: Patch observabilityclient into openstackclient
           ansible.builtin.shell:
             cmd: |
@@ -50,50 +30,7 @@
           tags:
             - setup
 
-        - name: Wait until the oscp is resolved the changes to continue
-          ansible.builtin.shell:
-            cmd: |
-              oc get oscp | grep "Setup complete"
-          retries: 24
-          timeout: 5
-          until: output.stdout_lines | length == 1
-          register: output
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
-        - name: Wait until the osdpd is redeployed to continue
-          ansible.builtin.shell:
-            cmd: |
-              oc get osdpd | grep "Setup complete"
-          retries: 60
-          delay: 45
-          until: output.stdout_lines | length == 1
-          register: output
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-
         - name: "Run Telemetry Autoscaling tests"
           ansible.builtin.import_role:
             name: telemetry_autoscaling
           ignore_errors: true
-
-      always:
-        - name: Revert the version update
-          ansible.builtin.shell:
-            cmd: |
-              oc patch  openstackversions controlplane  --type json -p='[{"op": "replace", "path": "/spec/customContainerImages", "value": {} }]'
-          when: "{{ patch_openstackversions | bool }}"
-
-        - name: Redeploy the dataplane
-          ansible.builtin.shell:
-            cmd: |
-              oc get osdpd edpm-deployment -oyaml > /tmp/osdpd.yaml
-              oc delete osdpd edpm-deployment
-              sleep 10
-              oc apply -f /tmp/osdpd.yaml
-          when: "{{ patch_openstackversions | bool }}"
-          tags:
-            - setup
-

ci/use-master-containers.yml

@@ -0,0 +1,18 @@
+- name: Run telemetry autoscaling tests on osp18
+  hosts:  "{{ cifmw_target_hook_host | default('localhost')  }}"
+  gather_facts: true
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars_files:
+    - vars/common.yml
+    - vars/osp18_env.yml
+  tasks:
+    - name: Patch the openstackversions to use the master containers for aodh, heat and ceilometer
+      ansible.builtin.shell:
+        cmd: |
+          oc patch openstackversions controlplane --type merge --patch-file ci/patch-openstack-versions.yaml
+        chdir: "{{ fvt_dir }}"
+      tags:
+        - setup
+

ci/vars-functional-test.yml

@@ -1,5 +1,9 @@
 ---
-# temp: skip os-net-setup
+# This is added so we skip the os_net_setup role, which would normally be used
+# to create a network.
+# A network gets created by `make edpm_deploy_instance` during the autoscaling
+# tests.
+# skip os-net-setup
 cifmw_os_net_setup_config: []
 post_deploy_00_run_functional_test:
   source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/run_autoscaling_osp18.yml"

ci/vars-use-master-containers.yml

@@ -0,0 +1,4 @@
+---
+post_ctlplane_deploy_99_modify_openstackversions:
+  source: "{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/use-master-containers.yml"
+  type: playbook

roles/common/README.md

@@ -155,6 +155,7 @@ can be set at the play level.
                 - ceilometer_agent_compute
                 - ceilometer_agent_ipmi
                 - node_exporter
+                - kepler
 
       - name: "Verify Endpoint"
         ansible.builtin.import_role:

roles/common/tasks/container_test.yml

@@ -6,7 +6,9 @@
   changed_when: false
   register: container_status
 
-- name: Verify container status {{ common_container_test_id }}
+- name: |
+    TEST Verify {{ container_name }} container status
+    {{ common_container_test_id }}
   ansible.builtin.assert:
     that:
       - "'unhealthy' not in container_status.stdout"

roles/common/tasks/cr_tests.yml

@@ -1,4 +1,7 @@
-- name: Verify that a CR exists {{ common_cr_test_id }}
+---
+- name: |
+    TEST Verify that the {{ item.kind }} {{ item.name }} CR exists
+    {{ common_cr_test_id }}
   ansible.builtin.command:
     cmd: |
       oc get {{ item.kind }} {{ item.name }}

roles/common/tasks/crd_tests.yml

@@ -6,7 +6,9 @@
   changed_when: false
   register: output
 
-- name: Verify CRD is found "{{ common_crd_test_id }}"
+- name: |
+    TEST Verify CRD {{ item }} is found
+    {{ common_crd_test_id }}
   ansible.builtin.assert:
     that:
       - "'NotFound' not in output.stderr"

roles/common/tasks/endpoint_tests.yml

@@ -6,9 +6,11 @@
   changed_when: false
   register: output
 
-- name: Check that endpoint exist {{ common_endpoint_test_id }}
+- name: |
+    TEST Check that {{ item[0] }} {{ item[2] }} endpoint exists
+    {{ common_endpoint_test_id }}
   ansible.builtin.assert:
     that:
       - output.stdout != ""
     success_msg: "The endpoint {{ item[0] }} is found."
-    fail_msg: "The endpoint for service {{ item[0] }} "
+    fail_msg: "The endpoint for service {{ item[0] }} on the {{ item[2] }} interface was not found"

roles/common/tasks/file_tests.yml

@@ -3,8 +3,20 @@
   ansible.builtin.stat:
     path: "{{ item }}"
   register: fstats
-  failed_when: 
-    - fstats.stat.pw_name != "root"
-    - fstats.stat.size | int < 300
-    - not fstats.stat.exists
-    - not fstats.stat.isreg
+
+- name: |
+    TEST Verify the file {{ item }} exists
+    {{ common_file_test_id }}
+  ansible.builtin.assert:
+    that:
+      - fstats.stat.exists
+      - fstats.stat.pw_name == "root"
+      - fstats.stat.size | int > 300
+      - fstats.stat.isreg
+    fail_msg: |
+      File {{ item }} does not meet the required conditions:
+      {%- if not fstats.stat.exists %} file does not exist.{% endif %}
+      {%- if fstats.stat.exists and fstats.stat.pw_name != "root" %} file owner must be root (got {{ fstats.stat.pw_name }});{% endif %}
+      {%- if fstats.stat.exists and fstats.stat.size | int < 300 %} file must be larger than 300 bytes (got {{ fstats.stat.size }} bytes);{% endif %}
+      {%- if fstats.stat.exists and not fstats.stat.isreg | bool %} file must be a regular file.{% endif %}
+    success_msg: "The test passed"

roles/common/tasks/main.yml

@@ -1,5 +1,5 @@
 ---
-- name: "Verify pod - {{ common_pod_test_id }}"
+- name: "Run pod tests"
   when:
     - common_pod_test_id is defined
     - common_pod_list is defined

roles/common/tasks/manifest_tests.yml

@@ -13,10 +13,12 @@
   register: pack_name
   changed_when: false
 
-- name: Get packagemanifest "{{ common_manifest_test_id }}"
+- name: |
+    TEST Get {{ item }} packagemanifest
+    {{ common_manifest_test_id }}
   ansible.builtin.shell:
     cmd: |
       oc get packagemanifests | grep "{{ pack_name.stdout }}" | wc -l
   register: num_found 
   changed_when: false
-  failed_when: num_expected.stdout != num_found.stdout
+  failed_when: num_expected.stdout != num_found.stdout

roles/common/tasks/pod_tests.yml

@@ -11,7 +11,9 @@
         - podinstance.stdout_lines | length != 1
       changed_when: false
 
-    - name: Check pod {{ common_pod_test_id }}
+    - name: |
+        TEST Check {{ item }} pod is {{ common_pod_status_str }} in {{ common_pod_nspace }} namespace
+        {{ common_pod_test_id }}
       ansible.builtin.command:
         cmd: |
           oc get pod -n "{{ common_pod_nspace }}" "{{ podinstance.stdout }}"

roles/common/tasks/project_tests.yml

@@ -1,12 +1,14 @@
 ---
-- name: Verify project "{{ item }}"
+- name: Get the {{ item }} project
   ansible.builtin.shell:
     cmd: |
       oc get project "{{ item }}"
   changed_when: false
   register: output
 
-- name: Verify project is found "{{ common_project_test_id }}"
+- name: |
+   TEST Verify {{ item }} project exists
+   {{ common_project_test_id }}
   ansible.builtin.assert:
     that:
       - "'NotFound' not in output.stderr"

roles/common/tasks/service_tests.yml

@@ -7,7 +7,7 @@
   register: output
 
 - name: |
-    Verify {{ item }} service is running
+    TEST Verify {{ item }} service is running in {{ common_service_nspace }} namespace
     {{ common_service_test_id }}
   ansible.builtin.assert:
     that:

roles/common/tasks/subscription_tests.yml

@@ -6,7 +6,9 @@
   changed_when: false
   register: output
 
-- name: Check that subscription exist {{ common_subscription_test_id }}
+- name: |
+    TEST Check that {{ item }} subscription exists in {{ common_subscription_nspace }} namespace
+    {{ common_subscription_test_id }}
   ansible.builtin.assert:
     that:
       - "'NotFound' not in output.stderr"

roles/telemetry_autoscaling/tasks/test_autoscaling.yml

@@ -59,6 +59,30 @@
   register: busy_process
   with_items: "{{ vnf_instance_ip.stdout_lines }}"
 
+- name: Show ceilometer_cpu metrics from Prometheus
+  when: metrics_backend == "prometheus"
+  block:
+    - name: Register autoscaling query results
+      shell: |
+        export STACK_ID=$({{ openstack_cmd }} stack show {{ stack_name }} -c id -f value)
+        {{ openstack_cmd }} metric query '(rate(ceilometer_cpu{server_group=~"$STACK_ID"}[150s]))/10000000'
+      ignore_errors: true
+      register: result
+
+    - name: Show the query result
+      ansible.builtin.debug:
+        var: result
+
+    - name: Register ceilometer_cpu metrics
+      shell: |
+        {{ openstack_cmd }} metric show ceilometer_cpu --disable-rbac
+      ignore_errors: true
+      register: result
+
+    - name: Show the ceilometer_cpu metrics
+      ansible.builtin.debug:
+        var: result
+
 - name: |
     TEST Verify cpu high alarm has been triggered
     RHOSO-12660

roles/telemetry_logging/tasks/journal_tests.yml

@@ -1,6 +1,6 @@
 ---
 - name: |
-    Get journals {{ item }}
+    TEST Get journals {{ item }}
     {{ journal_test_id }}
   become: true
   ansible.builtin.shell:

roles/telemetry_verify_metrics/README.md

@@ -24,13 +24,16 @@ Tests:
 - Verify Ceilometer metrics are being exposed and stored
     - Use openstack observabilityclient to verify Ceilometer central metrics are stored in Prometheus
     - Use openstack observabilityclient to verify Ceilometer compute metrics are stored in Prometheus
+    - Use openstack observabilityclient to verify Ceilometer ipmi metrics are stored in Prometheus
 - Verify NodeExporter metrics are being exposed and stored
     - Use openstack observabilityclient to verify NodeExporter metrics are stored in Prometheus
+- Verify Kepler metrics are being exposed and stored
+    - Use openstack observabilityclient to verify Kepler metrics are stored in Prometheus
 
 Role Variables
 --------------
 openstack\_cmd - command to access openstack cli. For example: "oc rsh openstackclient openstack"
-telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, node\_exporter, rabbitmq
+telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler
 
 Example Playbook
 ----------------

roles/telemetry_verify_metrics/defaults/main.yml

@@ -4,3 +4,6 @@ telemetry_verify_metrics_metric_sources_to_test:
   - ceilometer_central_agent
   - node_exporter
   - rabbitmq
+  - kepler
+  # NOTE: IPMI metrics can only be verified when compute nodes are baremetal
+  # - ceilometer_ipmi_agent