add extended testing for k3s - check if pods has started

molecule/airgapped-node/converge.yml

@@ -30,3 +30,9 @@
             jump: DROP
   roles:
     - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
+
+  post_tasks:
+    - name: Wait for servicelb port
+      ansible.builtin.wait_for:
+        port: 80
+        delay: 10

molecule/cluster/converge.yml

@@ -1,16 +1,9 @@
 ---
 - name: Converge
   hosts: all
-  vars:
-    k3s_agent_additional_config:
-      snapshotter: native
-    k3s_master_additional_config:
-      snapshotter: native
-    k3s_extra_env:
-      - INVOCATION_ID=
 
   pre_tasks:
-    - name: Update apt cache.
+    - name: Update apt cache
       ansible.builtin.apt:
         update_cache: true
         cache_valid_time: 600
@@ -19,3 +12,9 @@
 
   roles:
     - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
+
+  post_tasks:
+    - name: Wait for servicelb port
+      ansible.builtin.wait_for:
+        port: 80
+        delay: 10

molecule/cluster/molecule.yml

@@ -61,7 +61,15 @@ provisioner:
     group_vars:
       k3s_master:
         k3s_master: true
+        k3s_master_additional_config:
+          snapshotter: native
+        k3s_extra_env:
+          - INVOCATION_ID=
       k3s_agent:
         k3s_agent: true
+        k3s_agent_additional_config:
+          snapshotter: native
+        k3s_extra_env:
+          - INVOCATION_ID=
   playbooks:
     converge: ${MOLECULE_PLAYBOOK:-converge.yml}

molecule/cluster/tests/test-k3s-lb.yml

@@ -0,0 +1,13 @@
+addr:
+  tcp://master-1:80:
+    title: k3s-servicelb port on master-1 is reachable
+    reachable: true
+  tcp://master-2:80:
+    title: k3s-servicelb port on master-2 is reachable
+    reachable: true
+  tcp://master-3:80:
+    title: k3s-servicelb port on master-3 is reachable
+    reachable: true
+  tcp://node-1:80:
+    title: k3s-servicelb port on node-1 is reachable
+    reachable: true

molecule/cluster/verify.yml

@@ -0,0 +1,96 @@
+---
+- name: Verify
+  hosts: all
+  become: true
+  vars:
+    goss_version: v0.4.4
+    goss_arch_table:
+      aarch64:
+        arch: arm64
+        checksum: "sha256:fd76103dd6f373d48317b19542e4556cd27a06d415638561d410dfc139af19c8"
+      x86_64:
+        arch: amd64
+        checksum: "sha256:1c4f54b22fde9d4d5687939abc2606b0660a5d14a98afcd09b04b793d69acdc5"
+    goss_bin: /usr/local/bin/goss
+    goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-{{ ansible_system | lower }}-{{ goss_arch }}"
+    goss_test_directory: /tmp/molecule/goss
+    goss_format: tap
+  tasks:
+    - name: Detect host system architecture
+      ansible.builtin.set_fact:
+        goss_arch: "{{ goss_arch_table[ansible_architecture].arch }}"
+
+    - name: Download and install Goss
+      ansible.builtin.get_url:
+        url: "{{ goss_url }}"
+        dest: "{{ goss_bin }}"
+        checksum: "{{ goss_arch_table[ansible_architecture].checksum }}"
+        mode: "u=rwx,go=rx"
+      register: download_goss
+      until: download_goss is succeeded
+      retries: 3
+
+    - name: Create Molecule directory for test files
+      ansible.builtin.file:
+        path: "{{ goss_test_directory }}"
+        state: directory
+        mode: "0755"
+
+    - name: Find Goss tests on localhost
+      ansible.builtin.find:
+        paths: "{{ lookup('env', 'MOLECULE_VERIFIER_TEST_DIRECTORY') }}"
+        patterns:
+          - "test[-.\\w]*.yml"
+          - "test_host_{{ ansible_hostname }}[-.\\w]*.yml"
+        excludes:
+          - "test_host_(?!{{ ansible_hostname }})[-.\\w]*.yml"
+        use_regex: true
+      delegate_to: localhost
+      register: test_files
+      changed_when: false
+      become: false
+
+    - name: Copy Goss tests to remote
+      ansible.builtin.copy:
+        src: "{{ item.path }}"
+        dest: "{{ goss_test_directory }}/{{ item.path | basename }}"
+        mode: "0644"
+      with_items:
+        - "{{ test_files.files }}"
+      loop_control:
+        label: "{{ item.path | basename }}"
+
+    - name: Register test files
+      ansible.builtin.find:
+        paths:
+          - "{{ goss_test_directory }}"
+        patterns:
+          - "test_*.yml"
+          - "test-*.yml"
+      register: test_files
+
+    - name: Run verify
+      when: test_files is succeeded
+      block:
+        - name: Execute Goss tests # noqa: no-changed-when
+          ansible.builtin.command: "{{ goss_bin }} -g {{ item }} validate --format {{ goss_format }}"
+          register: test_results
+          with_items: "{{ test_files.files | map(attribute='path') | list }}"
+          loop_control:
+            label: "{{ item | basename }}"
+          failed_when: false
+
+        - name: Display details about the Goss results
+          ansible.builtin.debug:
+            msg: "{{ item.stdout_lines }}"
+          with_items: "{{ test_results.results }}"
+          loop_control:
+            label: "{{ item[item.ansible_loop_var] | basename }}"
+
+        - name: Fail when tests fail
+          ansible.builtin.fail:
+            msg: "Goss failed to validate"
+          when: item.rc != 0
+          with_items: "{{ test_results.results }}"
+          loop_control:
+            label: "{{ item[item.ansible_loop_var] | basename }}"

molecule/extended-settings/converge.yml

@@ -1,44 +1,9 @@
 ---
 - name: Converge
   hosts: all
-  vars:
-    k3s_agent_additional_config:
-      snapshotter: native
-    k3s_master_additional_config:
-      snapshotter: native
-    k3s_extra_env:
-      - INVOCATION_ID=
-    # test adding additonal manifests
-    k3s_additional_manifests:
-      - name: kata
-        state: present
-        definition:
-          apiVersion: node.k8s.io/v1
-          kind: RuntimeClass
-          metadata:
-            name: kata
-          handler: kata
-    # test adding additional configfiles
-    k3s_additional_config_files:
-      - name: apiserver-tracing.yaml
-        content: |
-          apiVersion: apiserver.config.k8s.io/v1alpha1
-          kind: TracingConfiguration
-          endpoint: 127.0.0.1:4317
-          samplingRatePerMillion: 100
-    k3s_kubelet_additional_config:
-      - "cloud-provider=external"
-    k3s_additional_packages:
-      - open-iscsi
-    k3s_gvisor: true
-    k3s_gvisor_config:
-      network: host
-    k3s_server_disable:
-      - metrics-server
-      - traefik
 
   pre_tasks:
-    - name: Update apt cache.
+    - name: Update apt cache
       ansible.builtin.apt:
         update_cache: true
         cache_valid_time: 600
@@ -47,3 +12,9 @@
 
   roles:
     - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
+
+  post_tasks:
+    - name: Wait for servicelb port
+      ansible.builtin.wait_for:
+        port: 80
+        delay: 10

molecule/extended-settings/molecule.yml

@@ -23,7 +23,35 @@ provisioner:
     group_vars:
       k3s_master:
         k3s_master: true
-      k3s_agent:
-        k3s_agent: true
+        k3s_master_additional_config:
+          snapshotter: native
+        k3s_extra_env:
+          - INVOCATION_ID=
+        # test adding additonal manifests
+        k3s_additional_manifests:
+          - name: kata
+            state: present
+            definition:
+              apiVersion: node.k8s.io/v1
+              kind: RuntimeClass
+              metadata:
+                name: kata
+              handler: kata
+        # test adding additional configfiles
+        k3s_additional_config_files:
+          - name: apiserver-tracing.yaml
+            content: |
+              apiVersion: apiserver.config.k8s.io/v1alpha1
+              kind: TracingConfiguration
+              endpoint: 127.0.0.1:4317
+              samplingRatePerMillion: 100
+        k3s_additional_packages:
+          - open-iscsi
+        k3s_gvisor: true
+        k3s_gvisor_config:
+          network: host
+        k3s_master_disable:
+          - metrics-server
+
   playbooks:
     converge: ${MOLECULE_PLAYBOOK:-converge.yml}

molecule/extended-settings/tests/test-k3s-lb.yml

@@ -0,0 +1,4 @@
+addr:
+  tcp://instance:80:
+    title: k3s-servicelb port is reachable
+    reachable: true

molecule/extended-settings/verify.yml

@@ -0,0 +1,96 @@
+---
+- name: Verify
+  hosts: all
+  become: true
+  vars:
+    goss_version: v0.4.4
+    goss_arch_table:
+      aarch64:
+        arch: arm64
+        checksum: "sha256:fd76103dd6f373d48317b19542e4556cd27a06d415638561d410dfc139af19c8"
+      x86_64:
+        arch: amd64
+        checksum: "sha256:1c4f54b22fde9d4d5687939abc2606b0660a5d14a98afcd09b04b793d69acdc5"
+    goss_bin: /usr/local/bin/goss
+    goss_url: "https://github.com/aelsabbahy/goss/releases/download/{{ goss_version }}/goss-{{ ansible_system | lower }}-{{ goss_arch }}"
+    goss_test_directory: /tmp/molecule/goss
+    goss_format: tap
+  tasks:
+    - name: Detect host system architecture
+      ansible.builtin.set_fact:
+        goss_arch: "{{ goss_arch_table[ansible_architecture].arch }}"
+
+    - name: Download and install Goss
+      ansible.builtin.get_url:
+        url: "{{ goss_url }}"
+        dest: "{{ goss_bin }}"
+        checksum: "{{ goss_arch_table[ansible_architecture].checksum }}"
+        mode: "u=rwx,go=rx"
+      register: download_goss
+      until: download_goss is succeeded
+      retries: 3
+
+    - name: Create Molecule directory for test files
+      ansible.builtin.file:
+        path: "{{ goss_test_directory }}"
+        state: directory
+        mode: "0755"
+
+    - name: Find Goss tests on localhost
+      ansible.builtin.find:
+        paths: "{{ lookup('env', 'MOLECULE_VERIFIER_TEST_DIRECTORY') }}"
+        patterns:
+          - "test[-.\\w]*.yml"
+          - "test_host_{{ ansible_hostname }}[-.\\w]*.yml"
+        excludes:
+          - "test_host_(?!{{ ansible_hostname }})[-.\\w]*.yml"
+        use_regex: true
+      delegate_to: localhost
+      register: test_files
+      changed_when: false
+      become: false
+
+    - name: Copy Goss tests to remote
+      ansible.builtin.copy:
+        src: "{{ item.path }}"
+        dest: "{{ goss_test_directory }}/{{ item.path | basename }}"
+        mode: "0644"
+      with_items:
+        - "{{ test_files.files }}"
+      loop_control:
+        label: "{{ item.path | basename }}"
+
+    - name: Register test files
+      ansible.builtin.find:
+        paths:
+          - "{{ goss_test_directory }}"
+        patterns:
+          - "test_*.yml"
+          - "test-*.yml"
+      register: test_files
+
+    - name: Run verify
+      when: test_files is succeeded
+      block:
+        - name: Execute Goss tests # noqa: no-changed-when
+          ansible.builtin.command: "{{ goss_bin }} -g {{ item }} validate --format {{ goss_format }}"
+          register: test_results
+          with_items: "{{ test_files.files | map(attribute='path') | list }}"
+          loop_control:
+            label: "{{ item | basename }}"
+          failed_when: false
+
+        - name: Display details about the Goss results
+          ansible.builtin.debug:
+            msg: "{{ item.stdout_lines }}"
+          with_items: "{{ test_results.results }}"
+          loop_control:
+            label: "{{ item[item.ansible_loop_var] | basename }}"
+
+        - name: Fail when tests fail
+          ansible.builtin.fail:
+            msg: "Goss failed to validate"
+          when: item.rc != 0
+          with_items: "{{ test_results.results }}"
+          loop_control:
+            label: "{{ item[item.ansible_loop_var] | basename }}"

molecule/single-node/converge.yml

@@ -10,7 +10,7 @@
       - INVOCATION_ID=
 
   pre_tasks:
-    - name: Update apt cache.
+    - name: Update apt cache
       ansible.builtin.apt:
         update_cache: true
         cache_valid_time: 600
@@ -19,3 +19,9 @@
 
   roles:
     - role: "{{ lookup('env', 'MOLECULE_PROJECT_DIRECTORY') | basename }}"
+
+  post_tasks:
+    - name: Wait for servicelb port
+      ansible.builtin.wait_for:
+        port: 80
+        delay: 10

molecule/single-node/molecule.yml

@@ -23,7 +23,5 @@ provisioner:
     group_vars:
       k3s_master:
         k3s_master: true
-      k3s_agent:
-        k3s_agent: true
   playbooks:
     converge: ${MOLECULE_PLAYBOOK:-converge.yml}

molecule/single-node/tests/test-k3s-lb.yml

@@ -0,0 +1,4 @@
+addr:
+  tcp://instance:80:
+    title: k3s-servicelb port is reachable
+    reachable: true