Support for uperf multi-pair vm runs, adjust vm requirements to reflect usage

Author: sarahbx

docs/uperf.md

@@ -209,14 +209,14 @@ You must have configured your k8s cluster with [Kubevirt](https://kubevirt.io) p
 ```yaml
 server_vm:
   dedicatedcpuplacement: false # cluster would need have the CPUManager feature enabled
-  sockets: 2
-  cores: 1
+  sockets: 1
+  cores: 2
   threads: 1
   image: kubevirt/fedora-cloud-container-disk-demo:latest # your image must've ethtool installed if enabling multiqueue
   limits:
-    memory: 2Gi
+    memory: 4Gi
   requests:
-    memory: 2Gi
+    memory: 4Gi
   network:
     front_end: bridge # or masquerade
     multiqueue:
@@ -227,14 +227,14 @@ server_vm:
     #- hostpassthrough
 client_vm:
   dedicatedcpuplacement: false # cluster would need have the CPUManager feature enabled
-  sockets: 2
-  cores: 1
+  sockets: 1
+  cores: 2
   threads: 1
   image: kubevirt/fedora-cloud-container-disk-demo:latest # your image must've ethtool installed if enabling multiqueue
   limits:
-    memory: 2Gi
+    memory: 4Gi
   requests:
-    memory: 2Gi
+    memory: 4Gi
   network:
     front_end: bridge # or masquerade
     multiqueue:

resources/crds/ripsaw_v1alpha1_uperf_vm.yaml

@@ -20,18 +20,18 @@ spec:
       pin_server: "node-0"
       pin_client: "node-1"
       samples: 2
-      pair: 1 # has to be one 1 for vm
+      pair: 1
       kind: vm
       server_vm:
         dedicatedcpuplacement: false
-        sockets: 2
-        cores: 1
+        sockets: 1
+        cores: 2
         threads: 1
         image: kubevirt/fedora-cloud-container-disk-demo:latest # your image must've ethtool installed if enabling multiqueue
         limits:
-          memory: 2Gi
+          memory: 4Gi
         requests:
-          memory: 2Gi
+          memory: 4Gi
         network:
           front_end: bridge # or masquerade
           multiqueue:
@@ -42,14 +42,14 @@ spec:
           #- hostpassthrough
       client_vm:
         dedicatedcpuplacement: false
-        sockets: 2
-        cores: 1
+        sockets: 1
+        cores: 2
         threads: 1
         image: kubevirt/fedora-cloud-container-disk-demo:latest
         limits:
-          memory: 2Gi
+          memory: 4Gi
         requests:
-          memory: 2Gi
+          memory: 4Gi
         network:
           front_end: bridge # or masquerade
           multiqueue:
@@ -59,15 +59,11 @@ spec:
           - none
           #- hostpassthrough
       test_types:
-        #- stream
-        - rr
+        - stream
       protos:
         - tcp
-        #- udp
       sizes:
-        #- 1024
-        - 512
+        - 16384
       nthrs:
-        #- 1
-        - 2
-      runtime: 5
+        - 1
+      runtime: 30

roles/uperf/tasks/main.yml

@@ -153,11 +153,11 @@
     when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int  == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)"
 
   - name: blocking client from running uperf
-    command: "redis-cli set {{ trunc_uuid }} false"
+    command: "redis-cli set start false"
     with_items: "{{ server_vms.resources }}"
     when: "workload_args.pair|default('1')|int == server_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int  == (server_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)"
 
-  when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm" and workload_args.pair|default('1')|int|int == 1
+  when: resource_state.resources[0].status.state == "Starting Servers" and resource_kind == "vm"
 
 - block:
 
@@ -200,7 +200,6 @@
           - type = uperf-bench-server-{{ trunc_uuid }}
       register: server_vms
 
-
     - name: Generate uperf test files
       k8s:
         definition: "{{ lookup('template', 'configmap_script.yml.j2') | from_yaml }}"
@@ -253,27 +252,22 @@
     - name: set complete to false
       command: "redis-cli set complete false"
 
-    - name: Get client vm status
-      k8s_facts:
-        kind: VirtualMachineInstance
-        api_version: kubevirt.io/v1alpha3
-        namespace: '{{ operator_namespace }}'
-        label_selectors:
-          - app = uperf-bench-client-{{ trunc_uuid }}
-      register: client_vms
-
-    - name: Update resource state
-      operator_sdk.util.k8s_status:
-        api_version: ripsaw.cloudbulldozer.io/v1alpha1
-        kind: Benchmark
-        name: "{{ meta.name }}"
-        namespace: "{{ operator_namespace }}"
-        status:
-          state: Clients Running
-      when: "workload_args.pair|default('1')|int == client_vms | json_query('resources[].status[]')|selectattr('phase','match','Running')|list|length and workload_args.pair|default('1')|int  == (client_vms | json_query('resources[].status.interfaces[0].ipAddress')|length)"
+    - name: Get count of clients ready
+      command: "redis-cli get clients-{{ trunc_uuid }}"
+      register: clients_ready_count
 
     when: resource_kind == "vm"
 
+  - name: Update resource state
+    operator_sdk.util.k8s_status:
+      api_version: ripsaw.cloudbulldozer.io/v1alpha1
+      kind: Benchmark
+      name: "{{ meta.name }}"
+      namespace: "{{ operator_namespace }}"
+      status:
+        state: Clients Running
+    when: "workload_args.pair|default('1')|int == clients_ready_count.stdout|int"
+
   when: resource_state.resources[0].status.state == "Waiting for Clients"
 
 - block:
@@ -365,6 +359,12 @@
       when: cleanup
     when: resource_kind == "pod"
 
+  - name: delete redis keys
+    command: "redis-cli del {{ item }}"
+    loop:
+      - "{{ trunc_uuid }}"
+      - "clients-{{ trunc_uuid }}"
+
   - operator_sdk.util.k8s_status:
       api_version: ripsaw.cloudbulldozer.io/v1alpha1
       kind: Benchmark

roles/uperf/templates/configmap_script.yml.j2

@@ -23,8 +23,14 @@ data:
     export clustername={{clustername}}
     export hostnet={{workload_args.hostnetwork}}
     export ips=$(hostname -I)
+    echo "Setting ready, waiting for signal to start..."
+    redis-cli -h {{bo.resources[0].status.podIP}} setnx clients-{{trunc_uuid}} 0
+    redis-cli -h {{bo.resources[0].status.podIP}} incr clients-{{trunc_uuid}}
     while true; do
-      if [[ ($(redis-cli -h {{bo.resources[0].status.podIP}} get start) =~ 'true') && ($(redis-cli -h {{bo.resources[0].status.podIP}} get {{ trunc_uuid }}) =~ 'true') ]]; then
+      BO_START=$(redis-cli -h {{bo.resources[0].status.podIP}} get start)
+      CLIENTS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get clients-{{ trunc_uuid }})
+      SERVERS_READY=$(redis-cli -h {{bo.resources[0].status.podIP}} get {{ trunc_uuid }})
+      if [[ ("${BO_START}" =~ 'true') && ("${CLIENTS_READY}" == "${SERVERS_READY}") ]]; then
 {% for test in workload_args.test_types %}
 {% for proto in workload_args.protos %}
 {% for size in workload_args.sizes %}
@@ -43,9 +49,11 @@ data:
 {% endfor %}
 {% endfor %}
       else
+        sleep 0.1;
         continue;
       fi;
       break;
     done;
     redis-cli -h {{bo.resources[0].status.podIP}} set start false
+    redis-cli -h {{bo.resources[0].status.podIP}} del clients-{{trunc_uuid}}
     redis-cli -h {{bo.resources[0].status.podIP}} set complete true

roles/uperf/templates/server_vm.yml.j2

@@ -59,7 +59,8 @@ spec:
           - ethtool -L eth0 combined {{ workload_args.server_vm.network.multiqueue.queues }}
 {% endif %}
           - dnf install -y uperf redis git
-          - redis-cli -h {{ bo.resources[0].status.podIP }} set {{ trunc_uuid }} true
+          - redis-cli -h {{ bo.resources[0].status.podIP }} setnx {{ trunc_uuid }} 0
+          - redis-cli -h {{ bo.resources[0].status.podIP }} incr {{ trunc_uuid }}
           - uperf -s -v -P 20000
     name: cloudinitdisk
 status: {}