From 2362b5ef05e8af3c025a73b04de93b6e4ca84d8d Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 07:56:31 +0000 Subject: [PATCH 1/9] support multiple partitions in hpctests --- ansible/roles/hpctests/README.md | 3 ++- ansible/roles/hpctests/defaults/main.yml | 1 + ansible/roles/hpctests/library/slurm_node_info.py | 3 +-- ansible/roles/hpctests/tasks/hpl-solo.yml | 9 +++++++-- ansible/roles/hpctests/tasks/setup.yml | 12 +++++++++++- ansible/roles/hpctests/templates/hpl-build.sh.j2 | 1 + ansible/roles/hpctests/templates/hpl-solo.sh.j2 | 1 + ansible/roles/hpctests/templates/pingmatrix.sh.j2 | 1 + ansible/roles/hpctests/templates/pingpong.sh.j2 | 1 + 9 files changed, 26 insertions(+), 6 deletions(-) diff --git a/ansible/roles/hpctests/README.md b/ansible/roles/hpctests/README.md index f85d65c07..9d721c7f3 100644 --- a/ansible/roles/hpctests/README.md +++ b/ansible/roles/hpctests/README.md @@ -24,7 +24,8 @@ Role Variables -------------- - `hpctests_rootdir`: Required. Path to root of test directory tree, which must be on a r/w filesystem shared to all cluster nodes under test. The last directory component will be created. -- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the default partition are used. Note nodes selected **must** be in the default partition. +- `hpctests_partition`: Optional. Name of partition to use, otherwise default partition is used. +- `hpctests_nodes`: Optional. A Slurm node expression, e.g. `'compute-[0-15,19]'` defining the nodes to use. If not set all nodes in the selected partition are used. - `hpctests_ucx_net_devices`: Optional. Control which network device/interface to use, e.g. `mlx5_1:0`. The default of `all` (as per UCX) may not be appropriate for multi-rail nodes with different bandwidths on each device. See [here](https://openucx.readthedocs.io/en/master/faq.html#what-is-the-default-behavior-in-a-multi-rail-environment) and [here](https://github.com/openucx/ucx/wiki/UCX-environment-parameters#setting-the-devices-to-use). - `hpctests_outdir`: Optional. Directory to use for test output on local host. Defaults to `$HOME/hpctests` (for local user). - `hpctests_hpl_NB`: Optional, default 192. The HPL block size "NB" - for Intel CPUs see [here](https://software.intel.com/content/www/us/en/develop/documentation/onemkl-linux-developer-guide/top/intel-oneapi-math-kernel-library-benchmarks/intel-distribution-for-linpack-benchmark/configuring-parameters.html). diff --git a/ansible/roles/hpctests/defaults/main.yml b/ansible/roles/hpctests/defaults/main.yml index c64fff336..6659343d0 100644 --- a/ansible/roles/hpctests/defaults/main.yml +++ b/ansible/roles/hpctests/defaults/main.yml @@ -10,3 +10,4 @@ hpctests_hpl_NB: 192 hpctests_hpl_mem_frac: 0.8 hpctests_hpl_arch: linux64 #hpctests_nodes: +#hpctests_partition: diff --git a/ansible/roles/hpctests/library/slurm_node_info.py b/ansible/roles/hpctests/library/slurm_node_info.py index 277240d9c..52e680018 100644 --- a/ansible/roles/hpctests/library/slurm_node_info.py +++ b/ansible/roles/hpctests/library/slurm_node_info.py @@ -23,7 +23,7 @@ options nodes: description: - - Slurm nodenames for which information is required. These must be homogenous. + - Slurm nodenames for which information is required. required: true type: list requirements: @@ -56,7 +56,6 @@ def run_module(): print(values) for ix, param in enumerate(params): info[param] = [nodeinfo[ix].strip() for nodeinfo in values if nodeinfo[nodelist_ix].strip() in module.params['nodes']] - # info[param] = [nodeinfo[nodelist_ix] for nodeinfo in values] result['info'] = info module.exit_json(**result) diff --git a/ansible/roles/hpctests/tasks/hpl-solo.yml b/ansible/roles/hpctests/tasks/hpl-solo.yml index 3a3d27430..519fb3dc7 100644 --- a/ansible/roles/hpctests/tasks/hpl-solo.yml +++ b/ansible/roles/hpctests/tasks/hpl-solo.yml @@ -42,8 +42,8 @@ - debug: msg: "Using {{ hpctests_hplsolo_ntasks }} process per node with P={{ hpctests_hplsolo_pq.grid.P }}, Q={{ hpctests_hplsolo_pq.grid.Q }} targeting {{ (hpctests_hpl_mem_frac | float) * 100 }}% of {{ hpctests_nodeinfo.info['MEMORY'][0] }} MB memory per node, block size (NB) = {{ hpctests_hpl_NB }}, problem size (N) = {{ hpctests_hplsolo_N }}" -- name: Get all nodes - shell: "sinfo --Node --noheader --format %N" # TODO: assumes only one partition, although actually excluding nodes not in the default partition should be fine. +- name: Get all nodes in partition + shell: "sinfo --Node --noheader --format %N --partition={{ hpctests_partition }}" register: all_nodes changed_when: false @@ -74,6 +74,11 @@ vars: hpctests_hplsolo_ntasks: 2 # TODO: FIXME +- name: Remove previous outputs + # As depending on the number of nodes there will be different numbers of output files for different partitions so won't all get overwritten + shell: + cmd: "rm -f {{ hpctests_rootdir }}/hpl-solo/hpl-solo.sh.*.out" + - name: Run hpl-solo shell: sbatch --wait hpl-solo.sh become: no diff --git a/ansible/roles/hpctests/tasks/setup.yml b/ansible/roles/hpctests/tasks/setup.yml index 3b290b392..c3f36551d 100644 --- a/ansible/roles/hpctests/tasks/setup.yml +++ b/ansible/roles/hpctests/tasks/setup.yml @@ -1,7 +1,17 @@ --- +- name: Get partition information + shell: "sinfo --format %P --noheader" + register: _sinfo_partitions + changed_when: false + +- name: Select default partition if hpctests_partition not given + set_fact: + hpctests_partition: "{{ _sinfo_partitions.stdout_lines | select('contains', '*') | first | trim('*') }}" + when: hpctests_partition is not defined + - name: Get info about compute nodes - shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --format %N" + shell: "sinfo --Node --noheader{%if hpctests_nodes is defined %} --nodes {{hpctests_nodes}}{% endif %} --partition {{hpctests_partition}} --format %N" register: hpctests_computes changed_when: false failed_when: hpctests_computes.rc != 0 diff --git a/ansible/roles/hpctests/templates/hpl-build.sh.j2 b/ansible/roles/hpctests/templates/hpl-build.sh.j2 index 881c34a8d..f243a08f7 100644 --- a/ansible/roles/hpctests/templates/hpl-build.sh.j2 +++ b/ansible/roles/hpctests/templates/hpl-build.sh.j2 @@ -4,6 +4,7 @@ #SBATCH --output=%x.%a.out #SBATCH --error=%x.%a.out #SBATCH --exclusive +#SBATCH --partition={{ hpctests_partition }} {%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_computes.stdout_lines[0] }}{% endif %} echo HPL arch: {{ hpctests_hpl_arch }} diff --git a/ansible/roles/hpctests/templates/hpl-solo.sh.j2 b/ansible/roles/hpctests/templates/hpl-solo.sh.j2 index 9b83ea85b..cc7d2b4dd 100644 --- a/ansible/roles/hpctests/templates/hpl-solo.sh.j2 +++ b/ansible/roles/hpctests/templates/hpl-solo.sh.j2 @@ -5,6 +5,7 @@ #SBATCH --error=%x.%a.out #SBATCH --exclusive #SBATCH --array=0-{{ hpctests_computes.stdout_lines | length - 1 }} +#SBATCH --partition={{ hpctests_partition }} {% if hpctests_hplsolo_excluded_nodes | length > 0 %} #SBATCH --exclude={{ hpctests_hplsolo_excluded_nodes | join(',') }} {% endif %} diff --git a/ansible/roles/hpctests/templates/pingmatrix.sh.j2 b/ansible/roles/hpctests/templates/pingmatrix.sh.j2 index 4a368e180..17fb3fd6a 100644 --- a/ansible/roles/hpctests/templates/pingmatrix.sh.j2 +++ b/ansible/roles/hpctests/templates/pingmatrix.sh.j2 @@ -5,6 +5,7 @@ #SBATCH --output=%x.out #SBATCH --error=%x.out #SBATCH --exclusive +#SBATCH --partition={{ hpctests_partition }} {%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_nodes }}{% endif %} export UCX_NET_DEVICES={{ hpctests_ucx_net_devices }} diff --git a/ansible/roles/hpctests/templates/pingpong.sh.j2 b/ansible/roles/hpctests/templates/pingpong.sh.j2 index 1360b88d9..e74e52539 100644 --- a/ansible/roles/hpctests/templates/pingpong.sh.j2 +++ b/ansible/roles/hpctests/templates/pingpong.sh.j2 @@ -5,6 +5,7 @@ #SBATCH --output=%x.out #SBATCH --error=%x.out #SBATCH --exclusive +#SBATCH --partition={{ hpctests_partition }} {%if hpctests_nodes is defined %}#SBATCH --nodelist={{ hpctests_nodes }}{% endif %} export UCX_NET_DEVICES={{ hpctests_ucx_net_devices }} From 4d5bc937325f85394a1a63e93d7470addfbdba4f Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 07:57:07 +0000 Subject: [PATCH 2/9] remove unnecessary hpctests group --- ansible/adhoc/hpctests.yml | 2 +- environments/common/inventory/groups | 3 --- environments/common/layouts/everything | 3 --- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/ansible/adhoc/hpctests.yml b/ansible/adhoc/hpctests.yml index 6e733d340..6e23fc1ab 100644 --- a/ansible/adhoc/hpctests.yml +++ b/ansible/adhoc/hpctests.yml @@ -4,7 +4,7 @@ --- -- hosts: hpctests[0] # TODO: might want to make which node is used selectable? +- hosts: login[0] # TODO: might want to make which node is used selectable? become: false gather_facts: false tasks: diff --git a/environments/common/inventory/groups b/environments/common/inventory/groups index fa926dc23..165919c71 100644 --- a/environments/common/inventory/groups +++ b/environments/common/inventory/groups @@ -13,9 +13,6 @@ login control compute -[hpctests] -# Login group to use for running mpi-based testing. - [cluster:children] # All nodes in the appliance - add e.g. service nodes not running Slurm here. openhpc diff --git a/environments/common/layouts/everything b/environments/common/layouts/everything index cb67924bd..74cfdcdfd 100644 --- a/environments/common/layouts/everything +++ b/environments/common/layouts/everything @@ -1,9 +1,6 @@ [nfs:children] openhpc -[hpctests:children] -login - [mysql:children] control From ecda5b8df318ccc5fc1b599cf23c94698fa869e8 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 07:57:51 +0000 Subject: [PATCH 3/9] fix cookiecutter TF templating for multiple partitions --- .../{{cookiecutter.environment}}/terraform/inventory.tpl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tpl b/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tpl index 09dae3146..ca6bb24fb 100644 --- a/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tpl +++ b/environments/skeleton/{{cookiecutter.environment}}/terraform/inventory.tpl @@ -23,6 +23,8 @@ ${compute.name} ansible_host=${[for n in compute.network: n.fixed_ip_v4 if n.acc %{~ for type_name, type_descr in compute_types} [${cluster_name}_${type_name}] %{~ for node_name, node_type in compute_nodes ~} - %{~ if node_type == type_name }${cluster_name}-${node_name}%{ endif } - %{~ endfor ~} + %{~ if node_type == type_name ~} +${cluster_name}-${node_name} + %{~ endif ~} +%{~ endfor ~} %{ endfor ~} From 90212c5dfcbd686370f59817645d673dabc1e93e Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 08:03:44 +0000 Subject: [PATCH 4/9] use and test 2x partitions in arcus env --- environments/arcus/hooks/check_slurm.yml | 3 ++- environments/arcus/terraform/main.tf | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/environments/arcus/hooks/check_slurm.yml b/environments/arcus/hooks/check_slurm.yml index cfbf43a6b..9120417d8 100644 --- a/environments/arcus/hooks/check_slurm.yml +++ b/environments/arcus/hooks/check_slurm.yml @@ -17,4 +17,5 @@ vars: expected_sinfo: - - "{{ openhpc_cluster_name }}-compute-[0-1] {{ openhpc_slurm_partitions[0].name }}* up 60-00:00:00 2 idle" + - "{{ openhpc_cluster_name }}-compute-[0-1] small* up 60-00:00:00 2 idle" + - "{{ openhpc_cluster_name }}-compute-[2-3] extra up 60-00:00:00 2 idle" diff --git a/environments/arcus/terraform/main.tf b/environments/arcus/terraform/main.tf index 19f00bd06..d4b5da8e3 100644 --- a/environments/arcus/terraform/main.tf +++ b/environments/arcus/terraform/main.tf @@ -31,10 +31,16 @@ module "cluster" { flavor: "vm.alaska.cpu.general.small" image: "openhpc-220830-2042.qcow2" } + extra: { + flavor: "vm.alaska.cpu.general.small" + image: "openhpc-220830-2042.qcow2" + } } compute_nodes = { compute-0: "small" compute-1: "small" + compute-2: "extra" + compute-3: "extra" } environment_root = var.environment_root From 65fd7c411ab93b14c3cf935d078e56fa1d224eda Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 09:48:28 +0000 Subject: [PATCH 5/9] make podman available to mysql --- ansible/bootstrap.yml | 14 ++++++++++++++ ansible/monitoring.yml | 14 -------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ansible/bootstrap.yml b/ansible/bootstrap.yml index 1940ee64f..ae1074dc4 100644 --- a/ansible/bootstrap.yml +++ b/ansible/bootstrap.yml @@ -74,6 +74,20 @@ - import_role: name: fail2ban +- name: Setup podman + hosts: podman + tags: podman + tasks: + - import_role: + name: podman + tasks_from: prereqs.yml + tags: prereqs + + - import_role: + name: podman + tasks_from: config.yml + tags: config + - hosts: update gather_facts: false become: yes diff --git a/ansible/monitoring.yml b/ansible/monitoring.yml index 5d94ed3dd..0685f790e 100644 --- a/ansible/monitoring.yml +++ b/ansible/monitoring.yml @@ -1,20 +1,6 @@ # --- # # NOTE: Requires slurmdbd -- name: Setup podman - hosts: podman - tags: podman - tasks: - - import_role: - name: podman - tasks_from: prereqs.yml - tags: prereqs - - - import_role: - name: podman - tasks_from: config.yml - tags: config - - name: Setup elasticsearch hosts: opendistro tags: opendistro From a130e5601dd23bc0e436ed7ab1a414a66d2cb6f5 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 09:51:10 +0000 Subject: [PATCH 6/9] ensure state directories owned by podman user before containerised service start --- ansible/roles/mysql/templates/mysql.service.j2 | 1 + ansible/roles/opendistro/templates/opendistro.service.j2 | 1 + 2 files changed, 2 insertions(+) diff --git a/ansible/roles/mysql/templates/mysql.service.j2 b/ansible/roles/mysql/templates/mysql.service.j2 index 758e25ae4..e61bb4002 100644 --- a/ansible/roles/mysql/templates/mysql.service.j2 +++ b/ansible/roles/mysql/templates/mysql.service.j2 @@ -13,6 +13,7 @@ Restart=always EnvironmentFile=/etc/sysconfig/mysqld # The above EnvironmentFile must define MYSQL_INITIAL_ROOT_PASSWORD ExecStartPre=+install -d -o {{ mysql_podman_user }} -g {{ mysql_podman_user }} -Z container_file_t {{ mysql_datadir }} +ExecStartPre=+chown -R {{ mysql_podman_user }}:{{ mysql_podman_user }} {{ mysql_datadir }} ExecStart=/usr/bin/podman run \ --network slirp4netns:cidr={{ podman_cidr }} \ --sdnotify=conmon --cgroups=no-conmon \ diff --git a/ansible/roles/opendistro/templates/opendistro.service.j2 b/ansible/roles/opendistro/templates/opendistro.service.j2 index c78605afc..af7eb8e78 100644 --- a/ansible/roles/opendistro/templates/opendistro.service.j2 +++ b/ansible/roles/opendistro/templates/opendistro.service.j2 @@ -10,6 +10,7 @@ After=network-online.target Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always ExecStartPre=+install -d -o {{ opendistro_podman_user }} -g {{ opendistro_podman_user }} -Z container_file_t {{ opendistro_data_path }} +ExecStartPre=+chown -R {{ opendistro_podman_user }}:{{ opendistro_podman_user }} {{ opendistro_data_path }} ExecStart=/usr/bin/podman run \ --network slirp4netns:cidr={{ podman_cidr }} \ --sdnotify=conmon --cgroups=no-conmon \ From 11d6840e1c8bb27c85094caff67f69b5a7beb743 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 10:09:37 +0000 Subject: [PATCH 7/9] fix CI rebuild of compute nodes --- ansible/ci/test_reimage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/ci/test_reimage.yml b/ansible/ci/test_reimage.yml index 6c9b84561..6ebb4175f 100644 --- a/ansible/ci/test_reimage.yml +++ b/ansible/ci/test_reimage.yml @@ -45,10 +45,10 @@ gather_facts: no tags: reimage_compute tasks: - # TODO: This is specific to smslabs/arcus environment config - could generalise to all compute nodes + # TODO: This is specific to arcus environment config - could generalise to all compute nodes - name: Request compute node rebuild via Slurm shell: - cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-1] + cmd: scontrol reboot ASAP nextstate=RESUME reason='rebuild image:{{ compute_build.artifact_id }}' {{ openhpc_cluster_name }}-compute-[0-3] become: yes - name: Check compute node rebuild completed From c6b10fb0c5ded768a3ca40010168d9644ddaf16c Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 10:37:15 +0000 Subject: [PATCH 8/9] bump mysql startup wait (again) --- ansible/roles/mysql/tasks/configure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/mysql/tasks/configure.yml b/ansible/roles/mysql/tasks/configure.yml index a2a48d827..e29a536b5 100644 --- a/ansible/roles/mysql/tasks/configure.yml +++ b/ansible/roles/mysql/tasks/configure.yml @@ -24,7 +24,7 @@ # no_log: true # TODO: FIXME register: _mysql_info until: "'version' in _mysql_info" - retries: 60 + retries: 90 delay: 2 - name: Ensure mysql databases created From 017e9b0106c583edac14768289b94d930fea6e82 Mon Sep 17 00:00:00 2001 From: Steve Brasier Date: Tue, 27 Sep 2022 13:31:50 +0000 Subject: [PATCH 9/9] add partition info in output --- ansible/roles/hpctests/tasks/hpl-solo.yml | 5 +++-- ansible/roles/hpctests/tasks/pingmatrix.yml | 4 +++- ansible/roles/hpctests/tasks/pingpong.yml | 9 +++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/ansible/roles/hpctests/tasks/hpl-solo.yml b/ansible/roles/hpctests/tasks/hpl-solo.yml index 519fb3dc7..adf8e1823 100644 --- a/ansible/roles/hpctests/tasks/hpl-solo.yml +++ b/ansible/roles/hpctests/tasks/hpl-solo.yml @@ -116,10 +116,11 @@ tags: postpro debug: msg: | - Summary for hpl-solo ({{ hpctests_computes.stdout_lines | length }} nodes) job {{ hpctests_hplsolo_sbatch.stdout.split()[-1] }} using {{ hpctests_ucx_net_devices }}: + Summary for hpl-solo on {{ hpctests_computes.stdout_lines | length }} nodes in '{{ hpctests_partition }}' partition, job ID {{ hpctests_hplsolo_sbatch.stdout.split()[-1] }}, device '{{ hpctests_ucx_net_devices }}': + Max: {{ perf.stdout_lines | map('float') | max }} gflops Min: {{ perf.stdout_lines | map('float') | min }} gflops - Mean: {{ (perf.stdout_lines | map('float') | sum) / (hpctests_computes.stdout_lines | length) }} gflops + Mean: {{ (perf.stdout_lines | map('float') | sum) / (hpctests_computes.stdout_lines | length) }} gflops Individual node results (gflops): {{ dict(hpctests_computes.stdout_lines | zip(perf.stdout_lines | map('float') )) | to_nice_yaml }} diff --git a/ansible/roles/hpctests/tasks/pingmatrix.yml b/ansible/roles/hpctests/tasks/pingmatrix.yml index a40881608..4d32ffcd7 100644 --- a/ansible/roles/hpctests/tasks/pingmatrix.yml +++ b/ansible/roles/hpctests/tasks/pingmatrix.yml @@ -61,6 +61,8 @@ - name: Summarise results debug: msg: | - Summary for pingmatrix (pairwise on {{ slurm_names.stdout_lines | length }} nodes) job {{ hpctests_pingmatrix_sbatch.stdout.split()[-1] }} using {{ hpctests_ucx_net_devices }}: + Summary for pingmatrix pairwise over {{ slurm_names.stdout_lines | length }} nodes in '{{ hpctests_partition }}' partition, job ID {{ hpctests_pingmatrix_sbatch.stdout.split()[-1] }}, device '{{ hpctests_ucx_net_devices }}': + {{ nxnlatbw['stats'] | to_nice_yaml }} + Tabular output on ansible control host at {{ hpctests_outdir }}/pingmatrix.html diff --git a/ansible/roles/hpctests/tasks/pingpong.yml b/ansible/roles/hpctests/tasks/pingpong.yml index f01295bd7..56123205f 100644 --- a/ansible/roles/hpctests/tasks/pingpong.yml +++ b/ansible/roles/hpctests/tasks/pingpong.yml @@ -55,10 +55,11 @@ - debug: msg: | - Summary for pingpong (2x scheduler-selected nodes) job {{ _pingpong_jobid }} (using interface {{ hpctests_ucx_net_devices }}): - nodes: {{ hpctests_pingpong_run_nodes.stdout.split()[1] }} - zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us - max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s) + Summary for pingpong using 2x scheduler-selected nodes in '{{ hpctests_partition }}' partition, job ID {{ _pingpong_jobid }}, device '{{ hpctests_ucx_net_devices }}': + + Nodes: {{ hpctests_pingpong_run_nodes.stdout.split()[1] }} + Zero-size msg latency: {{ hpctests_pingpong_out['columns']['latency'][0] }} us + Max bandwidth: {{ hpctests_pingpong_out['columns']['bandwidth'] | max }} Mbytes/s ({{ (hpctests_pingpong_out['columns']['bandwidth'] | max) / 125.0 }} Gbit/s) See plot on localhost: {{ _pingpong_plot.stdout }}