ci: aarch64: Initial nightly performance test

.github/automation/aarch64/ci.json

@@ -2,6 +2,7 @@
     "dependencies": {
         "acl": "v25.02",
         "gcc": "13",
-        "clang": "17"
+        "clang": "17",
+        "onednn-base": "v3.7"
     }
-}
+}

.github/automation/performance/bench_nightly_performance.sh

@@ -0,0 +1,53 @@
+#! /bin/bash
+
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+
+# Usage: bash bench_nightly_performance.sh {baseline_benchdnn_executable} {benchdnn_executable} {baseline_results_file} {new_results_file}
+
+IFS=$'\n' # Prevents shuffling from using spaces as delimiters
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+TESTS=(
+        "$1 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul_nightly >> $3"
+        "$2 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul_nightly >> $4"
+        "$1 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv_nightly >> $3"
+        "$2 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv_nightly >> $4"
+        "$1 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise_nightly >> $3"
+        "$2 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise_nightly >> $4"
+        "$1 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder_nightly >> $3"
+        "$2 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder_nightly >> $4"
+    )
+
+N=5
+
+for i in $( seq $N )
+do
+    echo "Testing loop ${i} / ${N}..."
+
+    TESTS=( $(shuf -e "${TESTS[@]}") )
+
+    for test in "${TESTS[@]}"
+    do
+        echo "Starting ${test}"
+        SECONDS=0
+        eval $test
+        duration=$SECONDS
+        echo "Completed in $((duration / 60)):$((duration % 60))"
+    done
+done

.github/automation/performance/bench_pr_performance.sh

@@ -0,0 +1,52 @@
+#! /bin/bash
+
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+
+# Usage: bash bench_pr_performance.sh {baseline_benchdnn_executable} {benchdnn_executable} {baseline_results_file} {new_results_file}
+
+IFS=$'\n' # Prevents shuffling from using spaces as delimiters
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+TESTS=(
+        "$1 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul >> $3"
+        "$2 --matmul --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/matmul >> $4"
+        "$1 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv >> $3"
+        "$2 --conv --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/conv >> $4"
+        "$1 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise >> $3"
+        "$2 --eltwise --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/eltwise >> $4"
+        "$1 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder >> $3"
+        "$2 --reorder --mode=P --perf-template=%prb%,%-time% --batch=${SCRIPT_DIR}/inputs/reorder >> $4"
+    )
+
+N=5
+
+for i in $( seq $N )
+do
+    echo "Testing loop ${i} / ${N}..."
+
+    TESTS=( $(shuf -e "${TESTS[@]}") )
+
+    for test in "${TESTS[@]}"
+    do
+        echo "Starting ${test}"
+        SECONDS=0
+        eval $test
+        duration=$SECONDS
+        echo "Completed in $((duration / 60)):$((duration % 60))"
+    done
+done

.github/automation/performance/benchdnn_comparison.py

@@ -21,6 +21,8 @@
 import os
 from collections import defaultdict
 from scipy.stats import ttest_ind
+import warnings
+import statistics
 
 
 def compare_two_benchdnn(file1, file2, tolerance=0.05):
@@ -38,9 +40,9 @@ def compare_two_benchdnn(file1, file2, tolerance=0.05):
     r2 = [x.split(",") for x in r2 if x[0:8] == "--mode=P"]
 
     if (len(r1) == 0) or (len(r2) == 0):
-        raise Exception("One or both of the test results have zero lines")
+        warnings.warn("One or both of the test results have zero lines")
     if len(r1) != len(r2):
-        raise Exception("The number of benchdnn runs do not match")
+        warnings.warn("The number of benchdnn runs do not match")
 
     r1_samples = defaultdict(list)
     r2_samples = defaultdict(list)
@@ -50,26 +52,37 @@ def compare_two_benchdnn(file1, file2, tolerance=0.05):
     for k, v in r2:
         r2_samples[k].append(float(v[:-1]))
 
-    passed = True
     failed_tests = []
+    times = {}
     for prb, r1_times in r1_samples.items():
         if prb not in r2_samples:
-            raise Exception(f"{prb} exists in {file1} but not {file2}")
+            warnings.warn(f"{prb} exists in {file1} but not {file2}")
+            continue
+
         r2_times = r2_samples[prb]
 
         res = ttest_ind(r2_times, r1_times, alternative='greater')
-
-        if res.pvalue < 0.05:
-            failed_tests.append(prb)
+        r1_med = statistics.median(r1_times)
+        r2_med = statistics.median(r2_times)
+        times[prb] = (r1_med, r2_med)
+        times_str = f" {times[prb][0]} vs {times[prb][1]}"
+
+        # pass the test if:
+        # the t-test passes (i.e. pvalue > 0.05) OR
+        # both the median time and min time has not 
+        # slowed down by more than 10%
+        passed = res.pvalue > 0.05 or \
+                ((r2_med - r1_med) / r1_med < 0.1 and \
+                (min(r2_times) - min(r1_times)) / min(r1_times) < 0.1)
+        if not passed:
+            failed_tests.append(prb + times_str)
             passed = False
 
-        print(prb + (" passed" if passed else " failed"))
-
     if "GITHUB_OUTPUT" in os.environ:
         with open(os.environ["GITHUB_OUTPUT"], "a") as f:
-            print(f"pass={passed}", file=f)
+            print(f"pass={not failed_tests}", file=f)
 
-    if passed:
+    if not failed_tests:
         print("Regression tests passed")
     else:
         message = "\n----The following regression tests failed:----\n" + \

.github/automation/performance/inputs/conv

@@ -15,8 +15,13 @@
 # limitations under the License.
 # *******************************************************************************
 
-# From Resnet
 --reset
 --dir=FWD_D
---dt=bf16,f32
+--dt=f32
+mb1_ic64oc256_ih200oh200kh1sh1dh0ph0_iw267ow267kw1sw1dw0pw0
+
+--reset
+--dir=FWD_D
+--dt=f32
+--attr-fpmath=bf16
 mb1_ic64oc256_ih200oh200kh1sh1dh0ph0_iw267ow267kw1sw1dw0pw0

.github/automation/performance/inputs/conv_nightly

@@ -0,0 +1,25 @@
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+--reset
+--batch=conv
+
+--reset 
+--dt=f32
+--alg=auto
+--dir=FWD_D,FWD_B
+--attr-fpmath=,bf16
+--batch=shapes_resnet_50

.github/automation/performance/inputs/eltwise

@@ -18,6 +18,6 @@
 --inplace=true
 --alg=gelu_erf
 --dir=FWD_D
---dt=bf16
+--dt=f32,bf16
 --tag=abc
-1x1x1536
+1536x384

.github/automation/performance/inputs/eltwise_nightly

@@ -0,0 +1,41 @@
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+
+--reset
+--batch=eltwise
+
+--reset
+
+--dt=f32
+--tag=abx,axb
+--dir=FWD_D
+--attr-post-ops=,
+
+## algs which do not support alpha and beta + relu with alpha=0
+--alpha=0 --beta=0
+--alg=exp,exp_dst,gelu_erf,gelu_tanh,relu_dst,tanh,tanh_dst
+384x384
+
+## algs which support negative alpha
+--alpha=-2 --beta=0
+--alg=elu,relu,swish
+384x384
+
+## algs which support alpha and beta
+--alpha=-2 --beta=3
+--alg=linear
+384x384

.github/automation/performance/inputs/matmul

@@ -17,7 +17,16 @@
 --reset
 --stag=ab
 --wtag=any
+--dtag=ab
 --attr-post-ops=sum
+--dt=f32
+1500x1536:1536x384
+
+--reset
+--stag=ab
+--wtag=any
 --dtag=ab
---dt=f32,bf16
+--attr-post-ops=sum
+--attr-fpmath=bf16
+--dt=f32
 1500x1536:1536x384

.github/automation/performance/inputs/matmul_nightly

@@ -0,0 +1,43 @@
+# *******************************************************************************
+# Copyright 2025 Arm Limited and affiliates.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# *******************************************************************************
+--reset
+--batch=matmul
+
+# Plain cases
+--reset
+--dt=f32,s8:s8:f32
+--bia-dt=f32,undef
+--bia_mask=2
+--batch=shapes_2d_ci
+--bia_mask=4
+--batch=shapes_3d
+
+--dt=f32
+--bia-dt=f32,undef
+--bia_mask=2
+--attr-fpmath=bf16
+--batch=shapes_2d_ci
+--bia_mask=4
+--batch=shapes_3d
+
+#f16
+--dt=f16:f16:f16
+--bia-dt=undef
+--bia_mask=2
+--batch=shapes_2d_ci
+--bia_mask=4
+--batch=shapes_3d