Add CI workflows that repeats bazel tests

This adds workflows that can run tests repeatedly
in bazel to gain information about flaky tests.
Since bazel targets can be annotated as flaky,
distinct workflows are created to execute flaky
and non-flaky tests separately.

Any failures of tests annotated as non-flaky are
a signal that the offending test should be annotated
as flaky. The results from repeated executions of
flaky tests will generate flakiness statistics
and may aid in diagnosing the root cause of
flakiness.

Signed-off-by: Steve Peters <scpeters@intrinsic.ai>

Author: scpeters

.github/workflows/bazel_repeat_flaky_tests.yml

@@ -0,0 +1,18 @@
+name: Bazel repeat flaky tests
+on:
+  push:
+    branches:
+      - 'scpeters/bazel_repeat_tests'
+  workflow_dispatch:
+    inputs:
+      runs_per_test:
+        default: 10
+        required: false
+        type: string
+
+jobs:
+  call-repeat-tests:
+    uses: ./.github/workflows/bazel_repeat_tests.yml
+    with:
+      flaky: 1
+      runs_per_test: 10

.github/workflows/bazel_repeat_non_flaky_tests.yml

@@ -0,0 +1,18 @@
+name: Bazel repeat non-flaky tests
+on:
+  push:
+    branches:
+      - 'scpeters/bazel_repeat_tests'
+  workflow_dispatch:
+    inputs:
+      runs_per_test:
+        default: 10
+        required: false
+        type: string
+
+jobs:
+  call-repeat-tests:
+    uses: ./.github/workflows/bazel_repeat_tests.yml
+    with:
+      flaky: 0
+      runs_per_test: 10

.github/workflows/bazel_repeat_tests.yml

@@ -0,0 +1,43 @@
+name: Bazel repeat tests
+on:
+  workflow_call:
+    inputs:
+      flaky:
+        default: 0
+        required: false
+        type: string
+      runs_per_test:
+        default: 10
+        required: false
+        type: string
+  workflow_dispatch:
+    inputs:
+      flaky:
+        default: 0
+        required: false
+        type: string
+      runs_per_test:
+        default: 10
+        required: false
+        type: string
+
+jobs:
+  repeat-tests:
+    uses: bazel-contrib/.github/.github/workflows/bazel.yaml@v7.7.0
+    with:
+      folders: |
+        [
+          "."
+        ]
+      exclude: |
+        [
+          {"folder": ".", "bzlmodEnabled": false}
+        ]
+      exclude_windows: true
+      bazel_test_command: |
+        # Exit with success code 0 if no targets match the query
+        # `grep -q '^' fails if the query output is empty
+        bazel query "attr(flaky, ${{ inputs.flaky }}, tests(//...))" \
+            | grep -q '^' || exit 0
+        bazel test --runs_per_test=${{ inputs.runs_per_test }} \
+            $(bazel query "attr(flaky, ${{ inputs.flaky }}, tests(//...))")