Use bazel CI to repeat tests and identify flakes (#820)

This adds workflows that can run tests repeatedly
in bazel to gain information about flaky tests.
Since bazel targets can be annotated as flaky,
distinct workflows are created to execute flaky
and non-flaky tests separately.

Any failures of tests annotated as non-flaky are
a signal that the offending test should be annotated
as flaky. The results from repeated executions of
flaky tests will generate flakiness statistics
and may aid in diagnosing the root cause of
flakiness.

The repeated test workflows are scheduled to run
twice a week.

Signed-off-by: Steve Peters <scpeters@intrinsic.ai>

Author: scpeters

.github/workflows/bazel_repeat_flaky_tests.yml

@@ -0,0 +1,14 @@
+name: Bazel repeat flaky tests
+on:
+  schedule:
+    # GitHub caches are purged after 7 days of inactivity, so
+    # running twice a week helps protect the bazel cache
+    # Run at 16:00 UTC, Monday and Friday
+    - cron: '0 16 * * 1,5'
+
+jobs:
+  repeat-flaky-tests:
+    uses: ./.github/workflows/bazel_repeat_tests.yml
+    with:
+      flaky: 1
+      runs_per_test: 10

.github/workflows/bazel_repeat_non_flaky_tests.yml

@@ -0,0 +1,14 @@
+name: Bazel repeat non-flaky tests
+on:
+  schedule:
+    # GitHub caches are purged after 7 days of inactivity, so
+    # running twice a week helps protect the bazel cache
+    # Run at 17:00 UTC, Monday and Friday
+    - cron: '0 17 * * 1,5'
+
+jobs:
+  repeat-non-flaky-tests:
+    uses: ./.github/workflows/bazel_repeat_tests.yml
+    with:
+      flaky: 0
+      runs_per_test: 10

.github/workflows/bazel_repeat_tests.yml

@@ -0,0 +1,38 @@
+name: Bazel repeat tests
+on:
+  workflow_call:
+    inputs: &shared_inputs
+      flaky:
+        description: "0 to execute non-flaky tests, 1 to execute flaky tests"
+        default: 0
+        required: false
+        type: string
+      runs_per_test:
+        description: "Number of times to repeat each test"
+        default: 10
+        required: false
+        type: string
+  workflow_dispatch:
+    inputs: *shared_inputs
+
+jobs:
+  repeat-tests:
+    uses: bazel-contrib/.github/.github/workflows/bazel.yaml@v7.7.0
+    with:
+      folders: |
+        [
+          "."
+        ]
+      exclude: |
+        [
+          {"folder": ".", "bzlmodEnabled": false}
+        ]
+      exclude_windows: true
+      bazel_test_command: |
+        # Exit with success code 0 if no targets match the query
+        # `grep -q '^' fails if the query output is empty
+        bazel query "attr(flaky, ${{ inputs.flaky }}, tests(//...))" \
+            | grep -q '^' || exit 0
+        bazel test --runs_per_test=${{ inputs.runs_per_test }} \
+            --test_output=errors \
+            $(bazel query "attr(flaky, ${{ inputs.flaky }}, tests(//...))")