Use bazel CI to repeat tests and identify flakes

.github/workflows/bazel_repeat_flaky_tests.yml

@@ -0,0 +1,14 @@
+name: Bazel repeat flaky tests
+on:
+  schedule:
+    # GitHub caches are purged after 7 days of inactivity, so
+    # running twice a week helps protect the bazel cache
+    # Run at 14:00 UTC, Monday and Friday
+    - cron: '0 14 * * 1,5'
+
+jobs:
+  repeat-flaky-tests:
+    uses: ./.github/workflows/bazel_repeat_tests.yml
+    with:
+      flaky: 1
+      runs_per_test: 10

.github/workflows/bazel_repeat_non_flaky_tests.yml

@@ -0,0 +1,14 @@
+name: Bazel repeat non-flaky tests
+on:
+  schedule:
+    # GitHub caches are purged after 7 days of inactivity, so
+    # running twice a week helps protect the bazel cache
+    # Run at 15:00 UTC, Monday and Friday
+    - cron: '0 15 * * 1,5'
+
+jobs:
+  repeat-non-flaky-tests:
+    uses: ./.github/workflows/bazel_repeat_tests.yml
+    with:
+      flaky: 0
+      runs_per_test: 10

.github/workflows/bazel_repeat_tests.yml

@@ -0,0 +1,38 @@
+name: Bazel repeat tests
+on:
+  workflow_call:
+    inputs: &shared_inputs
+      flaky:
+        description: "0 to execute non-flaky tests, 1 to execute flaky tests"
+        default: 0
+        required: false
+        type: string
+      runs_per_test:
+        description: "Number of times to repeat each test"
+        default: 10
+        required: false
+        type: string
+  workflow_dispatch:
+    inputs: *shared_inputs
+
+jobs:
+  repeat-tests:
+    uses: bazel-contrib/.github/.github/workflows/bazel.yaml@v7.7.0
+    with:
+      folders: |
+        [
+          "."
+        ]
+      exclude: |
+        [
+          {"folder": ".", "bzlmodEnabled": false}
+        ]
+      exclude_windows: true
+      bazel_test_command: |
+        # Exit with success code 0 if no targets match the query
+        # `grep -q '^' fails if the query output is empty
+        bazel query "attr(flaky, ${{ inputs.flaky }}, tests(//...))" \
+            | grep -q '^' || exit 0
+        bazel test --runs_per_test=${{ inputs.runs_per_test }} \
+            --test_output=errors \
+            $(bazel query "attr(flaky, ${{ inputs.flaky }}, tests(//...))")

BUILD.bazel

@@ -143,13 +143,27 @@ test_sources = glob(
     include = ["src/*_TEST.cc"],
 )
 
+# Found with
+# bazel test test:all --runs_per_test 10
+flaky_test_srcs = [
+    "src/Clock_TEST.cc",
+    "src/Node_TEST.cc",
+]
+
+# Tests that should not be executed in parallel
+exclusive_test_srcs = [
+    "src/Clock_TEST.cc",
+]
+
 [cc_test(
     name = src.replace("/", "_").replace(".cc", "").replace("src_", ""),
     srcs = [src],
     env = {
         "GZ_BAZEL": "1",
         "GZ_IP": "127.0.0.1",
     },
+    flaky = src in flaky_test_srcs,
+    tags = ["exclusive"] if src in exclusive_test_srcs else [],
     deps = [
         ":gz-transport",
         ":gz-transport-discovery-header",