Merge pull request #17785 from siyuanfoundation/flaky

testgrid: print out all failed tests for visibility.

scripts/measure-testgrid-flakiness.sh

@@ -10,6 +10,12 @@ then
 fi
 
 pushd ./tools/testgrid-analysis
-go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64
-go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64
+# ci-etcd-e2e-amd64 and ci-etcd-unit-test-amd64 runs 6 times a day. Keeping a rolling window of 14 days.
+go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14
+go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14
+
+# do not create issues for presubmit tests
+go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64
+go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test
+
 popd

tools/testgrid-analysis/cmd/data.go

@@ -20,6 +20,7 @@ import (
  "net/http"
  "os"
  "strings"
+ "time"
 
  apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1"
  statuspb "github.com/GoogleCloudPlatform/testgrid/pb/test_status"
@@ -77,6 +78,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
  if !strings.HasPrefix(row.Name, "go.etcd.io") {
  return &t
  }
+ earliestTimeToConsider := time.Now().AddDate(0, 0, -1*maxDays)
  total := 0
  failed := 0
  logs := []string{}
@@ -89,13 +91,19 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
  }
  continue
  }
+ header := headers[i]
+ if maxDays > 0 && header.Started.AsTime().Before(earliestTimeToConsider) {
+ continue
+ }
  total += 1
  if _, ok := failureTestStatusesInt[cell.Result]; ok {
  failed += 1
- header := headers[i]
  // markdown table format of | commit | log |
  logs = append(logs, fmt.Sprintf("| %s | %s | https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), tab, header.Build))
  }
+ if maxRuns > 0 && total >= maxRuns {
+ break
+ }
  }
  t.FailedRuns = failed
  t.TotalRuns = total
@@ -106,6 +114,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
  t.IssueBody = fmt.Sprintf("## %s Test: %s \nTest failed %.1f%% (%d/%d) of the time\n\nfailure logs are:\n| commit | started | log |\n| --- | --- | --- |\n%s\n",
  dashboardUrl, t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n"))
  t.IssueBody += "\nPlease follow the [instructions in the contributing guide](https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#check-for-flaky-tests) to reproduce the issue.\n"
+ fmt.Printf("%s failed %.1f%% (%d/%d) of the time\n", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns)
  }
  return &t
 }

tools/testgrid-analysis/cmd/flaky.go

@@ -31,6 +31,8 @@ var flakyCmd = &cobra.Command{
 var (
  flakyThreshold float32
  minRuns int
+ maxRuns int
+ maxDays int
  createGithubIssue bool
  githubOwner string
  githubRepo string
@@ -44,6 +46,8 @@ func init() {
  flakyCmd.Flags().BoolVar(&createGithubIssue, "create-issue", false, "create Github issue for each flaky test")
  flakyCmd.Flags().Float32Var(&flakyThreshold, "flaky-threshold", 0.1, "fraction threshold of test failures for a test to be considered flaky")
  flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be included in flaky analysis")
+ flakyCmd.Flags().IntVar(&maxRuns, "max-runs", 0, "maximum test runs for a test to be included in flaky analysis, 0 to include all")
+ flakyCmd.Flags().IntVar(&maxDays, "max-days", 0, "maximum days of results before today to be included in flaky analysis, 0 to include all")
  flakyCmd.Flags().StringVar(&githubOwner, "github-owner", "etcd-io", "the github organization to create the issue for")
  flakyCmd.Flags().StringVar(&githubRepo, "github-repo", "etcd", "the github repo to create the issue for")
 }
@@ -59,7 +63,7 @@ func flakyFunc(cmd *cobra.Command, args []string) {
  }
  }
  fmt.Println(lineSep)
- fmt.Printf("Detected total %d flaky tests for %s#%s\n", len(flakyTests), dashboard, tab)
+ fmt.Printf("Detected total %d flaky tests above the %.0f%% threshold for %s#%s\n", len(flakyTests), flakyThreshold*100, dashboard, tab)
  fmt.Println(lineSep)
  if len(flakyTests) == 0 {
  return