From b03368485cec6c24d05ef853f9bfd7ba622c8dca Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Fri, 12 Apr 2024 17:52:36 +0000 Subject: [PATCH] testgrid: print out all failed tests for visibility. Signed-off-by: Siyuan Zhang --- scripts/measure-testgrid-flakiness.sh | 10 ++++++++-- tools/testgrid-analysis/cmd/data.go | 11 ++++++++++- tools/testgrid-analysis/cmd/flaky.go | 6 +++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/measure-testgrid-flakiness.sh b/scripts/measure-testgrid-flakiness.sh index f0765f93136..6cde5418727 100755 --- a/scripts/measure-testgrid-flakiness.sh +++ b/scripts/measure-testgrid-flakiness.sh @@ -10,6 +10,12 @@ then fi pushd ./tools/testgrid-analysis -go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 -go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 +# ci-etcd-e2e-amd64 and ci-etcd-unit-test-amd64 runs 6 times a day. Keeping a rolling window of 14 days. +go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14 +go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14 + +# do not create issues for presubmit tests +go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64 +go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test + popd diff --git a/tools/testgrid-analysis/cmd/data.go b/tools/testgrid-analysis/cmd/data.go index 7b6af8dcb64..5ea284e78c8 100644 --- a/tools/testgrid-analysis/cmd/data.go +++ b/tools/testgrid-analysis/cmd/data.go @@ -20,6 +20,7 @@ import ( "net/http" "os" "strings" + "time" apipb "github.com/GoogleCloudPlatform/testgrid/pb/api/v1" statuspb "github.com/GoogleCloudPlatform/testgrid/pb/test_status" @@ -77,6 +78,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests if !strings.HasPrefix(row.Name, "go.etcd.io") { return &t } + earliestTimeToConsider := time.Now().AddDate(0, 0, -1*maxDays) total := 0 failed := 0 logs := []string{} @@ -89,13 +91,19 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests } continue } + header := headers[i] + if maxDays > 0 && header.Started.AsTime().Before(earliestTimeToConsider) { + continue + } total += 1 if _, ok := failureTestStatusesInt[cell.Result]; ok { failed += 1 - header := headers[i] // markdown table format of | commit | log | logs = append(logs, fmt.Sprintf("| %s | %s | https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), tab, header.Build)) } + if maxRuns > 0 && total >= maxRuns { + break + } } t.FailedRuns = failed t.TotalRuns = total @@ -106,6 +114,7 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests t.IssueBody = fmt.Sprintf("## %s Test: %s \nTest failed %.1f%% (%d/%d) of the time\n\nfailure logs are:\n| commit | started | log |\n| --- | --- | --- |\n%s\n", dashboardUrl, t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n")) t.IssueBody += "\nPlease follow the [instructions in the contributing guide](https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#check-for-flaky-tests) to reproduce the issue.\n" + fmt.Printf("%s failed %.1f%% (%d/%d) of the time\n", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns) } return &t } diff --git a/tools/testgrid-analysis/cmd/flaky.go b/tools/testgrid-analysis/cmd/flaky.go index 48a115d56b0..2752b81a873 100644 --- a/tools/testgrid-analysis/cmd/flaky.go +++ b/tools/testgrid-analysis/cmd/flaky.go @@ -31,6 +31,8 @@ var flakyCmd = &cobra.Command{ var ( flakyThreshold float32 minRuns int + maxRuns int + maxDays int createGithubIssue bool githubOwner string githubRepo string @@ -44,6 +46,8 @@ func init() { flakyCmd.Flags().BoolVar(&createGithubIssue, "create-issue", false, "create Github issue for each flaky test") flakyCmd.Flags().Float32Var(&flakyThreshold, "flaky-threshold", 0.1, "fraction threshold of test failures for a test to be considered flaky") flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be included in flaky analysis") + flakyCmd.Flags().IntVar(&maxRuns, "max-runs", 0, "maximum test runs for a test to be included in flaky analysis, 0 to include all") + flakyCmd.Flags().IntVar(&maxDays, "max-days", 0, "maximum days of results before today to be included in flaky analysis, 0 to include all") flakyCmd.Flags().StringVar(&githubOwner, "github-owner", "etcd-io", "the github organization to create the issue for") flakyCmd.Flags().StringVar(&githubRepo, "github-repo", "etcd", "the github repo to create the issue for") } @@ -59,7 +63,7 @@ func flakyFunc(cmd *cobra.Command, args []string) { } } fmt.Println(lineSep) - fmt.Printf("Detected total %d flaky tests for %s#%s\n", len(flakyTests), dashboard, tab) + fmt.Printf("Detected total %d flaky tests above the %.0f%% threshold for %s#%s\n", len(flakyTests), flakyThreshold*100, dashboard, tab) fmt.Println(lineSep) if len(flakyTests) == 0 { return