tools: generate tab level test summary instead of test level.

Signed-off-by: Siyuan Zhang <[email protected]>

scripts/measure-testgrid-flakiness.sh

@@ -11,11 +11,12 @@ fi
 
 pushd ./tools/testgrid-analysis
 # ci-etcd-e2e-amd64 and ci-etcd-unit-test-amd64 runs 6 times a day. Keeping a rolling window of 14 days.
-go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14
-go run main.go flaky --create-issue --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14
+go run main.go flaky --auto-create-issues --dashboard=sig-etcd-periodics --tab=ci-etcd-e2e-amd64 --max-days=14
+go run main.go flaky --auto-create-issues --dashboard=sig-etcd-periodics --tab=ci-etcd-unit-test-amd64 --max-days=14
 
-# do not create issues for presubmit tests
-go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64
-go run main.go flaky --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test
+go run main.go flaky --auto-create-issues --dashboard=sig-etcd-presubmits --tab=pull-etcd-e2e-amd64 --max-days=14
+go run main.go flaky --auto-create-issues --dashboard=sig-etcd-presubmits --tab=pull-etcd-unit-test --max-days=14
+
+go run main.go auto-close-stale-issues --days-before-auto-close=14
 
 popd

tools/testgrid-analysis/cmd/data.go

@@ -19,6 +19,7 @@ import (
  "io"
  "net/http"
  "os"
+ "sort"
  "strings"
  "time"
 
@@ -36,20 +37,41 @@ var (
  skippedTestStatuses = make(map[int32]struct{})
 )
 
+type TabResultSummary struct {
+ DashboardName, TabName string
+ TestsWithFailures []*TestResultSummary
+ FailureRate float32
+ IssueBody string
+ allBuilds map[string]struct{}
+ failedBuilds map[string]struct{}
+}
+
 type TestResultSummary struct {
  Name string
  FullName string
  TotalRuns, FailedRuns int
  FailureRate float32
  FailureLogs []string
  IssueBody string
+ allBuilds map[string]struct{}
+ failedBuilds map[string]struct{}
 }
 
-func fetchTestResultSummaries(dashboard, tab string) []*TestResultSummary {
- // Fetch test data
- rowsURL := fmt.Sprintf("http://testgrid-data.k8s.io/api/v1/dashboards/%s/tabs/%s/rows", dashboard, tab)
- headersURL := fmt.Sprintf("http://testgrid-data.k8s.io/api/v1/dashboards/%s/tabs/%s/headers", dashboard, tab)
+func FetchTabResultSummary(dashboard, tab string) *TabResultSummary {
+ summary := TabResultSummary{DashboardName: dashboard, TabName: tab}
+ summary.analyzeTestResults()
+ return &summary
+}
 
+func (tab *TabResultSummary) dataURLs() (rowsURL, headersURL string) {
+ rowsURL = fmt.Sprintf("http://testgrid-data.k8s.io/api/v1/dashboards/%s/tabs/%s/rows", tab.DashboardName, tab.TabName)
+ headersURL = fmt.Sprintf("http://testgrid-data.k8s.io/api/v1/dashboards/%s/tabs/%s/headers", tab.DashboardName, tab.TabName)
+ return
+}
+
+func (tab *TabResultSummary) analyzeTestResults() {
+ // Fetch test data
+ rowsURL, headersURL := tab.dataURLs()
  var testData apipb.ListRowsResponse
  var headerData apipb.ListHeadersResponse
  protojson.Unmarshal(fetchJSON(rowsURL), &testData)
@@ -60,13 +82,48 @@ func fetchTestResultSummaries(dashboard, tab string) []*TestResultSummary {
  allTests = append(allTests, row.Name)
  }
 
- summaries := []*TestResultSummary{}
+ tab.allBuilds = map[string]struct{}{}
+ tab.failedBuilds = map[string]struct{}{}
+
  // Process rows
  for _, row := range testData.Rows {
- t := processRow(dashboard, tab, row, allTests, headerData.Headers)
- summaries = append(summaries, t)
+ t := processRow(tab.DashboardName, tab.TabName, row, allTests, headerData.Headers)
+ mergeMaps(t.allBuilds, tab.allBuilds)
+ mergeMaps(t.failedBuilds, tab.failedBuilds)
+ if t.FailedRuns > 0 {
+ tab.TestsWithFailures = append(tab.TestsWithFailures, t)
+ }
+ }
+ sort.Slice(tab.TestsWithFailures, func(i, j int) bool {
+ ti := tab.TestsWithFailures[i]
+ tj := tab.TestsWithFailures[j]
+ if ti.FailureRate == tj.FailureRate {
+ if ti.FailedRuns == tj.FailedRuns {
+ return ti.FullName < tj.FullName
+ }
+ return ti.FailedRuns > tj.FailedRuns
+ }
+ return ti.FailureRate > tj.FailureRate
+ })
+ if len(tab.allBuilds) > 0 {
+ tab.FailureRate = float32(len(tab.failedBuilds)) / float32(len(tab.allBuilds))
+ }
+ tab.IssueBody += fmt.Sprintf("%s#%s failed %.1f%% (%d/%d) of the time\n", tab.DashboardName, tab.TabName,
+ 100*tab.FailureRate, len(tab.failedBuilds), len(tab.allBuilds))
+ if len(tab.failedBuilds) > 0 {
+ tab.IssueBody += "<details>\n<summary><b>Recent failed test logs</b></summary>\n"
+ for _, header := range headerData.Headers {
+ if _, found := tab.failedBuilds[header.Build]; found {
+ tab.IssueBody += fmt.Sprintf("\n* %s", buildLogURL(tab.TabName, header))
+ }
+ }
+ tab.IssueBody += "\n</details>\n<details>\n<summary><b>Failed tests</b></summary>\n"
+ for _, t := range tab.TestsWithFailures {
+ tab.IssueBody += fmt.Sprintf("\n* %s failed %.1f%% (%d/%d) of the time", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns)
+ }
+ tab.IssueBody += "\n</details>\n"
  }
- return summaries
+ fmt.Println(tab.IssueBody)
 }
 
 func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests []string, headers []*apipb.ListHeadersResponse_Header) *TestResultSummary {
@@ -81,6 +138,8 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
  earliestTimeToConsider := time.Now().AddDate(0, 0, -1*maxDays)
  total := 0
  failed := 0
+ allBuilds := map[string]struct{}{}
+ failedBuilds := map[string]struct{}{}
  logs := []string{}
  for i, cell := range row.Cells {
  // ignore tests with status not in the validTestStatuses
@@ -96,10 +155,12 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
  continue
  }
  total++
+ allBuilds[header.Build] = struct{}{}
  if _, ok := failureTestStatusesInt[cell.Result]; ok {
  failed++
+ failedBuilds[header.Build] = struct{}{}
  // markdown table format of | commit | log |
- logs = append(logs, fmt.Sprintf("| %s | %s | https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), tab, header.Build))
+ logs = append(logs, fmt.Sprintf("| %s | %s | %s |", strings.Join(header.Extra, ","), header.Started.AsTime().String(), buildLogURL(tab, header)))
  }
  if maxRuns > 0 && total >= maxRuns {
  break
@@ -109,12 +170,12 @@ func processRow(dashboard, tab string, row *apipb.ListRowsResponse_Row, allTests
  t.TotalRuns = total
  t.FailureLogs = logs
  t.FailureRate = float32(failed) / float32(total)
+ t.failedBuilds = failedBuilds
+ t.allBuilds = allBuilds
  if t.FailedRuns > 0 {
- dashboardUrl := fmt.Sprintf("[%s](https://testgrid.k8s.io/%s#%s)", tab, dashboard, tab)
  t.IssueBody = fmt.Sprintf("## %s Test: %s \nTest failed %.1f%% (%d/%d) of the time\n\nfailure logs are:\n| commit | started | log |\n| --- | --- | --- |\n%s\n",
- dashboardUrl, t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n"))
+ dashboardTabURL(dashboard, tab), t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns, strings.Join(t.FailureLogs, "\n"))
  t.IssueBody += "\nPlease follow the [instructions in the contributing guide](https://github.com/etcd-io/etcd/blob/main/CONTRIBUTING.md#check-for-flaky-tests) to reproduce the issue.\n"
- fmt.Printf("%s failed %.1f%% (%d/%d) of the time\n", t.FullName, t.FailureRate*100, t.FailedRuns, t.TotalRuns)
  }
  return &t
 }
@@ -150,6 +211,29 @@ func intStatusSet(statuses []statuspb.TestStatus) map[int32]struct{} {
 }
 
 func shortenTestName(fullname string) string {
- parts := strings.Split(fullname, ".")
- return parts[len(parts)-1]
+ parts := strings.Split(fullname, "/")
+ keepParts := []string{}
+ // keep the package name of the test.
+ for i := len(parts) - 1; i >= 0; i-- {
+ part := parts[i]
+ keepParts = append([]string{part}, keepParts...)
+ if strings.Contains(part, ".") {
+ break
+ }
+ }
+ return strings.Join(keepParts, "/")
+}
+
+func mergeMaps(from, to map[string]struct{}) {
+ for k, v := range from {
+ to[k] = v
+ }
+}
+
+func dashboardTabURL(dashboard, tab string) string {
+ return fmt.Sprintf("[%s](https://testgrid.k8s.io/%s#%s)", tab, dashboard, tab)
+}
+
+func buildLogURL(tab string, header *apipb.ListHeadersResponse_Header) string {
+ return fmt.Sprintf("https://prow.k8s.io/view/gs/kubernetes-jenkins/logs/%s/%s", tab, header.Build)
 }

tools/testgrid-analysis/cmd/data_test.go

@@ -0,0 +1,51 @@
+// Copyright 2024 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+ "testing"
+)
+
+func TestShortenTestName(t *testing.T) {
+ tests := []struct {
+ testName string
+ shortName string
+ }{
+ {
+ testName: "go.etcd.io/etcd/tests/v3/common.TestKVGet/ClientTLS",
+ shortName: "common.TestKVGet/ClientTLS",
+ },
+ {
+ testName: "go.etcd.io/etcd/tests/v3/common.TestKVDelete/ClientTLS",
+ shortName: "common.TestKVDelete/ClientTLS",
+ },
+ {
+ testName: "go.etcd.io/etcd/tests/v3/common.TestLeaseGrantAndList/ClientAutoTLS/many_leases",
+ shortName: "common.TestLeaseGrantAndList/ClientAutoTLS/many_leases",
+ },
+ {
+ testName: "go.etcd.io/etcd/tests/v3/common.TestMoveLeaderWithInvalidAuth",
+ shortName: "common.TestMoveLeaderWithInvalidAuth",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.testName, func(t *testing.T) {
+ shortName := shortenTestName(tt.testName)
+ if shortName != tt.shortName {
+ t.Errorf("Want %s, got %s", tt.shortName, shortName)
+ }
+ })
+ }
+}

tools/testgrid-analysis/cmd/flaky.go

@@ -28,52 +28,59 @@ var flakyCmd = &cobra.Command{
  Run: flakyFunc,
 }
 
+var closeStaleIssuesCmd = &cobra.Command{
+ Use: "auto-close-stale-issues",
+ Short: "auto close stale flaky test issues",
+ Long: `automatically close stale Github issues for flaky test.`,
+ Run: closeStaleIssuesFunc,
+}
+
 var (
- flakyThreshold float32
- minRuns  int
- maxRuns int
- maxDays int
- createGithubIssue bool
- githubOwner string
- githubRepo  string
+ flakyThreshold  float32
+ maxSubIssuesForTestSet int
+ minRuns  int
+ maxRuns  int
+ maxDays int
+ autoCreateIssues bool
+ daysBeforeAutoClose int
 
  lineSep = "-------------------------------------------------------------"
 )
 
 func init() {
  rootCmd.AddCommand(flakyCmd)
+ rootCmd.AddCommand(closeStaleIssuesCmd)
 
- flakyCmd.Flags().BoolVar(&createGithubIssue, "create-issue", false, "create Github issue for each flaky test")
  flakyCmd.Flags().Float32Var(&flakyThreshold, "flaky-threshold", 0.1, "fraction threshold of test failures for a test to be considered flaky")
- flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be included in flaky analysis")
+ flakyCmd.Flags().IntVar(&minRuns, "min-runs", 20, "minimum test runs for a test to be created an issue for")
  flakyCmd.Flags().IntVar(&maxRuns, "max-runs", 0, "maximum test runs for a test to be included in flaky analysis, 0 to include all")
- flakyCmd.Flags().IntVar(&maxDays, "max-days", 0, "maximum days of results before today to be included in flaky analysis, 0 to include all")
- flakyCmd.Flags().StringVar(&githubOwner, "github-owner", "etcd-io", "the github organization to create the issue for")
- flakyCmd.Flags().StringVar(&githubRepo, "github-repo", "etcd", "the github repo to create the issue for")
+ flakyCmd.Flags().IntVar(&maxDays, "max-days", 30, "maximum days of results before today to be included in flaky analysis, 0 to include all")
+ flakyCmd.Flags().BoolVar(&autoCreateIssues, "auto-create-issues", false, "automatically create Github issue for flaky test")
+ flakyCmd.Flags().IntVar(&maxSubIssuesForTestSet, "max-sub-issues", 3, "maximum number of sub-issues to create for a test set")
+
+ closeStaleIssuesCmd.Flags().IntVar(&daysBeforeAutoClose, "days-before-auto-close", 30, "maximum days of no updates before an issue is automatically closed")
 }
 
 func flakyFunc(cmd *cobra.Command, args []string) {
- fmt.Printf("flaky called, for %s#%s, createGithubIssue=%v, githubRepo=%s/%s, flakyThreshold=%f, minRuns=%d\n", dashboard, tab, createGithubIssue, githubOwner, githubRepo, flakyThreshold, minRuns)
-
- allTests := fetchTestResultSummaries(dashboard, tab)
- flakyTests := []*TestResultSummary{}
- for _, t := range allTests {
- if t.TotalRuns >= minRuns && t.FailureRate >= flakyThreshold {
- flakyTests = append(flakyTests, t)
- }
- }
  fmt.Println(lineSep)
- fmt.Printf("Detected total %d flaky tests above the %.0f%% threshold for %s#%s\n", len(flakyTests), flakyThreshold*100, dashboard, tab)
+ fmt.Printf("flaky called, for %s#%s, createGithubIssue=%v, githubRepo=%s/%s, flakyThreshold=%f, minRuns=%d\n", dashboard, tab, autoCreateIssues, githubOwner, githubRepo, flakyThreshold, minRuns)
+
+ tabSummary := FetchTabResultSummary(dashboard, tab)
  fmt.Println(lineSep)
- if len(flakyTests) == 0 {
+ if tabSummary.FailureRate < flakyThreshold {
+ fmt.Printf("Failure rate for test set %s#%s is %.1f%%, below the flaky threshold %.0f%%\n", dashboard, tab, tabSummary.FailureRate*100, flakyThreshold*100)
  return
  }
- for _, t := range flakyTests {
- fmt.Println(lineSep)
- fmt.Println(t.IssueBody)
- fmt.Println(lineSep)
- }
- if createGithubIssue {
- createIssues(flakyTests, []string{"type/flake"})
+ fmt.Printf("Failure rate for test set %s#%s is %.1f%%, above the flaky threshold %.0f%%\n", dashboard, tab, tabSummary.FailureRate*100, flakyThreshold*100)
+ if autoCreateIssues {
+ createIssues(tabSummary, minRuns, maxSubIssuesForTestSet, []string{"type/flake"})
  }
+ fmt.Println(lineSep)
+}
+
+func closeStaleIssuesFunc(cmd *cobra.Command, args []string) {
+ fmt.Println(lineSep)
+ fmt.Printf("auto close stale issues with no updates for %d days in githubRepo=%s/%s\n", daysBeforeAutoClose, githubOwner, githubRepo)
+ closeStaleIssues(daysBeforeAutoClose, []string{"type/flake"})
+ fmt.Println(lineSep)
 }