fix: implement backoff + retry when GitLab SetCommitStatus returns 409

go.mod

@@ -23,6 +23,7 @@ require (
  github.com/hashicorp/go-version v1.6.0
  github.com/hashicorp/golang-lru/v2 v2.0.7
  github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72
+ github.com/jpillora/backoff v1.0.0
  github.com/kr/pretty v0.3.1
  github.com/mcdafydd/go-azuredevops v0.12.1
  github.com/microcosm-cc/bluemonday v1.0.26

go.sum

@@ -270,6 +270,7 @@ github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

server/events/vcs/gitlab_client.go

@@ -22,14 +22,13 @@ import (
  "strings"
  "time"
 
- "github.com/runatlantis/atlantis/server/events/command"
- "github.com/runatlantis/atlantis/server/events/vcs/common"
-
  version "github.com/hashicorp/go-version"
+ "github.com/jpillora/backoff"
  "github.com/pkg/errors"
- "github.com/runatlantis/atlantis/server/logging"
-
+ "github.com/runatlantis/atlantis/server/events/command"
  "github.com/runatlantis/atlantis/server/events/models"
+ "github.com/runatlantis/atlantis/server/events/vcs/common"
+ "github.com/runatlantis/atlantis/server/logging"
  gitlab "github.com/xanzy/go-gitlab"
 )
 
@@ -439,17 +438,63 @@ func (g *GitlabClient) UpdateStatus(logger logging.SimpleLogging, repo models.Re
  }
  }
 
- _, resp, err := g.Client.Commits.SetCommitStatus(repo.FullName, pull.HeadCommit, &gitlab.SetCommitStatusOptions{
- State: gitlabState,
- Context: gitlab.Ptr(src),
- Description: gitlab.Ptr(description),
- TargetURL: &url,
- Ref: gitlab.Ptr(refTarget),
- })
- if resp != nil {
- logger.Debug("POST /projects/%s/statuses/%s returned: %d", repo.FullName, pull.HeadCommit, resp.StatusCode)
+ var (
+ resp *gitlab.Response
+ maxAttempts = 10
+ b = &backoff.Backoff{Jitter: true}
+ )
+
+ for i := 0; i <= maxAttempts; i++ {
+ logger := logger.With(
+ "attempt", i+1,
+ "max_attempts", maxAttempts,
+ "repo", repo.FullName,
+ "commit", pull.HeadCommit,
+ "state", state.String(),
+ )
+
+ _, resp, err = g.Client.Commits.SetCommitStatus(repo.FullName, pull.HeadCommit, &gitlab.SetCommitStatusOptions{
+ State: gitlabState,
+ Context: gitlab.Ptr(src),
+ Description: gitlab.Ptr(description),
+ TargetURL: &url,
+ Ref: gitlab.Ptr(refTarget),
+ })
+
+ if resp != nil {
+ logger.Debug("POST /projects/%s/statuses/%s returned: %d", repo.FullName, pull.HeadCommit, resp.StatusCode)
+
+ // GitLab returns a `409 Conflict` status when the commit pipeline status is being changed/locked by another request,
+ // which is likely to happen if you use [`--parallel-pool-size > 1`] and [`parallel-plan|apply`].
+ //
+ // The likelihood of this happening is increased when the number of parallel apply jobs is increased.
+ //
+ // Returning the [err] without retrying will permanently leave the GitLab commit status in a "running" state,
+ // which would prevent Atlantis from merging the merge request on [apply].
+ //
+ // GitLab does not allow merge requests to be merged when the pipeline status is "running."
+
+ if resp.StatusCode == http.StatusConflict {
+ sleep := b.ForAttempt(float64(i))
+
+ logger.With("retry_in", sleep).Warn("GitLab returned HTTP [409 Conflict] when updating commit status")
+ time.Sleep(sleep)
+
+ continue
+ }
+ }
+
+ // Log we got a 200 OK response from GitLab after at least one retry to help with debugging/understanding delays/errors.
+ if err == nil && i > 0 {
+ logger.Info("GitLab returned HTTP [200 OK] after updating commit status")
+ }
+
+ // Return the err, which might be nil if everything worked out
+ return err
  }
- return err
+
+ // If we got here, we've exhausted all attempts to update the commit status and still failed, so return the error upstream
+ return errors.Wrap(err, fmt.Sprintf("failed to update commit status for '%s' @ '%s' to '%s' after %d attempts", repo.FullName, pull.HeadCommit, src, maxAttempts))
 }
 
 func (g *GitlabClient) GetMergeRequest(logger logging.SimpleLogging, repoFullName string, pullNum int) (*gitlab.MergeRequest, error) {
@@ -471,7 +516,7 @@ func (g *GitlabClient) WaitForSuccessPipeline(logger logging.SimpleLogging, ctx
  case <-ctx.Done():
  // validation check time out
  cancel()
- return //ctx.Err()
+ return // ctx.Err()
 
  default:
  mr, _ := g.GetMergeRequest(logger, pull.BaseRepo.FullName, pull.Num)