fix(automerge): implement exponential backoff (#181)

* fix(automerge): implement exponential backoff

* Update internal/pkg/githubapi/github.go

go.mod

@@ -5,10 +5,8 @@ go 1.21
 require (
  github.com/alexliesenfeld/health v0.8.0
  github.com/bradleyfalzon/ghinstallation/v2 v2.10.0
+ github.com/cenkalti/backoff/v4 v4.3.0
  github.com/go-test/deep v1.1.0
- github.com/google/go-github/v52 v52.0.0
- github.com/google/go-github/v54 v54.0.0
- github.com/google/go-github/v55 v55.0.0
  github.com/google/go-github/v62 v62.0.0
  github.com/hashicorp/golang-lru/v2 v2.0.7
  github.com/hexops/gotextdiff v1.0.3
@@ -21,49 +19,42 @@ require (
  golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
  golang.org/x/oauth2 v0.20.0
  gopkg.in/yaml.v2 v2.4.0
- gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
- github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
  github.com/a8m/envsubst v1.4.2 // indirect
  github.com/alecthomas/participle/v2 v2.1.1 // indirect
  github.com/beorn7/perks v1.0.1 // indirect
  github.com/cespare/xxhash/v2 v2.2.0 // indirect
- github.com/cloudflare/circl v1.3.3 // indirect
  github.com/dimchansky/utfbom v1.1.1 // indirect
  github.com/elliotchance/orderedmap v1.5.1 // indirect
  github.com/fatih/color v1.16.0 // indirect
  github.com/goccy/go-json v0.10.2 // indirect
  github.com/goccy/go-yaml v1.11.3 // indirect
  github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
- github.com/golang/protobuf v1.5.3 // indirect
- github.com/google/go-github/v50 v50.1.0 // indirect
- github.com/google/go-github/v56 v56.0.0 // indirect
  github.com/google/go-github/v59 v59.0.0 // indirect
  github.com/google/go-github/v60 v60.0.0 // indirect
  github.com/google/go-querystring v1.1.0 // indirect
  github.com/gorilla/mux v1.8.0 // indirect
  github.com/inconshreveable/mousetrap v1.1.0 // indirect
  github.com/jinzhu/copier v0.4.0 // indirect
+ github.com/kr/text v0.2.0 // indirect
  github.com/magiconair/properties v1.8.7 // indirect
  github.com/mattn/go-colorable v0.1.13 // indirect
  github.com/mattn/go-isatty v0.0.20 // indirect
- github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
  github.com/pelletier/go-toml/v2 v2.2.2 // indirect
  github.com/prometheus/client_model v0.5.0 // indirect
  github.com/prometheus/common v0.48.0 // indirect
  github.com/prometheus/procfs v0.12.0 // indirect
  github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect
  github.com/spf13/pflag v1.0.5 // indirect
  github.com/yuin/gopher-lua v1.1.1 // indirect
- golang.org/x/crypto v0.22.0 // indirect
  golang.org/x/net v0.24.0 // indirect
  golang.org/x/sys v0.19.0 // indirect
  golang.org/x/text v0.15.0 // indirect
  golang.org/x/time v0.3.0 // indirect
  golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
- google.golang.org/appengine v1.6.7 // indirect
  google.golang.org/protobuf v1.33.0 // indirect
  gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/pkg/githubapi/clients.go

@@ -2,13 +2,14 @@ package githubapi
 
 import (
  "context"
+ "fmt"
  "net/http"
  "os"
  "strconv"
  "time"
 
  "github.com/bradleyfalzon/ghinstallation/v2"
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  lru "github.com/hashicorp/golang-lru/v2"
  "github.com/shurcooL/githubv4"
  log "github.com/sirupsen/logrus"
@@ -41,25 +42,17 @@ func getAppInstallationId(githubAppPrivateKeyPath string, githubAppId int64, git
  if err != nil {
  panic(err)
  }
- var tempClient *github.Client
+ tempClient := github.NewClient(
+ &http.Client{
+ Transport: atr,
+ Timeout: time.Second * 30,
+ })
 
  if githubRestAltURL != "" {
- tempClient, err = github.NewEnterpriseClient(
- githubRestAltURL,
- githubRestAltURL,
- &http.Client{
- Transport: atr,
- Timeout: time.Second * 30,
- })
+ tempClient, err = tempClient.WithEnterpriseURLs(githubRestAltURL, githubRestAltURL)
  if err != nil {
  log.Fatalf("failed to create git client for app: %v\n", err)
  }
- } else {
- tempClient = github.NewClient(
- &http.Client{
- Transport: atr,
- Timeout: time.Second * 30,
- })
  }
 
  installations, _, err := tempClient.Apps.ListInstallations(ctx, &github.ListOptions{})
@@ -88,7 +81,7 @@ func createGithubAppRestClient(githubAppPrivateKeyPath string, githubAppId int64
 
  if githubRestAltURL != "" {
  itr.BaseURL = githubRestAltURL
- client, _ = github.NewEnterpriseClient(githubRestAltURL, githubRestAltURL, &http.Client{Transport: itr})
+ client, _ = github.NewClient(&http.Client{Transport: itr}).WithEnterpriseURLs(githubRestAltURL, githubRestAltURL)
  } else {
  client = github.NewClient(&http.Client{Transport: itr})
  }
@@ -100,11 +93,9 @@ func createGithubRestClient(githubOauthToken string, githubRestAltURL string, ct
  &oauth2.Token{AccessToken: githubOauthToken},
  )
  tc := oauth2.NewClient(ctx, ts)
- var client *github.Client
+ client := github.NewClient(tc)
  if githubRestAltURL != "" {
- client, _ = github.NewEnterpriseClient(githubRestAltURL, githubRestAltURL, tc)
- } else {
- client = github.NewClient(tc)
+ client, _ = client.WithEnterpriseURLs(githubRestAltURL, githubRestAltURL)
  }
 
  return client
@@ -147,8 +138,8 @@ func createGhAppClientPair(ctx context.Context, githubAppId int64, owner string,
  githubAppPrivateKeyPath := getCrucialEnv(ghAppPKeyPathEnvVarName)
  githubHost := getEnv("GITHUB_HOST", "")
  if githubHost != "" {
- githubRestAltURL = "https://" + githubHost + "/api/v3"
- githubGraphqlAltURL = "https://" + githubHost + "/api/graphql"
+ githubRestAltURL = fmt.Sprintf("https://%s/api/v3", githubHost)
+ githubGraphqlAltURL = fmt.Sprintf("https://%s/api/graphql", githubHost)
  log.Infof("Github REST API endpoint is configured to %s", githubRestAltURL)
  log.Infof("Github graphql API endpoint is configured to %s", githubGraphqlAltURL)
  } else {
@@ -174,8 +165,8 @@ func createGhTokenClientPair(ctx context.Context, ghOauthToken string) GhClientP
  var githubGraphqlAltURL string
  githubHost := getEnv("GITHUB_HOST", "")
  if githubHost != "" {
- githubRestAltURL = "https://" + githubHost + "/api/v3"
- githubGraphqlAltURL = "https://" + githubHost + "/api/graphql"
+ githubRestAltURL = fmt.Sprintf("https://%s/api/v3", githubHost)
+ githubGraphqlAltURL = fmt.Sprintf("https://%s/api/graphql", githubHost)
  log.Infof("Github REST API endpoint is configured to %s", githubRestAltURL)
  log.Infof("Github graphql API endpoint is configured to %s", githubGraphqlAltURL)
  } else {

internal/pkg/githubapi/drift_detection.go

@@ -5,7 +5,7 @@ import (
  "fmt"
  "strings"
 
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  "github.com/hexops/gotextdiff"
  "github.com/hexops/gotextdiff/myers"
  "github.com/hexops/gotextdiff/span"

internal/pkg/githubapi/drift_detection_test.go

@@ -5,7 +5,7 @@ import (
  "testing"
 
  "github.com/go-test/deep"
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  "github.com/hexops/gotextdiff"
  "github.com/hexops/gotextdiff/myers"
  "github.com/hexops/gotextdiff/span"

internal/pkg/githubapi/github.go

@@ -13,7 +13,8 @@ import (
  "strings"
  "text/template"
 
- "github.com/google/go-github/v52/github"
+ "github.com/cenkalti/backoff/v4"
+ "github.com/google/go-github/v62/github"
  lru "github.com/hashicorp/golang-lru/v2"
  log "github.com/sirupsen/logrus"
  cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
@@ -414,14 +415,40 @@ func handleMergedPrEvent(ghPrClientDetails GhPrClientDetails, prApproverGithubCl
 }
 
 func MergePr(details GhPrClientDetails, number *int) error {
- _, resp, err := details.GhClientPair.v3Client.PullRequests.Merge(details.Ctx, details.Owner, details.Repo, *number, "Auto-merge", nil)
- prom.InstrumentGhCall(resp)
+ operation := func() error {
+ err := tryMergePR(details, number)
+ if err != nil {
+ if isMergeErrorRetryable(err.Error()) {
+ if err != nil {
+ details.PrLogger.Warnf("Failed to merge PR: transient err=%v", err)
+ }
+ return err
+ }
+ details.PrLogger.Errorf("Failed to merge PR: permanent err=%v", err)
+ return backoff.Permanent(err)
+ }
+ return nil
+ }
+
+ // Using default values, see https://pkg.go.dev/github.com/cenkalti/backoff#pkg-constants
+ err := backoff.Retry(operation, backoff.NewExponentialBackOff())
  if err != nil {
- details.PrLogger.Errorf("Failed to merge PR: err=%v", err)
+ details.PrLogger.Errorf("Failed to merge PR: backoff err=%v", err)
  }
+
  return err
 }
 
+func tryMergePR(details GhPrClientDetails, number *int) error {
+ _, resp, err := details.GhClientPair.v3Client.PullRequests.Merge(details.Ctx, details.Owner, details.Repo, *number, "Auto-merge", nil)
+ prom.InstrumentGhCall(resp)
+ return err
+}
+
+func isMergeErrorRetryable(errMessage string) bool {
+ return strings.Contains(errMessage, "405") && strings.Contains(errMessage, "try the merge again")
+}
+
 func (pm *prMetadata) DeSerialize(s string) error {
  decoded, err := base64.StdEncoding.DecodeString(s)
  if err != nil {
@@ -711,7 +738,7 @@ func createCommit(ghPrClientDetails GhPrClientDetails, treeEntries []*github.Tre
  Tree: tree,
  }
 
- commit, resp, err := ghPrClientDetails.GhClientPair.v3Client.Git.CreateCommit(ghPrClientDetails.Ctx, ghPrClientDetails.Owner, ghPrClientDetails.Repo, newCommitConfig)
+ commit, resp, err := ghPrClientDetails.GhClientPair.v3Client.Git.CreateCommit(ghPrClientDetails.Ctx, ghPrClientDetails.Owner, ghPrClientDetails.Repo, newCommitConfig, nil)
  prom.InstrumentGhCall(resp)
  if err != nil {
  ghPrClientDetails.PrLogger.Errorf("Failed to create Git commit: err=%s\n", err) // TODO comment this error to PR

internal/pkg/githubapi/promotion.go

@@ -6,7 +6,7 @@ import (
  "sort"
  "strings"
 
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  log "github.com/sirupsen/logrus"
  cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
  prom "github.com/wayfair-incubator/telefonistka/internal/pkg/prometheus"

internal/pkg/githubapi/promotion_test.go

@@ -6,7 +6,7 @@ import (
  "testing"
 
  "github.com/go-test/deep"
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  "github.com/migueleliasweb/go-github-mock/src/mock"
  log "github.com/sirupsen/logrus"
  cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"

internal/pkg/githubapi/webhook_proxy.go

@@ -9,7 +9,7 @@ import (
  "regexp"
  "strings"
 
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  log "github.com/sirupsen/logrus"
  "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
  prom "github.com/wayfair-incubator/telefonistka/internal/pkg/prometheus"

internal/pkg/githubapi/webhook_proxy_test.go

@@ -9,7 +9,7 @@ import (
  "testing"
 
  "github.com/go-test/deep"
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
 )
 

internal/pkg/prometheus/prometheus.go

@@ -5,7 +5,7 @@ import (
  "strconv"
  "strings"
 
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  "github.com/prometheus/client_golang/prometheus"
  "github.com/prometheus/client_golang/prometheus/promauto"
 )

internal/pkg/prometheus/prometheus_test.go

@@ -6,7 +6,7 @@ import (
  "testing"
 
  "github.com/go-test/deep"
- "github.com/google/go-github/v52/github"
+ "github.com/google/go-github/v62/github"
  "github.com/prometheus/client_golang/prometheus"
 )