fix(automerge): implement exponential backoff (#181)

* fix(automerge): implement exponential backoff

* Update internal/pkg/githubapi/github.go

go.mod

@@ -5,10 +5,8 @@ go 1.21
 require (
 	github.com/alexliesenfeld/health v0.8.0
 	github.com/bradleyfalzon/ghinstallation/v2 v2.10.0
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/go-test/deep v1.1.0
-	github.com/google/go-github/v52 v52.0.0
-	github.com/google/go-github/v54 v54.0.0
-	github.com/google/go-github/v55 v55.0.0
 	github.com/google/go-github/v62 v62.0.0
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/hexops/gotextdiff v1.0.3
@@ -21,49 +19,42 @@ require (
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
 	golang.org/x/oauth2 v0.20.0
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
 	github.com/a8m/envsubst v1.4.2 // indirect
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/elliotchance/orderedmap v1.5.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/goccy/go-yaml v1.11.3 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/go-github/v50 v50.1.0 // indirect
-	github.com/google/go-github/v56 v56.0.0 // indirect
 	github.com/google/go-github/v59 v59.0.0 // indirect
 	github.com/google/go-github/v60 v60.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jinzhu/copier v0.4.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/shurcooL/graphql v0.0.0-20220606043923-3cf50f8a0a29 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
-	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/net v0.24.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

internal/pkg/githubapi/clients.go

@@ -2,13 +2,14 @@ package githubapi
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"os"
 	"strconv"
 	"time"
 
 	"github.com/bradleyfalzon/ghinstallation/v2"
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	lru "github.com/hashicorp/golang-lru/v2"
 	"github.com/shurcooL/githubv4"
 	log "github.com/sirupsen/logrus"
@@ -41,25 +42,17 @@ func getAppInstallationId(githubAppPrivateKeyPath string, githubAppId int64, git
 	if err != nil {
 		panic(err)
 	}
-	var tempClient *github.Client
+	tempClient := github.NewClient(
+		&http.Client{
+			Transport: atr,
+			Timeout:   time.Second * 30,
+		})
 
 	if githubRestAltURL != "" {
-		tempClient, err = github.NewEnterpriseClient(
-			githubRestAltURL,
-			githubRestAltURL,
-			&http.Client{
-				Transport: atr,
-				Timeout:   time.Second * 30,
-			})
+		tempClient, err = tempClient.WithEnterpriseURLs(githubRestAltURL, githubRestAltURL)
 		if err != nil {
 			log.Fatalf("failed to create git client for app: %v\n", err)
 		}
-	} else {
-		tempClient = github.NewClient(
-			&http.Client{
-				Transport: atr,
-				Timeout:   time.Second * 30,
-			})
 	}
 
 	installations, _, err := tempClient.Apps.ListInstallations(ctx, &github.ListOptions{})
@@ -88,7 +81,7 @@ func createGithubAppRestClient(githubAppPrivateKeyPath string, githubAppId int64
 
 	if githubRestAltURL != "" {
 		itr.BaseURL = githubRestAltURL
-		client, _ = github.NewEnterpriseClient(githubRestAltURL, githubRestAltURL, &http.Client{Transport: itr})
+		client, _ = github.NewClient(&http.Client{Transport: itr}).WithEnterpriseURLs(githubRestAltURL, githubRestAltURL)
 	} else {
 		client = github.NewClient(&http.Client{Transport: itr})
 	}
@@ -100,11 +93,9 @@ func createGithubRestClient(githubOauthToken string, githubRestAltURL string, ct
 		&oauth2.Token{AccessToken: githubOauthToken},
 	)
 	tc := oauth2.NewClient(ctx, ts)
-	var client *github.Client
+	client := github.NewClient(tc)
 	if githubRestAltURL != "" {
-		client, _ = github.NewEnterpriseClient(githubRestAltURL, githubRestAltURL, tc)
-	} else {
-		client = github.NewClient(tc)
+		client, _ = client.WithEnterpriseURLs(githubRestAltURL, githubRestAltURL)
 	}
 
 	return client
@@ -147,8 +138,8 @@ func createGhAppClientPair(ctx context.Context, githubAppId int64, owner string,
 	githubAppPrivateKeyPath := getCrucialEnv(ghAppPKeyPathEnvVarName)
 	githubHost := getEnv("GITHUB_HOST", "")
 	if githubHost != "" {
-		githubRestAltURL = "https://" + githubHost + "/api/v3"
-		githubGraphqlAltURL = "https://" + githubHost + "/api/graphql"
+		githubRestAltURL = fmt.Sprintf("https://%s/api/v3", githubHost)
+		githubGraphqlAltURL = fmt.Sprintf("https://%s/api/graphql", githubHost)
 		log.Infof("Github REST API endpoint is configured to %s", githubRestAltURL)
 		log.Infof("Github graphql API endpoint is configured to %s", githubGraphqlAltURL)
 	} else {
@@ -174,8 +165,8 @@ func createGhTokenClientPair(ctx context.Context, ghOauthToken string) GhClientP
 	var githubGraphqlAltURL string
 	githubHost := getEnv("GITHUB_HOST", "")
 	if githubHost != "" {
-		githubRestAltURL = "https://" + githubHost + "/api/v3"
-		githubGraphqlAltURL = "https://" + githubHost + "/api/graphql"
+		githubRestAltURL = fmt.Sprintf("https://%s/api/v3", githubHost)
+		githubGraphqlAltURL = fmt.Sprintf("https://%s/api/graphql", githubHost)
 		log.Infof("Github REST API endpoint is configured to %s", githubRestAltURL)
 		log.Infof("Github graphql API endpoint is configured to %s", githubGraphqlAltURL)
 	} else {

internal/pkg/githubapi/drift_detection.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	"github.com/hexops/gotextdiff"
 	"github.com/hexops/gotextdiff/myers"
 	"github.com/hexops/gotextdiff/span"

internal/pkg/githubapi/drift_detection_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/go-test/deep"
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	"github.com/hexops/gotextdiff"
 	"github.com/hexops/gotextdiff/myers"
 	"github.com/hexops/gotextdiff/span"

internal/pkg/githubapi/github.go

@@ -13,7 +13,8 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/google/go-github/v52/github"
+	"github.com/cenkalti/backoff/v4"
+	"github.com/google/go-github/v62/github"
 	lru "github.com/hashicorp/golang-lru/v2"
 	log "github.com/sirupsen/logrus"
 	cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
@@ -414,14 +415,40 @@ func handleMergedPrEvent(ghPrClientDetails GhPrClientDetails, prApproverGithubCl
 }
 
 func MergePr(details GhPrClientDetails, number *int) error {
-	_, resp, err := details.GhClientPair.v3Client.PullRequests.Merge(details.Ctx, details.Owner, details.Repo, *number, "Auto-merge", nil)
-	prom.InstrumentGhCall(resp)
+	operation := func() error {
+		err := tryMergePR(details, number)
+		if err != nil {
+			if isMergeErrorRetryable(err.Error()) {
+				if err != nil {
+					details.PrLogger.Warnf("Failed to merge PR: transient err=%v", err)
+				}
+				return err
+			}
+			details.PrLogger.Errorf("Failed to merge PR: permanent err=%v", err)
+			return backoff.Permanent(err)
+		}
+		return nil
+	}
+
+	// Using default values, see https://pkg.go.dev/github.com/cenkalti/backoff#pkg-constants
+	err := backoff.Retry(operation, backoff.NewExponentialBackOff())
 	if err != nil {
-		details.PrLogger.Errorf("Failed to merge PR: err=%v", err)
+		details.PrLogger.Errorf("Failed to merge PR: backoff err=%v", err)
 	}
+
 	return err
 }
 
+func tryMergePR(details GhPrClientDetails, number *int) error {
+	_, resp, err := details.GhClientPair.v3Client.PullRequests.Merge(details.Ctx, details.Owner, details.Repo, *number, "Auto-merge", nil)
+	prom.InstrumentGhCall(resp)
+	return err
+}
+
+func isMergeErrorRetryable(errMessage string) bool {
+	return strings.Contains(errMessage, "405") && strings.Contains(errMessage, "try the merge again")
+}
+
 func (pm *prMetadata) DeSerialize(s string) error {
 	decoded, err := base64.StdEncoding.DecodeString(s)
 	if err != nil {
@@ -711,7 +738,7 @@ func createCommit(ghPrClientDetails GhPrClientDetails, treeEntries []*github.Tre
 		Tree:    tree,
 	}
 
-	commit, resp, err := ghPrClientDetails.GhClientPair.v3Client.Git.CreateCommit(ghPrClientDetails.Ctx, ghPrClientDetails.Owner, ghPrClientDetails.Repo, newCommitConfig)
+	commit, resp, err := ghPrClientDetails.GhClientPair.v3Client.Git.CreateCommit(ghPrClientDetails.Ctx, ghPrClientDetails.Owner, ghPrClientDetails.Repo, newCommitConfig, nil)
 	prom.InstrumentGhCall(resp)
 	if err != nil {
 		ghPrClientDetails.PrLogger.Errorf("Failed to create Git commit: err=%s\n", err) // TODO comment this error to PR

internal/pkg/githubapi/promotion.go

@@ -6,7 +6,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	log "github.com/sirupsen/logrus"
 	cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
 	prom "github.com/wayfair-incubator/telefonistka/internal/pkg/prometheus"

internal/pkg/githubapi/promotion_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/go-test/deep"
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	"github.com/migueleliasweb/go-github-mock/src/mock"
 	log "github.com/sirupsen/logrus"
 	cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"

internal/pkg/githubapi/webhook_proxy.go

@@ -9,7 +9,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	log "github.com/sirupsen/logrus"
 	"github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
 	prom "github.com/wayfair-incubator/telefonistka/internal/pkg/prometheus"

internal/pkg/githubapi/webhook_proxy_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/go-test/deep"
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	cfg "github.com/wayfair-incubator/telefonistka/internal/pkg/configuration"
 )
 

internal/pkg/prometheus/prometheus.go

@@ -5,7 +5,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 )

internal/pkg/prometheus/prometheus_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 
 	"github.com/go-test/deep"
-	"github.com/google/go-github/v52/github"
+	"github.com/google/go-github/v62/github"
 	"github.com/prometheus/client_golang/prometheus"
 )