From 9f1d2fad1c07cd787f1f9ae7cdd247b5e8d46542 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Fri, 26 Jul 2024 15:40:45 -0400 Subject: [PATCH 01/48] use non-public repo in test --- pkg/sources/github/github_integration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 910490a76685..c88207e276c3 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -870,8 +870,8 @@ func TestSource_Chunks_TargetedScan(t *testing.T) { queryCriteria: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ - Repository: "test_keys", - Link: "https://github.com/trufflesecurity/test_keys/blob/fbc14303ffbf8fb1c2c1914e8dda7d0121633aca/keys#L4", + Repository: "test_keys_nonpublic", + Link: "https://github.com/trufflesecurity/test_keys_nonpublic/blob/fbc14303ffbf8fb1c2c1914e8dda7d0121633aca/keys#L4", Commit: "fbc14303ffbf8fb1c2c1914e8dda7d0121633aca", File: "keys", }, From 4df00479f59e2a439c9b75609b8263529b11b965 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 11:53:50 -0400 Subject: [PATCH 02/48] split out newBasicAuthClient --- pkg/sources/github/client.go | 33 +++++++++++++++++++++++++++++++++ pkg/sources/github/github.go | 16 +--------------- 2 files changed, 34 insertions(+), 15 deletions(-) create mode 100644 pkg/sources/github/client.go diff --git a/pkg/sources/github/client.go b/pkg/sources/github/client.go new file mode 100644 index 000000000000..776ff0bd27ad --- /dev/null +++ b/pkg/sources/github/client.go @@ -0,0 +1,33 @@ +package github + +import ( + "net/http" + "strings" + + "github.com/google/go-github/v62/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" +) + +type client struct { + github.Client +} + +func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) { + // If we're using public GitHub, make a regular client. + // Otherwise, make an enterprise client. + if strings.EqualFold(apiEndpoint, cloudEndpoint) { + return github.NewClient(httpClient), nil + } + + return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) +} + +func newBasicAuthClient(basicAuth *credentialspb.BasicAuth, apiEndpoint string) (*github.Client, error) { + httpClient := common.RetryableHTTPClientTimeout(60) + httpClient.Transport = &github.BasicAuthTransport{ + Username: basicAuth.Username, + Password: basicAuth.Password, + } + return createGitHubClient(httpClient, apiEndpoint) +} diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 0cd1b4209000..2cf310402c17 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -466,11 +466,7 @@ RepoLoop: } func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, basicAuth *credentialspb.BasicAuth) error { - s.httpClient.Transport = &github.BasicAuthTransport{ - Username: basicAuth.Username, - Password: basicAuth.Password, - } - ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) + ghClient, err := newBasicAuthClient(basicAuth, apiEndpoint) if err != nil { s.log.Error(err, "error creating GitHub client") } @@ -675,16 +671,6 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app * return installationClient, nil } -func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) { - // If we're using public GitHub, make a regular client. - // Otherwise, make an enterprise client. - if strings.EqualFold(apiEndpoint, cloudEndpoint) { - return github.NewClient(httpClient), nil - } - - return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) -} - func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error { var scannedCount uint64 = 1 From b9e382e8a9717ee09a86cbaa8ded40a865349bf9 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 11:55:49 -0400 Subject: [PATCH 03/48] create newUnauthenticatedClient --- pkg/sources/github/client.go | 4 ++++ pkg/sources/github/github.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/sources/github/client.go b/pkg/sources/github/client.go index 776ff0bd27ad..bd261057baa2 100644 --- a/pkg/sources/github/client.go +++ b/pkg/sources/github/client.go @@ -31,3 +31,7 @@ func newBasicAuthClient(basicAuth *credentialspb.BasicAuth, apiEndpoint string) } return createGitHubClient(httpClient, apiEndpoint) } + +func newUnauthenticatedClient(apiEndpoint string) (*github.Client, error) { + return createGitHubClient(common.RetryableHTTPClientTimeout(60), apiEndpoint) +} diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 2cf310402c17..8acdfff9679c 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -491,7 +491,7 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas } func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint string) { - ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) + ghClient, err := newUnauthenticatedClient(apiEndpoint) if err != nil { s.log.Error(err, "error creating GitHub client") } From 72a1ed30e20481eb67a2f3e489a65f05e02646f4 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 12:02:39 -0400 Subject: [PATCH 04/48] create newTokenClient --- pkg/sources/github/client.go | 10 ++++++++++ pkg/sources/github/github.go | 13 +------------ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/pkg/sources/github/client.go b/pkg/sources/github/client.go index bd261057baa2..645e0d5a9575 100644 --- a/pkg/sources/github/client.go +++ b/pkg/sources/github/client.go @@ -7,6 +7,7 @@ import ( "github.com/google/go-github/v62/github" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" + "golang.org/x/oauth2" ) type client struct { @@ -32,6 +33,15 @@ func newBasicAuthClient(basicAuth *credentialspb.BasicAuth, apiEndpoint string) return createGitHubClient(httpClient, apiEndpoint) } +func newTokenClient(token, apiEndpoint string) (*github.Client, error) { + httpClient := common.RetryableHTTPClientTimeout(60) + httpClient.Transport = &oauth2.Transport{ + Base: httpClient.Transport, + Source: oauth2.ReuseTokenSource(nil, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})), + } + return createGitHubClient(httpClient, apiEndpoint) +} + func newUnauthenticatedClient(apiEndpoint string) (*github.Client, error) { return createGitHubClient(common.RetryableHTTPClientTimeout(60), apiEndpoint) } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 8acdfff9679c..0287f34eaa3a 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -518,18 +518,7 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri // Needed for clones. s.githubToken = token - // Needed to list repos. - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - s.httpClient.Transport = &oauth2.Transport{ - Base: s.httpClient.Transport, - Source: oauth2.ReuseTokenSource(nil, ts), - } - - // If we're using public GitHub, make a regular client. - // Otherwise, make an enterprise client. - ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) + ghClient, err := newTokenClient(token, apiEndpoint) if err != nil { s.log.Error(err, "error creating GitHub client") } From 3c42ce1919d29b206cebd465233dca569886d78a Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 15:10:44 -0400 Subject: [PATCH 05/48] create (unused) newAppClient --- pkg/sources/github/client.go | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/pkg/sources/github/client.go b/pkg/sources/github/client.go index 645e0d5a9575..567712cac9ad 100644 --- a/pkg/sources/github/client.go +++ b/pkg/sources/github/client.go @@ -2,8 +2,10 @@ package github import ( "net/http" + "strconv" "strings" + "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v62/github" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" @@ -24,6 +26,51 @@ func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Cl return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) } +func newAppClient(app *credentialspb.GitHubApp, apiEndpoint string) (*github.Client, *github.Client, error) { + installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) + if err != nil { + return nil, nil, err + } + + appID, err := strconv.ParseInt(app.AppId, 10, 64) + if err != nil { + return nil, nil, err + } + + httpClient := common.RetryableHTTPClientTimeout(60) + installationTransport, err := ghinstallation.NewAppsTransport( + httpClient.Transport, + appID, + []byte(app.PrivateKey)) + if err != nil { + return nil, nil, err + } + installationTransport.BaseURL = apiEndpoint + installationHttpClient := common.RetryableHTTPClientTimeout(60) + installationHttpClient.Transport = installationTransport + installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + if err != nil { + return nil, nil, err + } + + apiTransport, err := ghinstallation.New( + httpClient.Transport, + appID, + installationID, + []byte(app.PrivateKey)) + if err != nil { + return nil, nil, err + } + apiTransport.BaseURL = apiEndpoint + httpClient.Transport = apiTransport + apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + if err != nil { + return nil, nil, err + } + + return apiClient, installationClient, nil +} + func newBasicAuthClient(basicAuth *credentialspb.BasicAuth, apiEndpoint string) (*github.Client, error) { httpClient := common.RetryableHTTPClientTimeout(60) httpClient.Transport = &github.BasicAuthTransport{ From 5890aa7794619ff87b7c444c1501f139719a8c88 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 16:03:02 -0400 Subject: [PATCH 06/48] stash --- pkg/sources/github/client.go | 77 +++++++++++++++++++++++------------- pkg/sources/github/github.go | 31 +++++++++++++-- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/pkg/sources/github/client.go b/pkg/sources/github/client.go index 567712cac9ad..c6a725272461 100644 --- a/pkg/sources/github/client.go +++ b/pkg/sources/github/client.go @@ -3,7 +3,6 @@ package github import ( "net/http" "strconv" - "strings" "github.com/bradleyfalzon/ghinstallation/v2" "github.com/google/go-github/v62/github" @@ -12,45 +11,39 @@ import ( "golang.org/x/oauth2" ) -type client struct { - github.Client +type clients struct { + httpClient *http.Client + apiClient *github.Client + installationClient *github.Client + githubToken string } -func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) { - // If we're using public GitHub, make a regular client. - // Otherwise, make an enterprise client. - if strings.EqualFold(apiEndpoint, cloudEndpoint) { - return github.NewClient(httpClient), nil - } - - return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) -} - -func newAppClient(app *credentialspb.GitHubApp, apiEndpoint string) (*github.Client, *github.Client, error) { +func newAppClients(apiEndpoint string, app *credentialspb.GitHubApp) (clients, error) { installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) if err != nil { - return nil, nil, err + return clients{}, err } appID, err := strconv.ParseInt(app.AppId, 10, 64) if err != nil { - return nil, nil, err + return clients{}, err } httpClient := common.RetryableHTTPClientTimeout(60) + installationTransport, err := ghinstallation.NewAppsTransport( httpClient.Transport, appID, []byte(app.PrivateKey)) if err != nil { - return nil, nil, err + return clients{}, nil } installationTransport.BaseURL = apiEndpoint installationHttpClient := common.RetryableHTTPClientTimeout(60) installationHttpClient.Transport = installationTransport installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) if err != nil { - return nil, nil, err + return clients{}, nil } apiTransport, err := ghinstallation.New( @@ -59,36 +52,64 @@ func newAppClient(app *credentialspb.GitHubApp, apiEndpoint string) (*github.Cli installationID, []byte(app.PrivateKey)) if err != nil { - return nil, nil, err + return clients{}, err } apiTransport.BaseURL = apiEndpoint httpClient.Transport = apiTransport apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) if err != nil { - return nil, nil, err + return clients{}, err } - return apiClient, installationClient, nil + return clients{ + httpClient: httpClient, + apiClient: apiClient, + installationClient: installationClient, + }, nil } -func newBasicAuthClient(basicAuth *credentialspb.BasicAuth, apiEndpoint string) (*github.Client, error) { +func newBasicAuthClients(apiEndpoint string, basicAuth *github.BasicAuthTransport) (clients, error) { httpClient := common.RetryableHTTPClientTimeout(60) httpClient.Transport = &github.BasicAuthTransport{ Username: basicAuth.Username, Password: basicAuth.Password, } - return createGitHubClient(httpClient, apiEndpoint) + apiClient, err := createGitHubClient(httpClient, apiEndpoint) + if err != nil { + return clients{}, err + } + return clients{ + httpClient: httpClient, + apiClient: apiClient, + }, nil } -func newTokenClient(token, apiEndpoint string) (*github.Client, error) { +func newTokenClients(apiEndpoint, token string) (clients, error) { httpClient := common.RetryableHTTPClientTimeout(60) + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) httpClient.Transport = &oauth2.Transport{ Base: httpClient.Transport, - Source: oauth2.ReuseTokenSource(nil, oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})), + Source: oauth2.ReuseTokenSource(nil, ts), + } + apiClient, err := createGitHubClient(httpClient, apiEndpoint) + if err != nil { + return clients{}, err } - return createGitHubClient(httpClient, apiEndpoint) + return clients{ + httpClient: httpClient, + apiClient: apiClient, + githubToken: token, + }, nil } -func newUnauthenticatedClient(apiEndpoint string) (*github.Client, error) { - return createGitHubClient(common.RetryableHTTPClientTimeout(60), apiEndpoint) +func newUnauthenticatedClients(apiEndpoint string) (clients, error) { + httpClient := common.RetryableHTTPClientTimeout(60) + apiClient, err := createGitHubClient(httpClient, apiEndpoint) + if err != nil { + return clients{}, err + } + return clients{ + httpClient: httpClient, + apiClient: apiClient, + }, nil } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 0287f34eaa3a..0cd1b4209000 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -466,7 +466,11 @@ RepoLoop: } func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, basicAuth *credentialspb.BasicAuth) error { - ghClient, err := newBasicAuthClient(basicAuth, apiEndpoint) + s.httpClient.Transport = &github.BasicAuthTransport{ + Username: basicAuth.Username, + Password: basicAuth.Password, + } + ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) if err != nil { s.log.Error(err, "error creating GitHub client") } @@ -491,7 +495,7 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas } func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint string) { - ghClient, err := newUnauthenticatedClient(apiEndpoint) + ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) if err != nil { s.log.Error(err, "error creating GitHub client") } @@ -518,7 +522,18 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri // Needed for clones. s.githubToken = token - ghClient, err := newTokenClient(token, apiEndpoint) + // Needed to list repos. + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + s.httpClient.Transport = &oauth2.Transport{ + Base: s.httpClient.Transport, + Source: oauth2.ReuseTokenSource(nil, ts), + } + + // If we're using public GitHub, make a regular client. + // Otherwise, make an enterprise client. + ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) if err != nil { s.log.Error(err, "error creating GitHub client") } @@ -660,6 +675,16 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app * return installationClient, nil } +func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) { + // If we're using public GitHub, make a regular client. + // Otherwise, make an enterprise client. + if strings.EqualFold(apiEndpoint, cloudEndpoint) { + return github.NewClient(httpClient), nil + } + + return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) +} + func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error { var scannedCount uint64 = 1 From 43bcedd41f2f3b8c5de4309c58ec12d827cf4192 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 16:52:25 -0400 Subject: [PATCH 07/48] add unauthed new way --- pkg/sources/github/client.go | 115 ---------- pkg/sources/github/connector.go | 74 +++++++ pkg/sources/github/github.go | 379 ++++++++++++++------------------ pkg/sources/github/repo.go | 198 ++++++++--------- 4 files changed, 336 insertions(+), 430 deletions(-) delete mode 100644 pkg/sources/github/client.go create mode 100644 pkg/sources/github/connector.go diff --git a/pkg/sources/github/client.go b/pkg/sources/github/client.go deleted file mode 100644 index c6a725272461..000000000000 --- a/pkg/sources/github/client.go +++ /dev/null @@ -1,115 +0,0 @@ -package github - -import ( - "net/http" - "strconv" - - "github.com/bradleyfalzon/ghinstallation/v2" - "github.com/google/go-github/v62/github" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" - "golang.org/x/oauth2" -) - -type clients struct { - httpClient *http.Client - apiClient *github.Client - installationClient *github.Client - githubToken string -} - -func newAppClients(apiEndpoint string, app *credentialspb.GitHubApp) (clients, error) { - installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) - if err != nil { - return clients{}, err - } - - appID, err := strconv.ParseInt(app.AppId, 10, 64) - if err != nil { - return clients{}, err - } - - httpClient := common.RetryableHTTPClientTimeout(60) - - installationTransport, err := ghinstallation.NewAppsTransport( - httpClient.Transport, - appID, - []byte(app.PrivateKey)) - if err != nil { - return clients{}, nil - } - installationTransport.BaseURL = apiEndpoint - installationHttpClient := common.RetryableHTTPClientTimeout(60) - installationHttpClient.Transport = installationTransport - installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - if err != nil { - return clients{}, nil - } - - apiTransport, err := ghinstallation.New( - httpClient.Transport, - appID, - installationID, - []byte(app.PrivateKey)) - if err != nil { - return clients{}, err - } - apiTransport.BaseURL = apiEndpoint - httpClient.Transport = apiTransport - apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - if err != nil { - return clients{}, err - } - - return clients{ - httpClient: httpClient, - apiClient: apiClient, - installationClient: installationClient, - }, nil -} - -func newBasicAuthClients(apiEndpoint string, basicAuth *github.BasicAuthTransport) (clients, error) { - httpClient := common.RetryableHTTPClientTimeout(60) - httpClient.Transport = &github.BasicAuthTransport{ - Username: basicAuth.Username, - Password: basicAuth.Password, - } - apiClient, err := createGitHubClient(httpClient, apiEndpoint) - if err != nil { - return clients{}, err - } - return clients{ - httpClient: httpClient, - apiClient: apiClient, - }, nil -} - -func newTokenClients(apiEndpoint, token string) (clients, error) { - httpClient := common.RetryableHTTPClientTimeout(60) - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) - httpClient.Transport = &oauth2.Transport{ - Base: httpClient.Transport, - Source: oauth2.ReuseTokenSource(nil, ts), - } - apiClient, err := createGitHubClient(httpClient, apiEndpoint) - if err != nil { - return clients{}, err - } - return clients{ - httpClient: httpClient, - apiClient: apiClient, - githubToken: token, - }, nil -} - -func newUnauthenticatedClients(apiEndpoint string) (clients, error) { - httpClient := common.RetryableHTTPClientTimeout(60) - apiClient, err := createGitHubClient(httpClient, apiEndpoint) - if err != nil { - return clients{}, err - } - return clients{ - httpClient: httpClient, - apiClient: apiClient, - }, nil -} diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go new file mode 100644 index 000000000000..008eccdf86d5 --- /dev/null +++ b/pkg/sources/github/connector.go @@ -0,0 +1,74 @@ +package github + +import ( + "fmt" + "net/http" + + gogit "github.com/go-git/go-git/v5" + "github.com/google/go-github/v62/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" +) + +type connector interface { + ApiClient() *github.Client + Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) + IsGithubEnterprise() bool + HttpClient() *http.Client + ListAppInstallations() ([]*github.Installation, error) +} + +func newConnector(connection *sourcespb.GitHub) (connector, error) { + apiEndpoint := connection.Endpoint + if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) { + apiEndpoint = cloudEndpoint + } + + switch _ := connection.GetCredential().(type) { + case *sourcespb.GitHub_Unauthenticated: + return newUnauthenticatedConnector(apiEndpoint) + default: + return nil, fmt.Errorf("unknown connection type") + } +} + +type unauthenticatedConnector struct { + httpClient *http.Client + apiClient *github.Client +} + +var _ connector = (*unauthenticatedConnector)(nil) + +func newUnauthenticatedConnector(apiEndpoint string) (connector, error) { + httpClient := common.RetryableHTTPClientTimeout(60) + apiClient, err := createGitHubClient(httpClient, apiEndpoint) + if err != nil { + return nil, err + } + return unauthenticatedConnector{ + httpClient: httpClient, + apiClient: apiClient, + }, nil +} + +func (c unauthenticatedConnector) ApiClient() *github.Client { + return c.apiClient +} + +func (c unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { + return git.CloneRepoUsingUnauthenticated(ctx, repoURL) +} + +func (c unauthenticatedConnector) IsGithubEnterprise() bool { + return false +} + +func (c unauthenticatedConnector) HttpClient() *http.Client { + return c.httpClient +} + +func (c unauthenticatedConnector) ListAppInstallations() ([]*github.Installation, error) { + return nil, nil +} diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 0cd1b4209000..965234a23093 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -8,18 +8,15 @@ import ( "os" "regexp" "sort" - "strconv" "strings" "sync" "sync/atomic" "time" - "github.com/bradleyfalzon/ghinstallation/v2" "github.com/go-logr/logr" "github.com/gobwas/glob" - "github.com/google/go-github/v63/github" + "github.com/google/go-github/v62/github" "golang.org/x/exp/rand" - "golang.org/x/oauth2" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" @@ -29,7 +26,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sanitizer" @@ -49,9 +45,9 @@ type Source struct { name string // Protects the user and token. - userMu sync.Mutex - githubUser string - githubToken string + //userMu sync.Mutex + //githubUser string + //githubToken string sourceID sources.SourceID jobID sources.JobID @@ -69,13 +65,14 @@ type Source struct { scanOptMu sync.Mutex // protects the scanOptions scanOptions *git.ScanOptions - httpClient *http.Client + //httpClient *http.Client log logr.Logger conn *sourcespb.GitHub jobPool *errgroup.Group resumeInfoMutex sync.Mutex resumeInfoSlice []string - apiClient *github.Client + //apiClient *github.Client + connector connector includePRComments bool includeIssueComments bool @@ -198,8 +195,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so s.jobPool = &errgroup.Group{} s.jobPool.SetLimit(concurrency) - s.httpClient = common.RetryableHTTPClientTimeout(60) - s.apiClient = github.NewClient(s.httpClient) + //s.httpClient = common.RetryableHTTPClientTimeout(60) + //s.apiClient = github.NewClient(s.httpClient) var conn sourcespb.GitHub err = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}) @@ -208,6 +205,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so } s.conn = &conn + s.connector = newConnector(s.conn) + s.orgsCache = memory.New[string]() for _, org := range s.conn.Organizations { s.orgsCache.Set(org, org) @@ -270,63 +269,13 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so return nil } -// Validate is used by enterprise CLI to validate the Github config file. +// Validate is used by enterprise CLI to validate the GitHub config file. func (s *Source) Validate(ctx context.Context) []error { - var ( - errs []error - ghClient *github.Client - err error - ) - apiEndpoint := s.conn.Endpoint - - switch cred := s.conn.GetCredential().(type) { - case *sourcespb.GitHub_BasicAuth: - s.httpClient.Transport = &github.BasicAuthTransport{ - Username: cred.BasicAuth.Username, - Password: cred.BasicAuth.Password, - } - ghClient, err = createGitHubClient(s.httpClient, apiEndpoint) - if err != nil { - errs = append(errs, fmt.Errorf("error creating GitHub client: %+v", err)) - } - case *sourcespb.GitHub_Unauthenticated: - ghClient, err = createGitHubClient(s.httpClient, apiEndpoint) - if err != nil { - errs = append(errs, fmt.Errorf("error creating GitHub client: %+v", err)) - } - case *sourcespb.GitHub_Token: - s.githubToken = cred.Token - - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: s.githubToken}, - ) - s.httpClient.Transport = &oauth2.Transport{ - Base: s.httpClient.Transport, - Source: oauth2.ReuseTokenSource(nil, ts), - } - - ghClient, err = createGitHubClient(s.httpClient, apiEndpoint) - if err != nil { - errs = append(errs, fmt.Errorf("error creating GitHub client: %+v", err)) - } - default: - errs = append(errs, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type())) - } - - // Run a simple query to check if the client is actually valid - if ghClient != nil { - err = checkGitHubConnection(ctx, ghClient) - if err != nil { - errs = append(errs, err) - } + if _, _, err := s.connector.ApiClient().Users.Get(ctx, ""); err != nil { + return []error{err} } - return errs -} - -func checkGitHubConnection(ctx context.Context, client *github.Client) error { - _, _, err := client.Users.Get(ctx, "") - return err + return nil } func (s *Source) visibilityOf(ctx context.Context, repoURL string) source_metadatapb.Visibility { @@ -351,10 +300,10 @@ const cloudEndpoint = "https://api.github.com" // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error { - apiEndpoint := s.conn.Endpoint - if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) { - apiEndpoint = cloudEndpoint - } + //apiEndpoint := s.conn.Endpoint + //if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) { + // apiEndpoint = cloudEndpoint + //} // If targets are provided, we're only scanning the data in those targets. // Otherwise, we're scanning all data. @@ -369,39 +318,39 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar githubSecondsSpentRateLimited.WithLabelValues(s.name).Set(0) githubReposScanned.WithLabelValues(s.name).Set(0) - installationClient, err := s.enumerate(ctx, apiEndpoint) + err := s.enumerate(ctx) if err != nil { return err } - return s.scan(ctx, installationClient, chunksChan) + return s.scan(ctx, chunksChan) } -func (s *Source) enumerate(ctx context.Context, apiEndpoint string) (*github.Client, error) { - var ( - installationClient *github.Client - err error - ) - - switch cred := s.conn.GetCredential().(type) { - case *sourcespb.GitHub_BasicAuth: - if err = s.enumerateBasicAuth(ctx, apiEndpoint, cred.BasicAuth); err != nil { - return nil, err - } - case *sourcespb.GitHub_Unauthenticated: - s.enumerateUnauthenticated(ctx, apiEndpoint) - case *sourcespb.GitHub_Token: - if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil { - return nil, err - } - case *sourcespb.GitHub_GithubApp: - if installationClient, err = s.enumerateWithApp(ctx, apiEndpoint, cred.GithubApp); err != nil { - return nil, err - } - default: - // TODO: move this error to Init - return nil, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()) - } +func (s *Source) enumerate(ctx context.Context) error { + //var ( + // installationClient *github.Client + // err error + //) + // + //switch cred := s.conn.GetCredential().(type) { + //case *sourcespb.GitHub_BasicAuth: + // if err = s.enumerateBasicAuth(ctx, apiEndpoint, cred.BasicAuth); err != nil { + // return nil, err + // } + //case *sourcespb.GitHub_Unauthenticated: + // s.enumerateUnauthenticated(ctx, apiEndpoint) + //case *sourcespb.GitHub_Token: + // if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil { + // return nil, err + // } + //case *sourcespb.GitHub_GithubApp: + // if installationClient, err = s.enumerateWithApp(ctx, apiEndpoint, cred.GithubApp); err != nil { + // return nil, err + // } + //default: + // // TODO: move this error to Init + // return nil, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()) + //} s.repos = make([]string, 0, s.filteredRepoCache.Count()) @@ -425,7 +374,7 @@ RepoLoop: // Cache gist info. for { gistID := extractGistID(urlParts) - gist, _, err := s.apiClient.Gists.Get(repoCtx, gistID) + gist, _, err := s.connector.ApiClient().Gists.Get(repoCtx, gistID) // Normalize the URL to the Gist's pull URL. // See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937 repo = gist.GetGitPullURL() @@ -442,7 +391,7 @@ RepoLoop: } else { // Cache repository info. for { - ghRepo, _, err := s.apiClient.Repositories.Get(repoCtx, urlParts[1], urlParts[2]) + ghRepo, _, err := s.connector.ApiClient().Repositories.Get(repoCtx, urlParts[1], urlParts[2]) if s.handleRateLimit(err) { continue } @@ -462,19 +411,19 @@ RepoLoop: // We must sort the repos so we can resume later if necessary. sort.Strings(s.repos) - return installationClient, nil + return nil } -func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, basicAuth *credentialspb.BasicAuth) error { - s.httpClient.Transport = &github.BasicAuthTransport{ - Username: basicAuth.Username, - Password: basicAuth.Password, - } - ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) - if err != nil { - s.log.Error(err, "error creating GitHub client") - } - s.apiClient = ghClient +func (s *Source) enumerateBasicAuth(ctx context.Context) error { + //s.httpClient.Transport = &github.BasicAuthTransport{ + // Username: basicAuth.Username, + // Password: basicAuth.Password, + //} + //ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) + //if err != nil { + // s.log.Error(err, "error creating GitHub client") + //} + //s.apiClient = ghClient for _, org := range s.orgsCache.Keys() { orgCtx := context.WithValue(ctx, "account", org) @@ -494,12 +443,12 @@ func (s *Source) enumerateBasicAuth(ctx context.Context, apiEndpoint string, bas return nil } -func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint string) { - ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) - if err != nil { - s.log.Error(err, "error creating GitHub client") - } - s.apiClient = ghClient +func (s *Source) enumerateUnauthenticated(ctx context.Context) { + //ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) + //if err != nil { + // s.log.Error(err, "error creating GitHub client") + //} + //s.apiClient = ghClient if s.orgsCache.Count() > unauthGithubOrgRateLimt { s.log.Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.") } @@ -518,31 +467,32 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context, apiEndpoint strin } } -func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token string) error { - // Needed for clones. - s.githubToken = token - - // Needed to list repos. - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - s.httpClient.Transport = &oauth2.Transport{ - Base: s.httpClient.Transport, - Source: oauth2.ReuseTokenSource(nil, ts), - } - - // If we're using public GitHub, make a regular client. - // Otherwise, make an enterprise client. - ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) - if err != nil { - s.log.Error(err, "error creating GitHub client") - } - s.apiClient = ghClient - - ctx.Logger().V(1).Info("Enumerating with token", "endpoint", apiEndpoint) +func (s *Source) enumerateWithToken(ctx context.Context) error { + //// Needed for clones. + //s.githubToken = token + // + //// Needed to list repos. + //ts := oauth2.StaticTokenSource( + // &oauth2.Token{AccessToken: token}, + //) + //s.httpClient.Transport = &oauth2.Transport{ + // Base: s.httpClient.Transport, + // Source: oauth2.ReuseTokenSource(nil, ts), + //} + // + //// If we're using public GitHub, make a regular client. + //// Otherwise, make an enterprise client. + //ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) + //if err != nil { + // s.log.Error(err, "error creating GitHub client") + //} + //s.apiClient = ghClient + + ctx.Logger().V(1).Info("Enumerating with token") var ghUser *github.User + var err error for { - ghUser, _, err = s.apiClient.Users.Get(ctx, "") + ghUser, _, err = s.connector.ApiClient().Users.Get(ctx, "") if s.handleRateLimit(err) { continue } @@ -562,8 +512,8 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri s.log.Error(err, "Unable to fetch gists for the current user", "user", ghUser.GetLogin()) } - isGHE := !strings.EqualFold(apiEndpoint, cloudEndpoint) - if isGHE { + //isGHE := !strings.EqualFold(apiEndpoint, cloudEndpoint) + if s.connector.IsGithubEnterprise() { s.addAllVisibleOrgs(ctx) } else { // Scan for orgs is default with a token. @@ -597,67 +547,67 @@ func (s *Source) enumerateWithToken(ctx context.Context, apiEndpoint, token stri return nil } -func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app *credentialspb.GitHubApp) (installationClient *github.Client, err error) { - installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) - if err != nil { - return nil, err - } - - appID, err := strconv.ParseInt(app.AppId, 10, 64) - if err != nil { - return nil, err - } - - // This client is required to create installation tokens for cloning. - // Otherwise, the required JWT is not in the request for the token :/ - // This client uses the source's original HTTP transport, so make sure - // to build it before modifying that transport (such as is done during - // the creation of the other API client below). - appItr, err := ghinstallation.NewAppsTransport( - s.httpClient.Transport, - appID, - []byte(app.PrivateKey)) - if err != nil { - return nil, err - } - appItr.BaseURL = apiEndpoint - - // Does this need to be separate from |s.httpClient|? - instHTTPClient := common.RetryableHTTPClientTimeout(60) - instHTTPClient.Transport = appItr - installationClient, err = github.NewClient(instHTTPClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - if err != nil { - return nil, err - } - - // This client is used for most APIs. - itr, err := ghinstallation.New( - s.httpClient.Transport, - appID, - installationID, - []byte(app.PrivateKey)) - if err != nil { - return nil, err - } - itr.BaseURL = apiEndpoint - - s.httpClient.Transport = itr - s.apiClient, err = github.NewClient(s.httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - if err != nil { - return nil, err - } +func (s *Source) enumerateWithApp(ctx context.Context) error { + //installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) + //if err != nil { + // return nil, err + //} + // + //appID, err := strconv.ParseInt(app.AppId, 10, 64) + //if err != nil { + // return nil, err + //} + // + //// This client is required to create installation tokens for cloning. + //// Otherwise, the required JWT is not in the request for the token :/ + //// This client uses the source's original HTTP transport, so make sure + //// to build it before modifying that transport (such as is done during + //// the creation of the other API client below). + //appItr, err := ghinstallation.NewAppsTransport( + // s.httpClient.Transport, + // appID, + // []byte(app.PrivateKey)) + //if err != nil { + // return nil, err + //} + //appItr.BaseURL = apiEndpoint + // + //// Does this need to be separate from |s.httpClient|? + //instHTTPClient := common.RetryableHTTPClientTimeout(60) + //instHTTPClient.Transport = appItr + //installationClient, err = github.NewClient(instHTTPClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + //if err != nil { + // return nil, err + //} + // + //// This client is used for most APIs. + //itr, err := ghinstallation.New( + // s.httpClient.Transport, + // appID, + // installationID, + // []byte(app.PrivateKey)) + //if err != nil { + // return nil, err + //} + //itr.BaseURL = apiEndpoint + // + //s.httpClient.Transport = itr + //s.apiClient, err = github.NewClient(s.httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + //if err != nil { + // return nil, err + //} // If no repos were provided, enumerate them. if len(s.repos) == 0 { - if err = s.getReposByApp(ctx); err != nil { - return nil, err + if err := s.getReposByApp(ctx); err != nil { + return err } // Check if we need to find user repos. if s.conn.ScanUsers { - err := s.addMembersByApp(ctx, installationClient) + err := s.addMembersByApp(ctx) if err != nil { - return nil, err + return err } s.log.Info("Scanning repos", "org_members", len(s.memberCache)) for member := range s.memberCache { @@ -672,7 +622,7 @@ func (s *Source) enumerateWithApp(ctx context.Context, apiEndpoint string, app * } } - return installationClient, nil + return nil } func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Client, error) { @@ -685,7 +635,7 @@ func createGitHubClient(httpClient *http.Client, apiEndpoint string) (*github.Cl return github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) } -func (s *Source) scan(ctx context.Context, installationClient *github.Client, chunksChan chan *sources.Chunk) error { +func (s *Source) scan(ctx context.Context, chunksChan chan *sources.Chunk) error { var scannedCount uint64 = 1 s.log.V(2).Info("Found repos to scan", "count", len(s.repos)) @@ -730,7 +680,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch return nil } repoCtx := context.WithValues(ctx, "repo", repoURL) - duration, err := s.cloneAndScanRepo(repoCtx, installationClient, repoURL, repoInfo, chunksChan) + duration, err := s.cloneAndScanRepo(repoCtx, repoURL, repoInfo, chunksChan) if err != nil { scanErrs.Add(err) return nil @@ -741,7 +691,7 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch wikiURL := strings.TrimSuffix(repoURL, ".git") + ".wiki.git" wikiCtx := context.WithValue(ctx, "repo", wikiURL) - _, err := s.cloneAndScanRepo(wikiCtx, installationClient, wikiURL, repoInfo, chunksChan) + _, err := s.cloneAndScanRepo(wikiCtx, wikiURL, repoInfo, chunksChan) if err != nil { // Ignore "Repository not found" errors. // It's common for GitHub's API to say a repo has a wiki when it doesn't. @@ -777,11 +727,11 @@ func (s *Source) scan(ctx context.Context, installationClient *github.Client, ch return nil } -func (s *Source) cloneAndScanRepo(ctx context.Context, client *github.Client, repoURL string, repoInfo repoInfo, chunksChan chan *sources.Chunk) (time.Duration, error) { +func (s *Source) cloneAndScanRepo(ctx context.Context, repoURL string, repoInfo repoInfo, chunksChan chan *sources.Chunk) (time.Duration, error) { var duration time.Duration ctx.Logger().V(2).Info("attempting to clone repo") - path, repo, err := s.cloneRepo(ctx, repoURL, client) + path, repo, err := s.cloneRepo(ctx, repoURL) if err != nil { return duration, err } @@ -894,7 +844,7 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string) error { gistOpts := &github.GistListOptions{} logger := s.log.WithValues("user", user) for { - gists, res, err := s.apiClient.Gists.List(ctx, user, gistOpts) + gists, res, err := s.connector.ApiClient().Gists.List(ctx, user, gistOpts) if s.handleRateLimit(err) { continue } @@ -916,13 +866,14 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string) error { return nil } -func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client) error { - opts := &github.ListOptions{ - PerPage: membersAppPagination, - } +func (s *Source) addMembersByApp(ctx context.Context) error { + //opts := &github.ListOptions{ + // PerPage: membersAppPagination, + //} // TODO: Check rate limit for this call. - installs, _, err := installationClient.Apps.ListInstallations(ctx, opts) + //installs, _, err := installationClient.Apps.ListInstallations(ctx, opts) + installs, err := s.connector.ListAppInstallations() if err != nil { return fmt.Errorf("could not enumerate installed orgs: %w", err) } @@ -951,7 +902,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context) { }, } for { - orgs, _, err := s.apiClient.Organizations.ListAll(ctx, orgOpts) + orgs, _, err := s.connector.ApiClient().Organizations.ListAll(ctx, orgOpts) if s.handleRateLimit(err) { continue } @@ -990,7 +941,7 @@ func (s *Source) addOrgsByUser(ctx context.Context, user string) { } logger := s.log.WithValues("user", user) for { - orgs, resp, err := s.apiClient.Organizations.List(ctx, "", orgOpts) + orgs, resp, err := s.connector.ApiClient().Organizations.List(ctx, "", orgOpts) if s.handleRateLimit(err) { continue } @@ -1023,7 +974,7 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string) error { logger := s.log.WithValues("org", org) for { - members, res, err := s.apiClient.Organizations.ListMembers(ctx, org, opts) + members, res, err := s.connector.ApiClient().Organizations.ListMembers(ctx, org, opts) if s.handleRateLimit(err) { continue } @@ -1145,7 +1096,7 @@ func (s *Source) processGistComments(ctx context.Context, gistURL string, urlPar Page: initialPage, } for { - comments, _, err := s.apiClient.Gists.ListComments(ctx, gistID, options) + comments, _, err := s.connector.ApiClient().Gists.ListComments(ctx, gistID, options) if s.handleRateLimit(err) { continue } @@ -1258,7 +1209,7 @@ func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, chunksCha } for { - issues, _, err := s.apiClient.Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts) + issues, _, err := s.connector.ApiClient().Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts) if s.handleRateLimit(err) { continue } @@ -1330,7 +1281,7 @@ func (s *Source) processIssueComments(ctx context.Context, repoInfo repoInfo, ch } for { - issueComments, _, err := s.apiClient.Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts) + issueComments, _, err := s.connector.ApiClient().Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts) if s.handleRateLimit(err) { continue } @@ -1395,7 +1346,7 @@ func (s *Source) processPRs(ctx context.Context, repoInfo repoInfo, chunksChan c } for { - prs, _, err := s.apiClient.PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts) + prs, _, err := s.connector.ApiClient().PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts) if s.handleRateLimit(err) { continue } @@ -1427,7 +1378,7 @@ func (s *Source) processPRComments(ctx context.Context, repoInfo repoInfo, chunk } for { - prComments, _, err := s.apiClient.PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts) + prComments, _, err := s.connector.ApiClient().PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts) if s.handleRateLimit(err) { continue } diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index 3b019638b518..f2379ece5729 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "net/http" - "strconv" "strings" "sync" @@ -15,8 +14,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/giturl" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" - "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) type repoInfoCache struct { @@ -56,105 +53,104 @@ type repoInfo struct { func (s *Source) cloneRepo( ctx context.Context, repoURL string, - installationClient *github.Client, ) (string, *gogit.Repository, error) { - var ( - path string - repo *gogit.Repository - err error - ) - - switch s.conn.GetCredential().(type) { - case *sourcespb.GitHub_BasicAuth: - path, repo, err = git.CloneRepoUsingToken(ctx, s.conn.GetBasicAuth().GetPassword(), repoURL, s.conn.GetBasicAuth().GetUsername()) - if err != nil { - return "", nil, err - } - case *sourcespb.GitHub_Unauthenticated: - path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL) - if err != nil { - return "", nil, err - } - - case *sourcespb.GitHub_GithubApp: - s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient) - if err != nil { - return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) - } - - path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) - if err != nil { - return "", nil, err - } - - case *sourcespb.GitHub_Token: - if err := s.getUserAndToken(ctx, repoURL, installationClient); err != nil { - return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) - } - path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) - if err != nil { - return "", nil, err - } - default: - return "", nil, fmt.Errorf("unhandled credential type for repo %s: %T", repoURL, s.conn.GetCredential()) + path, repo, err := s.connector.Clone(ctx, repoURL) + if err != nil { + return "", nil, err } return path, repo, nil -} -func (s *Source) getUserAndToken(ctx context.Context, repoURL string, installationClient *github.Client) error { - // We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again. - s.userMu.Lock() - defer s.userMu.Unlock() - if s.githubUser == "" || s.githubToken == "" { - var err error - s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient) - if err != nil { - return fmt.Errorf("error getting token for repo %s: %w", repoURL, err) - } - } - return nil + //switch s.conn.GetCredential().(type) { + //case *sourcespb.GitHub_BasicAuth: + // path, repo, err = git.CloneRepoUsingToken(ctx, s.conn.GetBasicAuth().GetPassword(), repoURL, s.conn.GetBasicAuth().GetUsername()) + // if err != nil { + // return "", nil, err + // } + //case *sourcespb.GitHub_Unauthenticated: + //path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL) + // if err != nil { + // return "", nil, err + // } + // + //case *sourcespb.GitHub_GithubApp: + // s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient) + // if err != nil { + // return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) + // } + // + // path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) + // if err != nil { + // return "", nil, err + // } + // + //case *sourcespb.GitHub_Token: + // if err := s.getUserAndToken(ctx, repoURL, installationClient); err != nil { + // return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) + // } + // path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) + // if err != nil { + // return "", nil, err + // } + //default: + // return "", nil, fmt.Errorf("unhandled credential type for repo %s: %T", repoURL, s.conn.GetCredential()) + //} + //return path, repo, nil } -func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) { - switch cred := s.conn.GetCredential().(type) { - case *sourcespb.GitHub_BasicAuth: - return cred.BasicAuth.Username, cred.BasicAuth.Password, nil - case *sourcespb.GitHub_Unauthenticated: - // do nothing - case *sourcespb.GitHub_GithubApp: - id, err := strconv.ParseInt(cred.GithubApp.InstallationId, 10, 64) - if err != nil { - return "", "", fmt.Errorf("unable to parse installation id: %w", err) - } - // TODO: Check rate limit for this call. - token, _, err := installationClient.Apps.CreateInstallationToken( - ctx, id, &github.InstallationTokenOptions{}) - if err != nil { - return "", "", fmt.Errorf("unable to create installation token: %w", err) - } - return "x-access-token", token.GetToken(), nil // TODO: multiple workers request this, track the TTL - case *sourcespb.GitHub_Token: - var ( - ghUser *github.User - err error - ) - for { - ghUser, _, err = s.apiClient.Users.Get(ctx, "") - if s.handleRateLimit(err) { - continue - } - if err != nil { - return "", "", fmt.Errorf("unable to get user: %w", err) - } - break - } - return ghUser.GetLogin(), cred.Token, nil - default: - return "", "", fmt.Errorf("unhandled credential type") - } - - return "", "", fmt.Errorf("unhandled credential type") -} +//func (s *Source) getUserAndToken(ctx context.Context, repoURL string, installationClient *github.Client) error { +// // We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again. +// s.userMu.Lock() +// defer s.userMu.Unlock() +// if s.githubUser == "" || s.githubToken == "" { +// var err error +// s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient) +// if err != nil { +// return fmt.Errorf("error getting token for repo %s: %w", repoURL, err) +// } +// } +// return nil +//} + +//func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) { +// switch cred := s.conn.GetCredential().(type) { +// case *sourcespb.GitHub_BasicAuth: +// return cred.BasicAuth.Username, cred.BasicAuth.Password, nil +// case *sourcespb.GitHub_Unauthenticated: +// // do nothing +// case *sourcespb.GitHub_GithubApp: +// id, err := strconv.ParseInt(cred.GithubApp.InstallationId, 10, 64) +// if err != nil { +// return "", "", fmt.Errorf("unable to parse installation id: %w", err) +// } +// // TODO: Check rate limit for this call. +// token, _, err := installationClient.Apps.CreateInstallationToken( +// ctx, id, &github.InstallationTokenOptions{}) +// if err != nil { +// return "", "", fmt.Errorf("unable to create installation token: %w", err) +// } +// return "x-access-token", token.GetToken(), nil // TODO: multiple workers request this, track the TTL +// case *sourcespb.GitHub_Token: +// var ( +// ghUser *github.User +// err error +// ) +// for { +// ghUser, _, err = s.apiClient.Users.Get(ctx, "") +// if s.handleRateLimit(err) { +// continue +// } +// if err != nil { +// return "", "", fmt.Errorf("unable to get user: %w", err) +// } +// break +// } +// return ghUser.GetLogin(), cred.Token, nil +// default: +// return "", "", fmt.Errorf("unhandled credential type") +// } +// +// return "", "", fmt.Errorf("unhandled credential type") +//} type repoListOptions interface { getListOptions() *github.ListOptions @@ -171,7 +167,7 @@ func (a *appListOptions) getListOptions() *github.ListOptions { } func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { - someRepos, res, err := s.apiClient.Apps.ListRepos(ctx, opts.getListOptions()) + someRepos, res, err := s.connector.ApiClient().Apps.ListRepos(ctx, opts.getListOptions()) if someRepos != nil { return someRepos.Repositories, res, err } @@ -195,7 +191,7 @@ func (u *userListOptions) getListOptions() *github.ListOptions { } func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { - return s.apiClient.Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions) + return s.connector.ApiClient().Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions) } func (s *Source) getReposByUser(ctx context.Context, user string) error { @@ -217,7 +213,7 @@ func (o *orgListOptions) getListOptions() *github.ListOptions { } func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { - return s.apiClient.Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions) + return s.connector.ApiClient().Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions) } func (s *Source) getReposByOrg(ctx context.Context, org string) error { @@ -371,7 +367,7 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool { return false } - res, err := s.httpClient.Do(req) + res, err := s.connector.HttpClient().Do(req) if err != nil { return false } @@ -398,7 +394,7 @@ func (s *Source) getDiffForFileInCommit(ctx context.Context, query commitQuery) err error ) for { - commit, _, err = s.apiClient.Repositories.GetCommit(ctx, query.owner, query.repo, query.sha, nil) + commit, _, err = s.connector.ApiClient().Repositories.GetCommit(ctx, query.owner, query.repo, query.sha, nil) if s.handleRateLimit(err) { continue } From 1df7a1490add39102cda63e6a7ee2f06ae9ad02b Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 17:15:33 -0400 Subject: [PATCH 08/48] add app connector --- pkg/sources/github/connector.go | 113 ++++++++++++++++++++++++++++++-- pkg/sources/github/github.go | 8 ++- 2 files changed, 114 insertions(+), 7 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 008eccdf86d5..97d7dfbb6b06 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -3,11 +3,15 @@ package github import ( "fmt" "net/http" + "strconv" + "strings" + "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v62/github" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) @@ -17,7 +21,7 @@ type connector interface { Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) IsGithubEnterprise() bool HttpClient() *http.Client - ListAppInstallations() ([]*github.Installation, error) + ListAppInstallations(ctx context.Context) ([]*github.Installation, error) } func newConnector(connection *sourcespb.GitHub) (connector, error) { @@ -26,7 +30,9 @@ func newConnector(connection *sourcespb.GitHub) (connector, error) { apiEndpoint = cloudEndpoint } - switch _ := connection.GetCredential().(type) { + switch cred := connection.GetCredential().(type) { + case *sourcespb.GitHub_GithubApp: + return newAppConnector(apiEndpoint, cred.GithubApp) case *sourcespb.GitHub_Unauthenticated: return newUnauthenticatedConnector(apiEndpoint) default: @@ -34,6 +40,103 @@ func newConnector(connection *sourcespb.GitHub) (connector, error) { } } +type appConnector struct { + httpClient *http.Client + apiClient *github.Client + installationClient *github.Client + apiEndpoint string + installationID int64 +} + +var _ connector = (*appConnector)(nil) + +func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { + installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) + if err != nil { + return nil, err + } + + appID, err := strconv.ParseInt(app.AppId, 10, 64) + if err != nil { + return nil, err + } + + httpClient := common.RetryableHTTPClientTimeout(60) + + installationTransport, err := ghinstallation.NewAppsTransport( + httpClient.Transport, + appID, + []byte(app.PrivateKey)) + if err != nil { + return nil, err + } + installationTransport.BaseURL = apiEndpoint + + installationHttpClient := common.RetryableHTTPClientTimeout(60) + installationHttpClient.Transport = installationTransport + installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + if err != nil { + return nil, err + } + + apiTransport, err := ghinstallation.New( + httpClient.Transport, + appID, + installationID, + []byte(app.PrivateKey)) + if err != nil { + return nil, err + } + apiTransport.BaseURL = apiEndpoint + + httpClient.Transport = apiTransport + apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + if err != nil { + return nil, err + } + + return &appConnector{ + httpClient: httpClient, + apiClient: apiClient, + installationClient: installationClient, + apiEndpoint: apiEndpoint, + installationID: installationID, + }, nil +} + +func (c appConnector) ApiClient() *github.Client { + return c.apiClient +} + +func (c appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { + // TODO: Check rate limit for this call. + token, _, err := c.installationClient.Apps.CreateInstallationToken( + ctx, + c.installationID, + &github.InstallationTokenOptions{}) + if err != nil { + return "", nil, err + } + + return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token") +} + +func (c appConnector) IsGithubEnterprise() bool { + return !strings.EqualFold(c.apiEndpoint, cloudEndpoint) +} + +func (c appConnector) HttpClient() *http.Client { + return c.httpClient +} + +func (c appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { + opts := &github.ListOptions{ + PerPage: membersAppPagination, + } + installs, _, err := c.installationClient.Apps.ListInstallations(ctx, opts) + return installs, err +} + type unauthenticatedConnector struct { httpClient *http.Client apiClient *github.Client @@ -41,13 +144,13 @@ type unauthenticatedConnector struct { var _ connector = (*unauthenticatedConnector)(nil) -func newUnauthenticatedConnector(apiEndpoint string) (connector, error) { +func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { httpClient := common.RetryableHTTPClientTimeout(60) apiClient, err := createGitHubClient(httpClient, apiEndpoint) if err != nil { return nil, err } - return unauthenticatedConnector{ + return &unauthenticatedConnector{ httpClient: httpClient, apiClient: apiClient, }, nil @@ -69,6 +172,6 @@ func (c unauthenticatedConnector) HttpClient() *http.Client { return c.httpClient } -func (c unauthenticatedConnector) ListAppInstallations() ([]*github.Installation, error) { +func (c unauthenticatedConnector) ListAppInstallations(context.Context) ([]*github.Installation, error) { return nil, nil } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 965234a23093..27c5b37c8dee 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -205,7 +205,11 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so } s.conn = &conn - s.connector = newConnector(s.conn) + connector, err := newConnector(s.conn) + if err != nil { + return fmt.Errorf("could not create connector: %w", err) + } + s.connector = connector s.orgsCache = memory.New[string]() for _, org := range s.conn.Organizations { @@ -873,7 +877,7 @@ func (s *Source) addMembersByApp(ctx context.Context) error { // TODO: Check rate limit for this call. //installs, _, err := installationClient.Apps.ListInstallations(ctx, opts) - installs, err := s.connector.ListAppInstallations() + installs, err := s.connector.ListAppInstallations(ctx) if err != nil { return fmt.Errorf("could not enumerate installed orgs: %w", err) } From c01bcdae6bcca56ab28493907c5915f6360c7071 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 17:43:54 -0400 Subject: [PATCH 09/48] stop checking for ghe in app connector --- pkg/sources/github/connector.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 97d7dfbb6b06..69160698dd0a 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -4,7 +4,6 @@ import ( "fmt" "net/http" "strconv" - "strings" "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" @@ -44,7 +43,6 @@ type appConnector struct { httpClient *http.Client apiClient *github.Client installationClient *github.Client - apiEndpoint string installationID int64 } @@ -99,7 +97,6 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn httpClient: httpClient, apiClient: apiClient, installationClient: installationClient, - apiEndpoint: apiEndpoint, installationID: installationID, }, nil } @@ -122,7 +119,7 @@ func (c appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit } func (c appConnector) IsGithubEnterprise() bool { - return !strings.EqualFold(c.apiEndpoint, cloudEndpoint) + return false } func (c appConnector) HttpClient() *http.Client { From 2240bf74a2911913e840c5d595b07d3afcd2de92 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 17:46:19 -0400 Subject: [PATCH 10/48] split into separate files --- pkg/sources/github/connector.go | 139 ------------------ pkg/sources/github/connector_app.go | 109 ++++++++++++++ .../github/connector_unauthenticated.go | 50 +++++++ 3 files changed, 159 insertions(+), 139 deletions(-) create mode 100644 pkg/sources/github/connector_app.go create mode 100644 pkg/sources/github/connector_unauthenticated.go diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 69160698dd0a..567c23e8ed05 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -3,16 +3,11 @@ package github import ( "fmt" "net/http" - "strconv" - "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v62/github" - "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" - "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" - "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) type connector interface { @@ -38,137 +33,3 @@ func newConnector(connection *sourcespb.GitHub) (connector, error) { return nil, fmt.Errorf("unknown connection type") } } - -type appConnector struct { - httpClient *http.Client - apiClient *github.Client - installationClient *github.Client - installationID int64 -} - -var _ connector = (*appConnector)(nil) - -func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { - installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) - if err != nil { - return nil, err - } - - appID, err := strconv.ParseInt(app.AppId, 10, 64) - if err != nil { - return nil, err - } - - httpClient := common.RetryableHTTPClientTimeout(60) - - installationTransport, err := ghinstallation.NewAppsTransport( - httpClient.Transport, - appID, - []byte(app.PrivateKey)) - if err != nil { - return nil, err - } - installationTransport.BaseURL = apiEndpoint - - installationHttpClient := common.RetryableHTTPClientTimeout(60) - installationHttpClient.Transport = installationTransport - installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - if err != nil { - return nil, err - } - - apiTransport, err := ghinstallation.New( - httpClient.Transport, - appID, - installationID, - []byte(app.PrivateKey)) - if err != nil { - return nil, err - } - apiTransport.BaseURL = apiEndpoint - - httpClient.Transport = apiTransport - apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - if err != nil { - return nil, err - } - - return &appConnector{ - httpClient: httpClient, - apiClient: apiClient, - installationClient: installationClient, - installationID: installationID, - }, nil -} - -func (c appConnector) ApiClient() *github.Client { - return c.apiClient -} - -func (c appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { - // TODO: Check rate limit for this call. - token, _, err := c.installationClient.Apps.CreateInstallationToken( - ctx, - c.installationID, - &github.InstallationTokenOptions{}) - if err != nil { - return "", nil, err - } - - return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token") -} - -func (c appConnector) IsGithubEnterprise() bool { - return false -} - -func (c appConnector) HttpClient() *http.Client { - return c.httpClient -} - -func (c appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { - opts := &github.ListOptions{ - PerPage: membersAppPagination, - } - installs, _, err := c.installationClient.Apps.ListInstallations(ctx, opts) - return installs, err -} - -type unauthenticatedConnector struct { - httpClient *http.Client - apiClient *github.Client -} - -var _ connector = (*unauthenticatedConnector)(nil) - -func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { - httpClient := common.RetryableHTTPClientTimeout(60) - apiClient, err := createGitHubClient(httpClient, apiEndpoint) - if err != nil { - return nil, err - } - return &unauthenticatedConnector{ - httpClient: httpClient, - apiClient: apiClient, - }, nil -} - -func (c unauthenticatedConnector) ApiClient() *github.Client { - return c.apiClient -} - -func (c unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { - return git.CloneRepoUsingUnauthenticated(ctx, repoURL) -} - -func (c unauthenticatedConnector) IsGithubEnterprise() bool { - return false -} - -func (c unauthenticatedConnector) HttpClient() *http.Client { - return c.httpClient -} - -func (c unauthenticatedConnector) ListAppInstallations(context.Context) ([]*github.Installation, error) { - return nil, nil -} diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go new file mode 100644 index 000000000000..2ae3cc070a2d --- /dev/null +++ b/pkg/sources/github/connector_app.go @@ -0,0 +1,109 @@ +package github + +import ( + "net/http" + "strconv" + + "github.com/bradleyfalzon/ghinstallation/v2" + gogit "github.com/go-git/go-git/v5" + "github.com/google/go-github/v62/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" +) + +type appConnector struct { + httpClient *http.Client + apiClient *github.Client + installationClient *github.Client + installationID int64 +} + +var _ connector = (*appConnector)(nil) + +func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { + installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) + if err != nil { + return nil, err + } + + appID, err := strconv.ParseInt(app.AppId, 10, 64) + if err != nil { + return nil, err + } + + httpClient := common.RetryableHTTPClientTimeout(60) + + installationTransport, err := ghinstallation.NewAppsTransport( + httpClient.Transport, + appID, + []byte(app.PrivateKey)) + if err != nil { + return nil, err + } + installationTransport.BaseURL = apiEndpoint + + installationHttpClient := common.RetryableHTTPClientTimeout(60) + installationHttpClient.Transport = installationTransport + installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + if err != nil { + return nil, err + } + + apiTransport, err := ghinstallation.New( + httpClient.Transport, + appID, + installationID, + []byte(app.PrivateKey)) + if err != nil { + return nil, err + } + apiTransport.BaseURL = apiEndpoint + + httpClient.Transport = apiTransport + apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) + if err != nil { + return nil, err + } + + return &appConnector{ + httpClient: httpClient, + apiClient: apiClient, + installationClient: installationClient, + installationID: installationID, + }, nil +} + +func (c appConnector) ApiClient() *github.Client { + return c.apiClient +} + +func (c appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { + // TODO: Check rate limit for this call. + token, _, err := c.installationClient.Apps.CreateInstallationToken( + ctx, + c.installationID, + &github.InstallationTokenOptions{}) + if err != nil { + return "", nil, err + } + + return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token") +} + +func (c appConnector) IsGithubEnterprise() bool { + return false +} + +func (c appConnector) HttpClient() *http.Client { + return c.httpClient +} + +func (c appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { + opts := &github.ListOptions{ + PerPage: membersAppPagination, + } + installs, _, err := c.installationClient.Apps.ListInstallations(ctx, opts) + return installs, err +} diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go new file mode 100644 index 000000000000..26c33dc6a06d --- /dev/null +++ b/pkg/sources/github/connector_unauthenticated.go @@ -0,0 +1,50 @@ +package github + +import ( + "net/http" + + gogit "github.com/go-git/go-git/v5" + "github.com/google/go-github/v62/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" +) + +type unauthenticatedConnector struct { + httpClient *http.Client + apiClient *github.Client +} + +var _ connector = (*unauthenticatedConnector)(nil) + +func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { + httpClient := common.RetryableHTTPClientTimeout(60) + apiClient, err := createGitHubClient(httpClient, apiEndpoint) + if err != nil { + return nil, err + } + return &unauthenticatedConnector{ + httpClient: httpClient, + apiClient: apiClient, + }, nil +} + +func (c unauthenticatedConnector) ApiClient() *github.Client { + return c.apiClient +} + +func (c unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { + return git.CloneRepoUsingUnauthenticated(ctx, repoURL) +} + +func (c unauthenticatedConnector) IsGithubEnterprise() bool { + return false +} + +func (c unauthenticatedConnector) HttpClient() *http.Client { + return c.httpClient +} + +func (c unauthenticatedConnector) ListAppInstallations(context.Context) ([]*github.Installation, error) { + return nil, nil +} From 75a7aed30f8c8dcc492ddf20a91f1991b98403de Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 18:11:34 -0400 Subject: [PATCH 11/48] add token connector --- pkg/sources/github/connector.go | 4 +- pkg/sources/github/connector_basicauth.go | 1 + pkg/sources/github/connector_token.go | 104 ++++++++++++++++++++++ pkg/sources/github/github.go | 2 +- 4 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 pkg/sources/github/connector_basicauth.go create mode 100644 pkg/sources/github/connector_token.go diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 567c23e8ed05..f4ddf9a9e495 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -18,7 +18,7 @@ type connector interface { ListAppInstallations(ctx context.Context) ([]*github.Installation, error) } -func newConnector(connection *sourcespb.GitHub) (connector, error) { +func newConnector(connection *sourcespb.GitHub, handleRateLimit func(error) bool) (connector, error) { apiEndpoint := connection.Endpoint if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) { apiEndpoint = cloudEndpoint @@ -27,6 +27,8 @@ func newConnector(connection *sourcespb.GitHub) (connector, error) { switch cred := connection.GetCredential().(type) { case *sourcespb.GitHub_GithubApp: return newAppConnector(apiEndpoint, cred.GithubApp) + case *sourcespb.GitHub_Token: + return newTokenConnector(apiEndpoint, cred.Token, handleRateLimit) case *sourcespb.GitHub_Unauthenticated: return newUnauthenticatedConnector(apiEndpoint) default: diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go new file mode 100644 index 000000000000..d2e73c266e17 --- /dev/null +++ b/pkg/sources/github/connector_basicauth.go @@ -0,0 +1 @@ +package github diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go new file mode 100644 index 000000000000..3d4a4b06a3dd --- /dev/null +++ b/pkg/sources/github/connector_token.go @@ -0,0 +1,104 @@ +package github + +import ( + "net/http" + "strings" + "sync" + + gogit "github.com/go-git/go-git/v5" + "github.com/google/go-github/v62/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" + "golang.org/x/oauth2" +) + +type tokenConnector struct { + httpClient *http.Client + apiClient *github.Client + token string + isGitHubEnterprise bool + handleRateLimit func(error) bool + user string + userMu sync.Mutex +} + +var _ connector = (*tokenConnector)(nil) + +func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(error) bool) (*tokenConnector, error) { + tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + httpClient := common.RetryableHTTPClientTimeout(60) + httpClient.Transport = &oauth2.Transport{ + Base: httpClient.Transport, + Source: oauth2.ReuseTokenSource(nil, tokenSource), + } + + apiClient, err := createGitHubClient(httpClient, apiEndpoint) + if err != nil { + return nil, err + } + + return &tokenConnector{ + httpClient: httpClient, + apiClient: apiClient, + token: token, + isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), + handleRateLimit: handleRateLimit, + }, nil +} + +func (c *tokenConnector) ApiClient() *github.Client { + return c.apiClient +} + +func (c *tokenConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { + if err := c.setUserIfUnset(ctx); err != nil { + return "", nil, err + } + return git.CloneRepoUsingToken(ctx, c.token, repoURL, c.user) +} + +func (c *tokenConnector) IsGithubEnterprise() bool { + return c.isGitHubEnterprise +} + +func (c *tokenConnector) HttpClient() *http.Client { + return c.httpClient +} + +func (c *tokenConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { + return nil, nil +} + +func (c *tokenConnector) getUser(ctx context.Context) (string, error) { + var ( + user *github.User + err error + ) + for { + user, _, err = c.apiClient.Users.Get(ctx, "") + if c.handleRateLimit(err) { + continue + } + if err != nil { + return "", err + } + break + } + return user.GetLogin(), nil +} + +func (c *tokenConnector) setUserIfUnset(ctx context.Context) error { + c.userMu.Lock() + defer c.userMu.Unlock() + + if c.user == "" { + if user, err := c.getUser(ctx); err != nil { + return err + } else { + c.user = user + } + } + + return nil +} diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 27c5b37c8dee..53157b39da14 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -205,7 +205,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so } s.conn = &conn - connector, err := newConnector(s.conn) + connector, err := newConnector(s.conn, s.handleRateLimit) if err != nil { return fmt.Errorf("could not create connector: %w", err) } From 88b15fced880297252205cd0fc0ca8962d2c42ae Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 18:20:43 -0400 Subject: [PATCH 12/48] add basic auth connector --- pkg/sources/github/connector.go | 2 + pkg/sources/github/connector_basicauth.go | 60 +++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index f4ddf9a9e495..20442bd079de 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -27,6 +27,8 @@ func newConnector(connection *sourcespb.GitHub, handleRateLimit func(error) bool switch cred := connection.GetCredential().(type) { case *sourcespb.GitHub_GithubApp: return newAppConnector(apiEndpoint, cred.GithubApp) + case *sourcespb.GitHub_BasicAuth: + return newBasicAuthConnector(apiEndpoint, cred.BasicAuth) case *sourcespb.GitHub_Token: return newTokenConnector(apiEndpoint, cred.Token, handleRateLimit) case *sourcespb.GitHub_Unauthenticated: diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index d2e73c266e17..a3caa452b892 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -1 +1,61 @@ package github + +import ( + "net/http" + + gogit "github.com/go-git/go-git/v5" + "github.com/google/go-github/v62/github" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" + "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" + "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" +) + +type basicAuthConnector struct { + httpClient *http.Client + apiClient *github.Client + username string + password string +} + +var _ connector = (*basicAuthConnector)(nil) + +func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) { + httpClient := common.RetryableHTTPClientTimeout(60) + httpClient.Transport = &github.BasicAuthTransport{ + Username: cred.Username, + Password: cred.Password, + } + + apiClient, err := createGitHubClient(httpClient, apiEndpoint) + if err != nil { + return nil, err + } + + return &basicAuthConnector{ + httpClient: httpClient, + apiClient: apiClient, + username: cred.Username, + password: cred.Password, + }, nil +} + +func (c basicAuthConnector) ApiClient() *github.Client { + return c.apiClient +} + +func (c basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { + return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username) +} + +func (c basicAuthConnector) IsGithubEnterprise() bool { + return false +} + +func (c basicAuthConnector) HttpClient() *http.Client { + return c.httpClient +} + +func (c basicAuthConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { + return nil, nil +} From 24332272c39f7c830511df59054daba50eb2b730 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Mon, 29 Jul 2024 18:32:03 -0400 Subject: [PATCH 13/48] update non-integration tests --- pkg/sources/github/github_test.go | 98 +++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 30 deletions(-) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 5071007af4d5..daa9b2fb2c5d 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -44,8 +44,9 @@ func initTestSource(src *sourcespb.GitHub) *Source { if err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1); err != nil { panic(err) } - s.apiClient = github.NewClient(s.httpClient) - gock.InterceptClient(s.httpClient) + //s.apiClient = github.NewClient(s.httpClient) + //gock.InterceptClient(s.httpClient) + gock.InterceptClient(s.connector.HttpClient()) return s } @@ -156,7 +157,7 @@ func TestAddGistsByUser(t *testing.T) { Reply(200). JSON([]map[string]string{{"id": "aa5a315d61ae9438b18d", "git_pull_url": "https://gist.github.com/aa5a315d61ae9438b18d.git"}}) - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.addUserGistsToCache(context.Background(), "super-secret-user") assert.Nil(t, err) assert.Equal(t, 1, s.filteredRepoCache.Count()) @@ -177,7 +178,7 @@ func TestAddMembersByOrg(t *testing.T) { {"login": "testman2"}, }) - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.addMembersByOrg(context.Background(), "org1") assert.Nil(t, err) assert.Equal(t, 2, len(s.memberCache)) @@ -207,8 +208,8 @@ func TestAddMembersByApp(t *testing.T) { {"login": "ssm3"}, }) - s := initTestSource(nil) - err := s.addMembersByApp(context.Background(), github.NewClient(nil)) + s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) + err := s.addMembersByApp(context.Background()) assert.Nil(t, err) assert.Equal(t, 3, len(s.memberCache)) _, ok := s.memberCache["ssm1"] @@ -234,7 +235,7 @@ func TestAddReposByApp(t *testing.T) { }, }) - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) err := s.getReposByApp(context.Background()) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) @@ -258,7 +259,7 @@ func TestAddOrgsByUser(t *testing.T) { {"login": "sso2"}, }) - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) s.addOrgsByUser(context.Background(), "super-secret-user") assert.Equal(t, 1, s.orgsCache.Count()) ok := s.orgsCache.Exists("sso2") @@ -312,7 +313,7 @@ func TestNormalizeRepos(t *testing.T) { t.Run(tt.name, func(t *testing.T) { defer gock.Off() tt.setup() - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) got, err := s.normalizeRepo(tt.repos[0]) if (err != nil) != tt.wantErr { @@ -337,7 +338,7 @@ func TestNormalizeRepos(t *testing.T) { } func TestHandleRateLimit(t *testing.T) { - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) assert.False(t, s.handleRateLimit(nil)) // Request @@ -381,10 +382,14 @@ func TestEnumerateUnauthenticated(t *testing.T) { Reply(200). JSON([]map[string]string{{"full_name": "super-secret-org/super-secret-repo", "clone_url": "https://github.com/super-secret-org/super-secret-repo.git"}}) - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{ + Endpoint: apiEndpoint, + Credential: &sourcespb.GitHub_Unauthenticated{}, + }) s.orgsCache = memory.New[string]() s.orgsCache.Set("super-secret-org", "super-secret-org") - s.enumerateUnauthenticated(context.Background(), apiEndpoint) + //s.enumerateUnauthenticated(context.Background(), apiEndpoint) + s.enumerateUnauthenticated(context.Background()) assert.Equal(t, 1, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-org/super-secret-repo") assert.True(t, ok) @@ -417,8 +422,14 @@ func TestEnumerateWithToken(t *testing.T) { Reply(200). JSON([]map[string]string{{"id": "super-secret-gist", "git_pull_url": "https://gist.github.com/super-secret-gist.git"}}) - s := initTestSource(nil) - err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token") + s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", + Credential: &sourcespb.GitHub_Token{ + Token: "token", + }, + }) + //err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token") + err := s.enumerateWithToken(context.Background()) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-user/super-secret-repo") @@ -453,11 +464,17 @@ func BenchmarkEnumerateWithToken(b *testing.B) { Reply(200). JSON([]map[string]string{{"git_pull_url": "https://github.com/super-secret-gist.git"}}) - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", + Credential: &sourcespb.GitHub_Token{ + Token: "token", + }, + }) b.ResetTimer() for i := 0; i < b.N; i++ { - _ = s.enumerateWithToken(context.Background(), "https://api.github.com", "token") + //_ = s.enumerateWithToken(context.Background(), "https://api.github.com", "token") + _ = s.enumerateWithToken(context.Background()) } } @@ -488,6 +505,7 @@ func TestEnumerate(t *testing.T) { JSON(`[{"git_pull_url": "https://gist.github.com/2801a2b0523099d0614a951579d99ba9.git", "id": "2801a2b0523099d0614a951579d99ba9"}]`) s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, @@ -521,7 +539,8 @@ func TestEnumerate(t *testing.T) { s.filteredRepoCache.Set(repo.GetFullName(), repo.GetCloneURL()) // Act - _, err := s.enumerate(context.Background(), "https://api.github.com") + //_, err := s.enumerate(context.Background(), "https://api.github.com") + err := s.enumerate(context.Background()) // Assert assert.Nil(t, err) @@ -586,6 +605,7 @@ func mockGists() []map[string]string { func BenchmarkEnumerate(b *testing.B) { for i := 0; i < b.N; i++ { s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_Token{ Token: "super secret token", }, @@ -593,7 +613,8 @@ func BenchmarkEnumerate(b *testing.B) { setupMocks(b) b.StartTimer() - _, _ = s.enumerate(context.Background(), "https://api.github.com") + //_, _ = s.enumerate(context.Background(), "https://api.github.com") + _ = s.enumerate(context.Background()) } } @@ -605,10 +626,16 @@ func TestEnumerateWithToken_IncludeRepos(t *testing.T) { Reply(200). JSON(map[string]string{"login": "super-secret-user"}) - s := initTestSource(nil) + s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", + Credential: &sourcespb.GitHub_Token{ + Token: "token", + }, + }) s.repos = []string{"some-special-repo"} - err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token") + //err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token") + err := s.enumerateWithToken(context.Background()) assert.Nil(t, err) assert.Equal(t, 1, len(s.repos)) assert.Equal(t, []string{"some-special-repo"}, s.repos) @@ -647,16 +674,26 @@ func TestEnumerateWithApp(t *testing.T) { Reply(200). JSON(map[string]string{}) - s := initTestSource(nil) - _, err := s.enumerateWithApp( - context.Background(), - "https://api.github.com", - &credentialspb.GitHubApp{ - InstallationId: "1337", - AppId: "4141", - PrivateKey: privateKey, + s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", + Credential: &sourcespb.GitHub_GithubApp{ + &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, }, - ) + }) + //_, err := s.enumerateWithApp( + // context.Background(), + // "https://api.github.com", + // &credentialspb.GitHubApp{ + // InstallationId: "1337", + // AppId: "4141", + // PrivateKey: privateKey, + // }, + //) + err := s.enumerateWithApp(context.Background()) assert.Nil(t, err) assert.Equal(t, 0, len(s.repos)) assert.False(t, gock.HasUnmatchedRequest()) @@ -775,10 +812,11 @@ func Test_scan_SetProgressComplete(t *testing.T) { t.Run(tc.name, func(t *testing.T) { src := initTestSource(&sourcespb.GitHub{ Repositories: tc.repos, + Credential: &sourcespb.GitHub_Unauthenticated{}, }) src.jobPool = &errgroup.Group{} - _ = src.scan(context.Background(), nil, nil) + _ = src.scan(context.Background(), nil) if !tc.wantErr { assert.Equal(t, "", src.GetProgress().EncodedResumeInfo) } From 3f7543241c99d6a45dbc57c25c5beb08a103e0ba Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 11:13:58 -0400 Subject: [PATCH 14/48] extract enumerate() --- pkg/sources/github/connector.go | 17 +++++++++-------- pkg/sources/github/connector_app.go | 12 +++++++++++- pkg/sources/github/connector_basicauth.go | 12 +++++++++++- pkg/sources/github/connector_token.go | 13 ++++++++++++- pkg/sources/github/connector_unauthenticated.go | 12 +++++++++++- pkg/sources/github/github.go | 2 +- 6 files changed, 55 insertions(+), 13 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 20442bd079de..3ecef41e4214 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -13,26 +13,27 @@ import ( type connector interface { ApiClient() *github.Client Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) - IsGithubEnterprise() bool + Enumerate(ctx context.Context) error HttpClient() *http.Client + IsGithubEnterprise() bool ListAppInstallations(ctx context.Context) ([]*github.Installation, error) } -func newConnector(connection *sourcespb.GitHub, handleRateLimit func(error) bool) (connector, error) { - apiEndpoint := connection.Endpoint +func newConnector(source *Source) (connector, error) { + apiEndpoint := source.conn.Endpoint if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) { apiEndpoint = cloudEndpoint } - switch cred := connection.GetCredential().(type) { + switch cred := source.conn.GetCredential().(type) { case *sourcespb.GitHub_GithubApp: - return newAppConnector(apiEndpoint, cred.GithubApp) + return newAppConnector(apiEndpoint, cred.GithubApp, source.enumerateWithApp) case *sourcespb.GitHub_BasicAuth: - return newBasicAuthConnector(apiEndpoint, cred.BasicAuth) + return newBasicAuthConnector(apiEndpoint, cred.BasicAuth, source.enumerateBasicAuth) case *sourcespb.GitHub_Token: - return newTokenConnector(apiEndpoint, cred.Token, handleRateLimit) + return newTokenConnector(apiEndpoint, cred.Token, source.handleRateLimit, source.enumerateWithToken) case *sourcespb.GitHub_Unauthenticated: - return newUnauthenticatedConnector(apiEndpoint) + return newUnauthenticatedConnector(apiEndpoint, source.enumerateUnauthenticated) default: return nil, fmt.Errorf("unknown connection type") } diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 2ae3cc070a2d..7a8cb16f066a 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -18,11 +18,16 @@ type appConnector struct { apiClient *github.Client installationClient *github.Client installationID int64 + enumerate func(ctx context.Context) error } var _ connector = (*appConnector)(nil) -func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { +func newAppConnector( + apiEndpoint string, + app *credentialspb.GitHubApp, + enumerate func(ctx context.Context) error) (*appConnector, error) { + installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) if err != nil { return nil, err @@ -72,6 +77,7 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn apiClient: apiClient, installationClient: installationClient, installationID: installationID, + enumerate: enumerate, }, nil } @@ -92,6 +98,10 @@ func (c appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token") } +func (c appConnector) Enumerate(ctx context.Context) error { + return c.enumerate(ctx) +} + func (c appConnector) IsGithubEnterprise() bool { return false } diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index a3caa452b892..a3a34cf5c8e8 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -16,11 +16,16 @@ type basicAuthConnector struct { apiClient *github.Client username string password string + enumerate func(ctx context.Context) error } var _ connector = (*basicAuthConnector)(nil) -func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) { +func newBasicAuthConnector( + apiEndpoint string, + cred *credentialspb.BasicAuth, + enumerate func(ctx context.Context) error) (*basicAuthConnector, error) { + httpClient := common.RetryableHTTPClientTimeout(60) httpClient.Transport = &github.BasicAuthTransport{ Username: cred.Username, @@ -37,6 +42,7 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (* apiClient: apiClient, username: cred.Username, password: cred.Password, + enumerate: enumerate, }, nil } @@ -48,6 +54,10 @@ func (c basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username) } +func (c basicAuthConnector) Enumerate(ctx context.Context) error { + return c.enumerate(ctx) +} + func (c basicAuthConnector) IsGithubEnterprise() bool { return false } diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 3d4a4b06a3dd..585b55adf610 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -18,6 +18,7 @@ type tokenConnector struct { apiClient *github.Client token string isGitHubEnterprise bool + enumerate func(ctx context.Context) error handleRateLimit func(error) bool user string userMu sync.Mutex @@ -25,7 +26,12 @@ type tokenConnector struct { var _ connector = (*tokenConnector)(nil) -func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(error) bool) (*tokenConnector, error) { +func newTokenConnector( + apiEndpoint string, + token string, + handleRateLimit func(error) bool, + enumerate func(ctx context.Context) error) (*tokenConnector, error) { + tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) httpClient := common.RetryableHTTPClientTimeout(60) httpClient.Transport = &oauth2.Transport{ @@ -43,6 +49,7 @@ func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(er apiClient: apiClient, token: token, isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), + enumerate: enumerate, handleRateLimit: handleRateLimit, }, nil } @@ -58,6 +65,10 @@ func (c *tokenConnector) Clone(ctx context.Context, repoURL string) (string, *go return git.CloneRepoUsingToken(ctx, c.token, repoURL, c.user) } +func (c *tokenConnector) Enumerate(ctx context.Context) error { + return c.enumerate(ctx) +} + func (c *tokenConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 26c33dc6a06d..076dd5812f90 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -13,11 +13,15 @@ import ( type unauthenticatedConnector struct { httpClient *http.Client apiClient *github.Client + enumerate func(ctx context.Context) } var _ connector = (*unauthenticatedConnector)(nil) -func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { +func newUnauthenticatedConnector( + apiEndpoint string, + enumerate func(ctx context.Context)) (*unauthenticatedConnector, error) { + httpClient := common.RetryableHTTPClientTimeout(60) apiClient, err := createGitHubClient(httpClient, apiEndpoint) if err != nil { @@ -26,6 +30,7 @@ func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, return &unauthenticatedConnector{ httpClient: httpClient, apiClient: apiClient, + enumerate: enumerate, }, nil } @@ -37,6 +42,11 @@ func (c unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (st return git.CloneRepoUsingUnauthenticated(ctx, repoURL) } +func (c unauthenticatedConnector) Enumerate(ctx context.Context) error { + c.enumerate(ctx) + return nil +} + func (c unauthenticatedConnector) IsGithubEnterprise() bool { return false } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 53157b39da14..1c10d4c6d320 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -205,7 +205,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so } s.conn = &conn - connector, err := newConnector(s.conn, s.handleRateLimit) + connector, err := newConnector(s) if err != nil { return fmt.Errorf("could not create connector: %w", err) } From 39a55b3a25fc218d61f8fa656d8cbf5d8ca17d32 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 11:15:21 -0400 Subject: [PATCH 15/48] update enumerate --- pkg/sources/github/github.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 1c10d4c6d320..4e3c7f69680b 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -355,6 +355,9 @@ func (s *Source) enumerate(ctx context.Context) error { // // TODO: move this error to Init // return nil, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()) //} + if err := s.connector.Enumerate(ctx); err != nil { + return err + } s.repos = make([]string, 0, s.filteredRepoCache.Count()) From e7e2afdea655bbc49032c2973773eaf8883623ce Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 11:17:01 -0400 Subject: [PATCH 16/48] fix test setup --- pkg/sources/github/github_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index daa9b2fb2c5d..bb9c6c3f6a0f 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -677,7 +677,7 @@ func TestEnumerateWithApp(t *testing.T) { s := initTestSource(&sourcespb.GitHub{ Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_GithubApp{ - &credentialspb.GitHubApp{ + GithubApp: &credentialspb.GitHubApp{ PrivateKey: privateKey, InstallationId: "1337", AppId: "4141", From 122ffbfee5afb2cfc32b3c23ef499669a4f5478d Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 11:24:24 -0400 Subject: [PATCH 17/48] init test correctly --- pkg/sources/github/github_test.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index bb9c6c3f6a0f..9f64b0a2b764 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -193,6 +193,24 @@ func TestAddMembersByOrg(t *testing.T) { func TestAddMembersByApp(t *testing.T) { defer gock.Off() + // generate a private key (it just needs to be in the right format) + privateKey := func() string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic(err) + } + + data := x509.MarshalPKCS1PrivateKey(key) + var pemKey bytes.Buffer + if err := pem.Encode(&pemKey, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: data, + }); err != nil { + panic(err) + } + return pemKey.String() + }() + gock.New("https://api.github.com"). Get("/app/installations"). Reply(200). @@ -208,7 +226,15 @@ func TestAddMembersByApp(t *testing.T) { {"login": "ssm3"}, }) - s := initTestSource(&sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}) + s := initTestSource(&sourcespb.GitHub{ + Endpoint: "https://api.github.com", + Credential: &sourcespb.GitHub_GithubApp{ + GithubApp: &credentialspb.GitHubApp{ + PrivateKey: privateKey, + InstallationId: "1337", + AppId: "4141", + }, + }}) err := s.addMembersByApp(context.Background()) assert.Nil(t, err) assert.Equal(t, 3, len(s.memberCache)) From f53f5e7c79f4f15784108ce620c0ea998248a4e8 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 11:57:10 -0400 Subject: [PATCH 18/48] stash --- pkg/sources/github/connector.go | 2 +- pkg/sources/github/connector_app.go | 16 ++++++++++------ pkg/sources/github/connector_basicauth.go | 4 ++-- pkg/sources/github/connector_token.go | 4 ++-- pkg/sources/github/connector_unauthenticated.go | 4 ++-- pkg/sources/github/github.go | 13 ++++++------- pkg/sources/github/github_test.go | 6 +++++- 7 files changed, 28 insertions(+), 21 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 3ecef41e4214..60d607061934 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -16,7 +16,7 @@ type connector interface { Enumerate(ctx context.Context) error HttpClient() *http.Client IsGithubEnterprise() bool - ListAppInstallations(ctx context.Context) ([]*github.Installation, error) + InstallationClient() *github.Client } func newConnector(source *Source) (connector, error) { diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 7a8cb16f066a..4387fbc04121 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -110,10 +110,14 @@ func (c appConnector) HttpClient() *http.Client { return c.httpClient } -func (c appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { - opts := &github.ListOptions{ - PerPage: membersAppPagination, - } - installs, _, err := c.installationClient.Apps.ListInstallations(ctx, opts) - return installs, err +func (c appConnector) InstallationClient() *github.Client { + return c.installationClient } + +//func (c appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { +// opts := &github.ListOptions{ +// PerPage: membersAppPagination, +// } +// installs, _, err := c.installationClient.Apps.ListInstallations(ctx, opts) +// return installs, err +//} diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index a3a34cf5c8e8..4fdfb2780f3c 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -66,6 +66,6 @@ func (c basicAuthConnector) HttpClient() *http.Client { return c.httpClient } -func (c basicAuthConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { - return nil, nil +func (c basicAuthConnector) InstallationClient() *github.Client { + return nil } diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 585b55adf610..76678eea3c7f 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -77,8 +77,8 @@ func (c *tokenConnector) HttpClient() *http.Client { return c.httpClient } -func (c *tokenConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { - return nil, nil +func (c *tokenConnector) InstallationClient() *github.Client { + return nil } func (c *tokenConnector) getUser(ctx context.Context) (string, error) { diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 076dd5812f90..fffee2223516 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -55,6 +55,6 @@ func (c unauthenticatedConnector) HttpClient() *http.Client { return c.httpClient } -func (c unauthenticatedConnector) ListAppInstallations(context.Context) ([]*github.Installation, error) { - return nil, nil +func (c unauthenticatedConnector) InstallationClient() *github.Client { + return nil } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 4e3c7f69680b..128bde6ed630 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -612,7 +612,7 @@ func (s *Source) enumerateWithApp(ctx context.Context) error { // Check if we need to find user repos. if s.conn.ScanUsers { - err := s.addMembersByApp(ctx) + err := s.addMembersByApp(ctx, s.connector.InstallationClient()) if err != nil { return err } @@ -873,14 +873,13 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string) error { return nil } -func (s *Source) addMembersByApp(ctx context.Context) error { - //opts := &github.ListOptions{ - // PerPage: membersAppPagination, - //} +func (s *Source) addMembersByApp(ctx context.Context, installationClient *github.Client) error { + opts := &github.ListOptions{ + PerPage: membersAppPagination, + } // TODO: Check rate limit for this call. - //installs, _, err := installationClient.Apps.ListInstallations(ctx, opts) - installs, err := s.connector.ListAppInstallations(ctx) + installs, _, err := installationClient.Apps.ListInstallations(ctx, opts) if err != nil { return fmt.Errorf("could not enumerate installed orgs: %w", err) } diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 9f64b0a2b764..0f058688fede 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -47,6 +47,10 @@ func initTestSource(src *sourcespb.GitHub) *Source { //s.apiClient = github.NewClient(s.httpClient) //gock.InterceptClient(s.httpClient) gock.InterceptClient(s.connector.HttpClient()) + gock.InterceptClient(s.connector.ApiClient().Client()) + if installationClient := s.connector.InstallationClient(); installationClient != nil { + gock.InterceptClient(installationClient.Client()) + } return s } @@ -235,7 +239,7 @@ func TestAddMembersByApp(t *testing.T) { AppId: "4141", }, }}) - err := s.addMembersByApp(context.Background()) + err := s.addMembersByApp(context.Background(), s.connector.InstallationClient()) assert.Nil(t, err) assert.Equal(t, 3, len(s.memberCache)) _, ok := s.memberCache["ssm1"] From 13cd94e27bdfa98223767742d18414039c2a4388 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 12:28:08 -0400 Subject: [PATCH 19/48] add mock to support TestAddMembersByApp --- pkg/sources/github/github_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 0f058688fede..4df5c5e0ae4c 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -47,6 +47,7 @@ func initTestSource(src *sourcespb.GitHub) *Source { //s.apiClient = github.NewClient(s.httpClient) //gock.InterceptClient(s.httpClient) gock.InterceptClient(s.connector.HttpClient()) + gock.Intercept() gock.InterceptClient(s.connector.ApiClient().Client()) if installationClient := s.connector.InstallationClient(); installationClient != nil { gock.InterceptClient(installationClient.Client()) @@ -221,6 +222,10 @@ func TestAddMembersByApp(t *testing.T) { JSON([]map[string]any{ {"account": map[string]string{"login": "super-secret-org", "type": "Organization"}}, }) + gock.New("https://api.github.com"). + Post("/app/installations/1337/access_tokens"). + Reply(200). + JSON(map[string]string{"token": "dontlook"}) gock.New("https://api.github.com"). Get("/orgs/super-secret-org/members"). Reply(200). From 4d144e79aa2eaf20f97201473cfc2e6432c16b7e Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 14:38:13 -0400 Subject: [PATCH 20/48] get integration tests compiling --- pkg/sources/github/github_integration_test.go | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index c88207e276c3..a1d47bf28b02 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -21,7 +21,6 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" "github.com/trufflesecurity/trufflehog/v3/pkg/sources" - "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" ) func TestSource_Token(t *testing.T) { @@ -42,7 +41,8 @@ func TestSource_Token(t *testing.T) { githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") - conn := &sourcespb.GitHub{ + src := &sourcespb.GitHub{ + Endpoint: "https://api.github.com", Credential: &sourcespb.GitHub_GithubApp{ GithubApp: &credentialspb.GitHubApp{ PrivateKey: githubPrivateKeyNew, @@ -51,32 +51,36 @@ func TestSource_Token(t *testing.T) { }, }, } + conn, err := anypb.New(src) + if err != nil { + panic(err) + } s := Source{ - conn: conn, - httpClient: common.SaneHttpClient(), + conn: src, log: logr.Discard(), memberCache: map[string]struct{}{}, repoInfoCache: newRepoInfoCache(), } + s.Init(ctx, "github integration test source", 0, 0, false, conn, 1) s.filteredRepoCache = s.newFilteredRepoCache(memory.New[string](), nil, nil) - installationClient, err := s.enumerateWithApp(ctx, "https://api.github.com", conn.GetGithubApp()) - assert.NoError(t, err) - - user, token, err := s.userAndToken(ctx, installationClient) - assert.NotEmpty(t, token) + err = s.enumerateWithApp(ctx) assert.NoError(t, err) - // user provided - _, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", user) - assert.NoError(t, err) - - // no user provided - _, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", "") - assert.Error(t, err) - - _, _, err = s.cloneRepo(ctx, "https://github.com/truffle-test-integration-org/another-test-repo.git", installationClient) + //user, token, err := s.userAndToken(ctx, installationClient) + //assert.NotEmpty(t, token) + //assert.NoError(t, err) + // + //// user provided + //_, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", user) + //assert.NoError(t, err) + // + //// no user provided + //_, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", "") + //assert.Error(t, err) + + _, _, err = s.cloneRepo(ctx, "https://github.com/truffle-test-integration-org/another-test-repo.git") assert.NoError(t, err) } From b48796a458a6f87e45e47a414c1aa3571f653bc6 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Tue, 30 Jul 2024 15:15:28 -0400 Subject: [PATCH 21/48] update to github v63 --- pkg/sources/github/connector.go | 2 +- pkg/sources/github/connector_app.go | 2 +- pkg/sources/github/connector_basicauth.go | 2 +- pkg/sources/github/connector_token.go | 2 +- pkg/sources/github/connector_unauthenticated.go | 2 +- pkg/sources/github/github.go | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 60d607061934..d1becb0cd554 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -5,7 +5,7 @@ import ( "net/http" gogit "github.com/go-git/go-git/v5" - "github.com/google/go-github/v62/github" + "github.com/google/go-github/v63/github" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" ) diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 4387fbc04121..4345f69426dd 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -6,7 +6,7 @@ import ( "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" - "github.com/google/go-github/v62/github" + "github.com/google/go-github/v63/github" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 4fdfb2780f3c..235ffe13fd31 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -4,7 +4,7 @@ import ( "net/http" gogit "github.com/go-git/go-git/v5" - "github.com/google/go-github/v62/github" + "github.com/google/go-github/v63/github" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/pb/credentialspb" diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 76678eea3c7f..43c2e0fb9ad8 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -6,7 +6,7 @@ import ( "sync" gogit "github.com/go-git/go-git/v5" - "github.com/google/go-github/v62/github" + "github.com/google/go-github/v63/github" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index fffee2223516..9c20505b9da8 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -4,7 +4,7 @@ import ( "net/http" gogit "github.com/go-git/go-git/v5" - "github.com/google/go-github/v62/github" + "github.com/google/go-github/v63/github" "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" "github.com/trufflesecurity/trufflehog/v3/pkg/sources/git" diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 128bde6ed630..ea0f818f0d2f 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -15,7 +15,7 @@ import ( "github.com/go-logr/logr" "github.com/gobwas/glob" - "github.com/google/go-github/v62/github" + "github.com/google/go-github/v63/github" "golang.org/x/exp/rand" "golang.org/x/sync/errgroup" "google.golang.org/protobuf/proto" From 86fa52985d6e9d4b4df42e55fdb47956ef88d6cb Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 10:38:39 -0400 Subject: [PATCH 22/48] un-extract enumeration --- pkg/sources/github/connector.go | 9 ++++--- pkg/sources/github/connector_app.go | 24 ++++++------------- pkg/sources/github/connector_basicauth.go | 22 +++++------------ pkg/sources/github/connector_token.go | 13 +--------- .../github/connector_unauthenticated.go | 21 +++++----------- pkg/sources/github/github.go | 20 ++++++++++++++-- 6 files changed, 42 insertions(+), 67 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index d1becb0cd554..0c134ebbda0a 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -13,7 +13,6 @@ import ( type connector interface { ApiClient() *github.Client Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) - Enumerate(ctx context.Context) error HttpClient() *http.Client IsGithubEnterprise() bool InstallationClient() *github.Client @@ -27,13 +26,13 @@ func newConnector(source *Source) (connector, error) { switch cred := source.conn.GetCredential().(type) { case *sourcespb.GitHub_GithubApp: - return newAppConnector(apiEndpoint, cred.GithubApp, source.enumerateWithApp) + return newAppConnector(apiEndpoint, cred.GithubApp) case *sourcespb.GitHub_BasicAuth: - return newBasicAuthConnector(apiEndpoint, cred.BasicAuth, source.enumerateBasicAuth) + return newBasicAuthConnector(apiEndpoint, cred.BasicAuth) case *sourcespb.GitHub_Token: - return newTokenConnector(apiEndpoint, cred.Token, source.handleRateLimit, source.enumerateWithToken) + return newTokenConnector(apiEndpoint, cred.Token, source.handleRateLimit) case *sourcespb.GitHub_Unauthenticated: - return newUnauthenticatedConnector(apiEndpoint, source.enumerateUnauthenticated) + return newUnauthenticatedConnector(apiEndpoint) default: return nil, fmt.Errorf("unknown connection type") } diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 4345f69426dd..22f46727de05 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -18,16 +18,11 @@ type appConnector struct { apiClient *github.Client installationClient *github.Client installationID int64 - enumerate func(ctx context.Context) error } var _ connector = (*appConnector)(nil) -func newAppConnector( - apiEndpoint string, - app *credentialspb.GitHubApp, - enumerate func(ctx context.Context) error) (*appConnector, error) { - +func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) if err != nil { return nil, err @@ -77,15 +72,14 @@ func newAppConnector( apiClient: apiClient, installationClient: installationClient, installationID: installationID, - enumerate: enumerate, }, nil } -func (c appConnector) ApiClient() *github.Client { +func (c *appConnector) ApiClient() *github.Client { return c.apiClient } -func (c appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { +func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { // TODO: Check rate limit for this call. token, _, err := c.installationClient.Apps.CreateInstallationToken( ctx, @@ -98,23 +92,19 @@ func (c appConnector) Clone(ctx context.Context, repoURL string) (string, *gogit return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token") } -func (c appConnector) Enumerate(ctx context.Context) error { - return c.enumerate(ctx) -} - -func (c appConnector) IsGithubEnterprise() bool { +func (c *appConnector) IsGithubEnterprise() bool { return false } -func (c appConnector) HttpClient() *http.Client { +func (c *appConnector) HttpClient() *http.Client { return c.httpClient } -func (c appConnector) InstallationClient() *github.Client { +func (c *appConnector) InstallationClient() *github.Client { return c.installationClient } -//func (c appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { +//func (c *appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { // opts := &github.ListOptions{ // PerPage: membersAppPagination, // } diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 235ffe13fd31..6718ad9a58cc 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -16,16 +16,11 @@ type basicAuthConnector struct { apiClient *github.Client username string password string - enumerate func(ctx context.Context) error } var _ connector = (*basicAuthConnector)(nil) -func newBasicAuthConnector( - apiEndpoint string, - cred *credentialspb.BasicAuth, - enumerate func(ctx context.Context) error) (*basicAuthConnector, error) { - +func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) { httpClient := common.RetryableHTTPClientTimeout(60) httpClient.Transport = &github.BasicAuthTransport{ Username: cred.Username, @@ -42,30 +37,25 @@ func newBasicAuthConnector( apiClient: apiClient, username: cred.Username, password: cred.Password, - enumerate: enumerate, }, nil } -func (c basicAuthConnector) ApiClient() *github.Client { +func (c *basicAuthConnector) ApiClient() *github.Client { return c.apiClient } -func (c basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { +func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username) } -func (c basicAuthConnector) Enumerate(ctx context.Context) error { - return c.enumerate(ctx) -} - -func (c basicAuthConnector) IsGithubEnterprise() bool { +func (c *basicAuthConnector) IsGithubEnterprise() bool { return false } -func (c basicAuthConnector) HttpClient() *http.Client { +func (c *basicAuthConnector) HttpClient() *http.Client { return c.httpClient } -func (c basicAuthConnector) InstallationClient() *github.Client { +func (c *basicAuthConnector) InstallationClient() *github.Client { return nil } diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 43c2e0fb9ad8..ada1b1880a4b 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -18,7 +18,6 @@ type tokenConnector struct { apiClient *github.Client token string isGitHubEnterprise bool - enumerate func(ctx context.Context) error handleRateLimit func(error) bool user string userMu sync.Mutex @@ -26,12 +25,7 @@ type tokenConnector struct { var _ connector = (*tokenConnector)(nil) -func newTokenConnector( - apiEndpoint string, - token string, - handleRateLimit func(error) bool, - enumerate func(ctx context.Context) error) (*tokenConnector, error) { - +func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(error) bool) (*tokenConnector, error) { tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) httpClient := common.RetryableHTTPClientTimeout(60) httpClient.Transport = &oauth2.Transport{ @@ -49,7 +43,6 @@ func newTokenConnector( apiClient: apiClient, token: token, isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), - enumerate: enumerate, handleRateLimit: handleRateLimit, }, nil } @@ -65,10 +58,6 @@ func (c *tokenConnector) Clone(ctx context.Context, repoURL string) (string, *go return git.CloneRepoUsingToken(ctx, c.token, repoURL, c.user) } -func (c *tokenConnector) Enumerate(ctx context.Context) error { - return c.enumerate(ctx) -} - func (c *tokenConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 9c20505b9da8..64a9be6e91a9 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -13,14 +13,11 @@ import ( type unauthenticatedConnector struct { httpClient *http.Client apiClient *github.Client - enumerate func(ctx context.Context) } var _ connector = (*unauthenticatedConnector)(nil) -func newUnauthenticatedConnector( - apiEndpoint string, - enumerate func(ctx context.Context)) (*unauthenticatedConnector, error) { +func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { httpClient := common.RetryableHTTPClientTimeout(60) apiClient, err := createGitHubClient(httpClient, apiEndpoint) @@ -30,31 +27,25 @@ func newUnauthenticatedConnector( return &unauthenticatedConnector{ httpClient: httpClient, apiClient: apiClient, - enumerate: enumerate, }, nil } -func (c unauthenticatedConnector) ApiClient() *github.Client { +func (c *unauthenticatedConnector) ApiClient() *github.Client { return c.apiClient } -func (c unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { +func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { return git.CloneRepoUsingUnauthenticated(ctx, repoURL) } -func (c unauthenticatedConnector) Enumerate(ctx context.Context) error { - c.enumerate(ctx) - return nil -} - -func (c unauthenticatedConnector) IsGithubEnterprise() bool { +func (c *unauthenticatedConnector) IsGithubEnterprise() bool { return false } -func (c unauthenticatedConnector) HttpClient() *http.Client { +func (c *unauthenticatedConnector) HttpClient() *http.Client { return c.httpClient } -func (c unauthenticatedConnector) InstallationClient() *github.Client { +func (c *unauthenticatedConnector) InstallationClient() *github.Client { return nil } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index ea0f818f0d2f..1a526874a4de 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -355,8 +355,24 @@ func (s *Source) enumerate(ctx context.Context) error { // // TODO: move this error to Init // return nil, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()) //} - if err := s.connector.Enumerate(ctx); err != nil { - return err + //if err := s.connector.Enumerate(ctx); err != nil { + // return err + //} + switch s.connector.(type) { + case *appConnector: + if err := s.enumerateWithApp(ctx); err != nil { + return err + } + case *basicAuthConnector: + if err := s.enumerateBasicAuth(ctx); err != nil { + return err + } + case *tokenConnector: + if err := s.enumerateWithToken(ctx); err != nil { + return err + } + case *unauthenticatedConnector: + s.enumerateUnauthenticated(ctx) } s.repos = make([]string, 0, s.filteredRepoCache.Count()) From 0d2aea6bf4327da5a37a3d7d09192aed06dc973c Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 12:46:09 -0400 Subject: [PATCH 23/48] implement isGHE everywhere --- pkg/sources/github/connector_app.go | 5 ++++- pkg/sources/github/connector_basicauth.go | 21 +++++++++++-------- .../github/connector_unauthenticated.go | 13 +++++++----- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 22f46727de05..f6ba37e8f410 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -3,6 +3,7 @@ package github import ( "net/http" "strconv" + "strings" "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" @@ -18,6 +19,7 @@ type appConnector struct { apiClient *github.Client installationClient *github.Client installationID int64 + isGitHubEnterprise bool } var _ connector = (*appConnector)(nil) @@ -72,6 +74,7 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn apiClient: apiClient, installationClient: installationClient, installationID: installationID, + isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), }, nil } @@ -93,7 +96,7 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogi } func (c *appConnector) IsGithubEnterprise() bool { - return false + return c.isGitHubEnterprise } func (c *appConnector) HttpClient() *http.Client { diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 6718ad9a58cc..20942778c5c7 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -2,6 +2,7 @@ package github import ( "net/http" + "strings" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v63/github" @@ -12,10 +13,11 @@ import ( ) type basicAuthConnector struct { - httpClient *http.Client - apiClient *github.Client - username string - password string + httpClient *http.Client + apiClient *github.Client + username string + password string + isGitHubEnterprise bool } var _ connector = (*basicAuthConnector)(nil) @@ -33,10 +35,11 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (* } return &basicAuthConnector{ - httpClient: httpClient, - apiClient: apiClient, - username: cred.Username, - password: cred.Password, + httpClient: httpClient, + apiClient: apiClient, + username: cred.Username, + password: cred.Password, + isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), }, nil } @@ -49,7 +52,7 @@ func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, } func (c *basicAuthConnector) IsGithubEnterprise() bool { - return false + return c.isGitHubEnterprise } func (c *basicAuthConnector) HttpClient() *http.Client { diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 64a9be6e91a9..b4715469444a 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -2,6 +2,7 @@ package github import ( "net/http" + "strings" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v63/github" @@ -11,8 +12,9 @@ import ( ) type unauthenticatedConnector struct { - httpClient *http.Client - apiClient *github.Client + httpClient *http.Client + apiClient *github.Client + isGitHubEnterprise bool } var _ connector = (*unauthenticatedConnector)(nil) @@ -25,8 +27,9 @@ func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, return nil, err } return &unauthenticatedConnector{ - httpClient: httpClient, - apiClient: apiClient, + httpClient: httpClient, + apiClient: apiClient, + isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), }, nil } @@ -39,7 +42,7 @@ func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (s } func (c *unauthenticatedConnector) IsGithubEnterprise() bool { - return false + return c.isGitHubEnterprise } func (c *unauthenticatedConnector) HttpClient() *http.Client { From 874fabf6141d908305810f062cbc563823e33519 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 13:07:53 -0400 Subject: [PATCH 24/48] remove old commented code --- pkg/sources/github/connector_app.go | 8 -- pkg/sources/github/github.go | 129 +----------------- pkg/sources/github/github_integration_test.go | 10 -- pkg/sources/github/repo.go | 91 ------------ 4 files changed, 1 insertion(+), 237 deletions(-) diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index f6ba37e8f410..08ad0426ac27 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -106,11 +106,3 @@ func (c *appConnector) HttpClient() *http.Client { func (c *appConnector) InstallationClient() *github.Client { return c.installationClient } - -//func (c *appConnector) ListAppInstallations(ctx context.Context) ([]*github.Installation, error) { -// opts := &github.ListOptions{ -// PerPage: membersAppPagination, -// } -// installs, _, err := c.installationClient.Apps.ListInstallations(ctx, opts) -// return installs, err -//} diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 1a526874a4de..f9f5c99dcd20 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -44,11 +44,6 @@ const ( type Source struct { name string - // Protects the user and token. - //userMu sync.Mutex - //githubUser string - //githubToken string - sourceID sources.SourceID jobID sources.JobID verify bool @@ -65,14 +60,12 @@ type Source struct { scanOptMu sync.Mutex // protects the scanOptions scanOptions *git.ScanOptions - //httpClient *http.Client log logr.Logger conn *sourcespb.GitHub jobPool *errgroup.Group resumeInfoMutex sync.Mutex resumeInfoSlice []string - //apiClient *github.Client - connector connector + connector connector includePRComments bool includeIssueComments bool @@ -195,9 +188,6 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so s.jobPool = &errgroup.Group{} s.jobPool.SetLimit(concurrency) - //s.httpClient = common.RetryableHTTPClientTimeout(60) - //s.apiClient = github.NewClient(s.httpClient) - var conn sourcespb.GitHub err = anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}) if err != nil { @@ -304,11 +294,6 @@ const cloudEndpoint = "https://api.github.com" // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error { - //apiEndpoint := s.conn.Endpoint - //if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) { - // apiEndpoint = cloudEndpoint - //} - // If targets are provided, we're only scanning the data in those targets. // Otherwise, we're scanning all data. // This allows us to only scan the commit where a vulnerability was found. @@ -331,33 +316,6 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar } func (s *Source) enumerate(ctx context.Context) error { - //var ( - // installationClient *github.Client - // err error - //) - // - //switch cred := s.conn.GetCredential().(type) { - //case *sourcespb.GitHub_BasicAuth: - // if err = s.enumerateBasicAuth(ctx, apiEndpoint, cred.BasicAuth); err != nil { - // return nil, err - // } - //case *sourcespb.GitHub_Unauthenticated: - // s.enumerateUnauthenticated(ctx, apiEndpoint) - //case *sourcespb.GitHub_Token: - // if err = s.enumerateWithToken(ctx, apiEndpoint, cred.Token); err != nil { - // return nil, err - // } - //case *sourcespb.GitHub_GithubApp: - // if installationClient, err = s.enumerateWithApp(ctx, apiEndpoint, cred.GithubApp); err != nil { - // return nil, err - // } - //default: - // // TODO: move this error to Init - // return nil, fmt.Errorf("Invalid configuration given for source. Name: %s, Type: %s", s.name, s.Type()) - //} - //if err := s.connector.Enumerate(ctx); err != nil { - // return err - //} switch s.connector.(type) { case *appConnector: if err := s.enumerateWithApp(ctx); err != nil { @@ -438,16 +396,6 @@ RepoLoop: } func (s *Source) enumerateBasicAuth(ctx context.Context) error { - //s.httpClient.Transport = &github.BasicAuthTransport{ - // Username: basicAuth.Username, - // Password: basicAuth.Password, - //} - //ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) - //if err != nil { - // s.log.Error(err, "error creating GitHub client") - //} - //s.apiClient = ghClient - for _, org := range s.orgsCache.Keys() { orgCtx := context.WithValue(ctx, "account", org) userType, err := s.getReposByOrgOrUser(ctx, org) @@ -467,11 +415,6 @@ func (s *Source) enumerateBasicAuth(ctx context.Context) error { } func (s *Source) enumerateUnauthenticated(ctx context.Context) { - //ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) - //if err != nil { - // s.log.Error(err, "error creating GitHub client") - //} - //s.apiClient = ghClient if s.orgsCache.Count() > unauthGithubOrgRateLimt { s.log.Info("You may experience rate limiting when using the unauthenticated GitHub api. Consider using an authenticated scan instead.") } @@ -491,26 +434,6 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context) { } func (s *Source) enumerateWithToken(ctx context.Context) error { - //// Needed for clones. - //s.githubToken = token - // - //// Needed to list repos. - //ts := oauth2.StaticTokenSource( - // &oauth2.Token{AccessToken: token}, - //) - //s.httpClient.Transport = &oauth2.Transport{ - // Base: s.httpClient.Transport, - // Source: oauth2.ReuseTokenSource(nil, ts), - //} - // - //// If we're using public GitHub, make a regular client. - //// Otherwise, make an enterprise client. - //ghClient, err := createGitHubClient(s.httpClient, apiEndpoint) - //if err != nil { - // s.log.Error(err, "error creating GitHub client") - //} - //s.apiClient = ghClient - ctx.Logger().V(1).Info("Enumerating with token") var ghUser *github.User var err error @@ -535,7 +458,6 @@ func (s *Source) enumerateWithToken(ctx context.Context) error { s.log.Error(err, "Unable to fetch gists for the current user", "user", ghUser.GetLogin()) } - //isGHE := !strings.EqualFold(apiEndpoint, cloudEndpoint) if s.connector.IsGithubEnterprise() { s.addAllVisibleOrgs(ctx) } else { @@ -571,55 +493,6 @@ func (s *Source) enumerateWithToken(ctx context.Context) error { } func (s *Source) enumerateWithApp(ctx context.Context) error { - //installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) - //if err != nil { - // return nil, err - //} - // - //appID, err := strconv.ParseInt(app.AppId, 10, 64) - //if err != nil { - // return nil, err - //} - // - //// This client is required to create installation tokens for cloning. - //// Otherwise, the required JWT is not in the request for the token :/ - //// This client uses the source's original HTTP transport, so make sure - //// to build it before modifying that transport (such as is done during - //// the creation of the other API client below). - //appItr, err := ghinstallation.NewAppsTransport( - // s.httpClient.Transport, - // appID, - // []byte(app.PrivateKey)) - //if err != nil { - // return nil, err - //} - //appItr.BaseURL = apiEndpoint - // - //// Does this need to be separate from |s.httpClient|? - //instHTTPClient := common.RetryableHTTPClientTimeout(60) - //instHTTPClient.Transport = appItr - //installationClient, err = github.NewClient(instHTTPClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - //if err != nil { - // return nil, err - //} - // - //// This client is used for most APIs. - //itr, err := ghinstallation.New( - // s.httpClient.Transport, - // appID, - // installationID, - // []byte(app.PrivateKey)) - //if err != nil { - // return nil, err - //} - //itr.BaseURL = apiEndpoint - // - //s.httpClient.Transport = itr - //s.apiClient, err = github.NewClient(s.httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) - //if err != nil { - // return nil, err - //} - // If no repos were provided, enumerate them. if len(s.repos) == 0 { if err := s.getReposByApp(ctx); err != nil { diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index a1d47bf28b02..73bd23e4f94c 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -320,16 +320,6 @@ func TestSource_Scan(t *testing.T) { githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") - // OLD app for breaking app change tests - // githubPrivateKeyB64 := secret.MustGetField("GITHUB_PRIVATE_KEY") - // githubPrivateKeyBytes, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64) - // if err != nil { - // t.Fatal(err) - // } - // githubPrivateKey := string(githubPrivateKeyBytes) - // githubInstallationID := secret.MustGetField("GITHUB_INSTALLATION_ID") - // githubAppID := secret.MustGetField("GITHUB_APP_ID") - type init struct { name string verify bool diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index f2379ece5729..4d203985b7a1 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -59,99 +59,8 @@ func (s *Source) cloneRepo( return "", nil, err } return path, repo, nil - - //switch s.conn.GetCredential().(type) { - //case *sourcespb.GitHub_BasicAuth: - // path, repo, err = git.CloneRepoUsingToken(ctx, s.conn.GetBasicAuth().GetPassword(), repoURL, s.conn.GetBasicAuth().GetUsername()) - // if err != nil { - // return "", nil, err - // } - //case *sourcespb.GitHub_Unauthenticated: - //path, repo, err = git.CloneRepoUsingUnauthenticated(ctx, repoURL) - // if err != nil { - // return "", nil, err - // } - // - //case *sourcespb.GitHub_GithubApp: - // s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient) - // if err != nil { - // return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) - // } - // - // path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) - // if err != nil { - // return "", nil, err - // } - // - //case *sourcespb.GitHub_Token: - // if err := s.getUserAndToken(ctx, repoURL, installationClient); err != nil { - // return "", nil, fmt.Errorf("error getting token for repo %s: %w", repoURL, err) - // } - // path, repo, err = git.CloneRepoUsingToken(ctx, s.githubToken, repoURL, s.githubUser) - // if err != nil { - // return "", nil, err - // } - //default: - // return "", nil, fmt.Errorf("unhandled credential type for repo %s: %T", repoURL, s.conn.GetCredential()) - //} - //return path, repo, nil } -//func (s *Source) getUserAndToken(ctx context.Context, repoURL string, installationClient *github.Client) error { -// // We never refresh user provided tokens, so if we already have them, we never need to try and fetch them again. -// s.userMu.Lock() -// defer s.userMu.Unlock() -// if s.githubUser == "" || s.githubToken == "" { -// var err error -// s.githubUser, s.githubToken, err = s.userAndToken(ctx, installationClient) -// if err != nil { -// return fmt.Errorf("error getting token for repo %s: %w", repoURL, err) -// } -// } -// return nil -//} - -//func (s *Source) userAndToken(ctx context.Context, installationClient *github.Client) (string, string, error) { -// switch cred := s.conn.GetCredential().(type) { -// case *sourcespb.GitHub_BasicAuth: -// return cred.BasicAuth.Username, cred.BasicAuth.Password, nil -// case *sourcespb.GitHub_Unauthenticated: -// // do nothing -// case *sourcespb.GitHub_GithubApp: -// id, err := strconv.ParseInt(cred.GithubApp.InstallationId, 10, 64) -// if err != nil { -// return "", "", fmt.Errorf("unable to parse installation id: %w", err) -// } -// // TODO: Check rate limit for this call. -// token, _, err := installationClient.Apps.CreateInstallationToken( -// ctx, id, &github.InstallationTokenOptions{}) -// if err != nil { -// return "", "", fmt.Errorf("unable to create installation token: %w", err) -// } -// return "x-access-token", token.GetToken(), nil // TODO: multiple workers request this, track the TTL -// case *sourcespb.GitHub_Token: -// var ( -// ghUser *github.User -// err error -// ) -// for { -// ghUser, _, err = s.apiClient.Users.Get(ctx, "") -// if s.handleRateLimit(err) { -// continue -// } -// if err != nil { -// return "", "", fmt.Errorf("unable to get user: %w", err) -// } -// break -// } -// return ghUser.GetLogin(), cred.Token, nil -// default: -// return "", "", fmt.Errorf("unhandled credential type") -// } -// -// return "", "", fmt.Errorf("unhandled credential type") -//} - type repoListOptions interface { getListOptions() *github.ListOptions } From 52e0d41a8b50551d9c295699b783b86f565f31c6 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 13:14:23 -0400 Subject: [PATCH 25/48] wrap some errors --- pkg/sources/github/connector_app.go | 3 ++- pkg/sources/github/connector_token.go | 3 ++- pkg/sources/github/repo.go | 11 ++--------- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 08ad0426ac27..d143eb568692 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -1,6 +1,7 @@ package github import ( + "fmt" "net/http" "strconv" "strings" @@ -89,7 +90,7 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogi c.installationID, &github.InstallationTokenOptions{}) if err != nil { - return "", nil, err + return "", nil, fmt.Errorf("could not create installation token: %w", err) } return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token") diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index ada1b1880a4b..551f2a38c2a5 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -1,6 +1,7 @@ package github import ( + "fmt" "net/http" "strings" "sync" @@ -81,7 +82,7 @@ func (c *tokenConnector) getUser(ctx context.Context) (string, error) { continue } if err != nil { - return "", err + return "", fmt.Errorf("could not get GitHub user: %w", err) } break } diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index 4d203985b7a1..a747dd05f075 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -50,15 +50,8 @@ type repoInfo struct { visibility source_metadatapb.Visibility } -func (s *Source) cloneRepo( - ctx context.Context, - repoURL string, -) (string, *gogit.Repository, error) { - path, repo, err := s.connector.Clone(ctx, repoURL) - if err != nil { - return "", nil, err - } - return path, repo, nil +func (s *Source) cloneRepo(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { + return s.connector.Clone(ctx, repoURL) } type repoListOptions interface { From ba14603fbca1cf193f12a26898fdb24b7ce0e95b Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 13:15:12 -0400 Subject: [PATCH 26/48] move cloudEndpoint --- pkg/sources/github/connector.go | 2 ++ pkg/sources/github/github.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 0c134ebbda0a..f05d6faebf79 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -10,6 +10,8 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb" ) +const cloudEndpoint = "https://api.github.com" + type connector interface { ApiClient() *github.Client Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index f9f5c99dcd20..40cbe552543f 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -290,8 +290,6 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) source_metada return repoInfo.visibility } -const cloudEndpoint = "https://api.github.com" - // Chunks emits chunks of bytes over a channel. func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, targets ...sources.ChunkingTarget) error { // If targets are provided, we're only scanning the data in those targets. From 909505df60bef744a6e0c7f60fd5004f447d807d Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 14:45:19 -0400 Subject: [PATCH 27/48] add error check to test --- pkg/sources/github/github_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 73bd23e4f94c..49a72eb457fd 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -738,7 +738,7 @@ func TestSource_paginateGists(t *testing.T) { } chunksCh := make(chan *sources.Chunk, 5) go func() { - s.addUserGistsToCache(ctx, tt.user) + assert.NoError(t, s.addUserGistsToCache(ctx, tt.user)) chunksCh <- &sources.Chunk{} }() var wantedRepo string From 6d9ca1dbd50b4981dea9b559722bdd2eb2cc875d Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 14:50:47 -0400 Subject: [PATCH 28/48] update+enable gist tests --- pkg/sources/github/github_integration_test.go | 80 +++---------------- 1 file changed, 13 insertions(+), 67 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 49a72eb457fd..64a65acec32e 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -617,19 +617,6 @@ func TestSource_paginateGists(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - secret, err := common.GetTestSecret(ctx) - if err != nil { - t.Fatal(fmt.Errorf("failed to access secret: %v", err)) - } - // For the NEW github app test (+Member enum) - githubPrivateKeyB64New := secret.MustGetField("GITHUB_PRIVATE_KEY_NEW") - githubPrivateKeyBytesNew, err := base64.StdEncoding.DecodeString(githubPrivateKeyB64New) - if err != nil { - t.Fatal(err) - } - githubPrivateKeyNew := string(githubPrivateKeyBytesNew) - githubInstallationIDNew := secret.MustGetField("GITHUB_INSTALLATION_ID_NEW") - githubAppIDNew := secret.MustGetField("GITHUB_APP_ID_NEW") type init struct { name string verify bool @@ -648,13 +635,7 @@ func TestSource_paginateGists(t *testing.T) { init: init{ name: "test source", connection: &sourcespb.GitHub{ - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, + Credential: &sourcespb.GitHub_Unauthenticated{}, }, }, wantChunk: &sources.Chunk{ @@ -672,54 +653,19 @@ func TestSource_paginateGists(t *testing.T) { user: "truffle-sandbox", minRepos: 1, }, - // { - // name: "get multiple pages of gists", - // init: init{ - // name: "test source", - // connection: &sourcespb.GitHub{ - // Credential: &sourcespb.GitHub_GithubApp{ - // GithubApp: &credentialspb.GitHubApp{ - // PrivateKey: githubPrivateKeyNew, - // InstallationId: githubInstallationIDNew, - // AppId: githubAppIDNew, - // }, - // }, - // }, - // }, - // wantChunk: nil, - // wantErr: false, - // user: "andrew", - // minRepos: 101, - // }, - /* { - name: "get multiple pages of gists", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKeyNew, - InstallationId: githubInstallationIDNew, - AppId: githubAppIDNew, - }, - }, - }, - }, - wantChunk: &sources.Chunk{ - SourceName: "test source", - SourceMetadata: &source_metadatapb.MetaData{ - Data: &source_metadatapb.MetaData_Github{ - Github: &source_metadatapb.Github{ - Repository: "https://gist.github.com/872df3b78b9ec3e7dbe597fb5a202121.git", - }, - }, - }, - Verify: false, - }, - wantErr: false, - user: "andrew", + { + name: "get multiple pages of gists", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Credential: &sourcespb.GitHub_Unauthenticated{}, }, - */ + }, + wantChunk: nil, + wantErr: false, + user: "andrew", + minRepos: 101, + }, } for _, tt := range tests { From bf7edc891695cecf3c19e79eabb8f3dc64d030b1 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Wed, 31 Jul 2024 14:58:21 -0400 Subject: [PATCH 29/48] stop pretending to auth in more places --- pkg/sources/github/github_integration_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 64a65acec32e..64f0142edf4f 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -824,7 +824,7 @@ func TestSource_Chunks_TargetedScan(t *testing.T) { name: "targeted scan, one file in med commit", init: init{ name: "test source", - connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Token{Token: githubToken}}, + connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}, queryCriteria: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ @@ -842,7 +842,7 @@ func TestSource_Chunks_TargetedScan(t *testing.T) { name: "no file in commit", init: init{ name: "test source", - connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Token{Token: githubToken}}, + connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}, queryCriteria: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ @@ -861,7 +861,7 @@ func TestSource_Chunks_TargetedScan(t *testing.T) { name: "invalid query criteria, malformed link", init: init{ name: "test source", - connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Token{Token: githubToken}}, + connection: &sourcespb.GitHub{Credential: &sourcespb.GitHub_Unauthenticated{}}, queryCriteria: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ From de3847b6a4325385a0167755e0e535691a64e592 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 10:37:34 -0400 Subject: [PATCH 30/48] update targeted scan test --- pkg/sources/github/github_integration_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 64f0142edf4f..f9e532c00c8d 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -810,10 +810,10 @@ func TestSource_Chunks_TargetedScan(t *testing.T) { queryCriteria: &source_metadatapb.MetaData{ Data: &source_metadatapb.MetaData_Github{ Github: &source_metadatapb.Github{ - Repository: "test_keys_nonpublic", - Link: "https://github.com/trufflesecurity/test_keys_nonpublic/blob/fbc14303ffbf8fb1c2c1914e8dda7d0121633aca/keys#L4", - Commit: "fbc14303ffbf8fb1c2c1914e8dda7d0121633aca", - File: "keys", + Repository: "test-secrets", + Link: "https://github.com/truffle-sandbox/test-secrets/blob/0416560b1330d8ac42045813251d85c688717eaf/new_key#L2", + Commit: "0416560b1330d8ac42045813251d85c688717eaf", + File: "new_key", }, }, }, From 5ef11fe5e4b11386ef42b8a02463b03aa2b493f6 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 10:41:29 -0400 Subject: [PATCH 31/48] update disabled tests --- pkg/sources/github/github_integration_test.go | 52 ++++++------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index f9e532c00c8d..c6b2813df7b5 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -397,42 +397,24 @@ func TestSource_Scan(t *testing.T) { wantChunk: nil, wantErr: false, minRepo: 1, - minOrg: 0, + minOrg: 1, + }, + { + name: "token authenticated, username in org", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Organizations: []string{"truffle-sandbox"}, + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, + }, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 2, + minOrg: 1, }, - // { - // name: "token authenticated, username in org", - // init: init{ - // name: "test source", - // connection: &sourcespb.GitHub{ - // Organizations: []string{"truffle-sandbox"}, - // Credential: &sourcespb.GitHub_Token{ - // Token: githubToken, - // }, - // }, - // }, - // wantChunk: nil, - // wantErr: false, - // minRepo: 0, // I think enumerating users with the org API does not work for newer users! Or maybe just newer users with a `-` in their name? - // // See also: https://github.com/trufflesecurity/trufflehog/issues/874 - // minOrg: 0, - // }, - // { - // name: "token authenticated, org in repo", - // // I do not think that this is a supported case, but adding the test to specify there is no requirement. - // init: init{ - // name: "test source", - // connection: &sourcespb.GitHub{ - // Repositories: []string{"truffle-test-integration-org"}, - // Credential: &sourcespb.GitHub_Token{ - // Token: githubToken, - // }, - // }, - // }, - // wantChunk: nil, - // wantErr: false, - // minRepo: 0, - // minOrg: 0, - // }, /* { name: "token authenticated, no org or user (enum)", From 585774ecf42d5819c3fc46c2f4dd43e01f06f00e Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 11:01:28 -0400 Subject: [PATCH 32/48] enable another test --- pkg/sources/github/github_integration_test.go | 29 ++++++++++--------- pkg/sources/github/repo.go | 1 + 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index c6b2813df7b5..bb2c886fefe7 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -453,20 +453,21 @@ func TestSource_Scan(t *testing.T) { minOrg: 0, }, */ - // { - // name: "unauthenticated, single org", - // init: init{ - // name: "test source", - // connection: &sourcespb.GitHub{ - // Organizations: []string{"trufflesecurity"}, - // Credential: &sourcespb.GitHub_Unauthenticated{}, - // }, - // }, - // wantChunk: nil, - // wantErr: false, - // minRepo: 3, - // minOrg: 1, - // }, + { + name: "unauthenticated, single org", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Organizations: []string{"trufflesecurity"}, + Credential: &sourcespb.GitHub_Unauthenticated{}, + IncludeForks: true, + }, + }, + wantChunk: nil, + wantErr: false, + minRepo: 40, + minOrg: 1, + }, // { // name: "unauthenticated, single repo", // init: init{ diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index a747dd05f075..cec89c1d48d2 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -115,6 +115,7 @@ func (o *orgListOptions) getListOptions() *github.ListOptions { } func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { + // TODO: It's possible to exclude forks when making the API request rather than doing post-request filtering return s.connector.ApiClient().Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions) } From ed047ee47c801f71ac4d62fd5514e5caceb8bd6c Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 11:02:57 -0400 Subject: [PATCH 33/48] enable another test --- pkg/sources/github/github_integration_test.go | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index bb2c886fefe7..310298c3df44 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -468,29 +468,29 @@ func TestSource_Scan(t *testing.T) { minRepo: 40, minOrg: 1, }, - // { - // name: "unauthenticated, single repo", - // init: init{ - // name: "test source", - // connection: &sourcespb.GitHub{ - // Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"}, - // Credential: &sourcespb.GitHub_Unauthenticated{}, - // }, - // }, - // wantChunk: &sources.Chunk{ - // SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, - // SourceName: "test source", - // SourceMetadata: &source_metadatapb.MetaData{ - // Data: &source_metadatapb.MetaData_Github{ - // Github: &source_metadatapb.Github{ - // Repository: "https://github.com/trufflesecurity/driftwood.git", - // }, - // }, - // }, - // Verify: false, - // }, - // wantErr: false, - // }, + { + name: "unauthenticated, single repo", + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Repositories: []string{"https://github.com/trufflesecurity/driftwood.git"}, + Credential: &sourcespb.GitHub_Unauthenticated{}, + }, + }, + wantChunk: &sources.Chunk{ + SourceType: sourcespb.SourceType_SOURCE_TYPE_GITHUB, + SourceName: "test source", + SourceMetadata: &source_metadatapb.MetaData{ + Data: &source_metadatapb.MetaData_Github{ + Github: &source_metadatapb.Github{ + Repository: "https://github.com/trufflesecurity/driftwood.git", + }, + }, + }, + Verify: false, + }, + wantErr: false, + }, { name: "app authenticated, no repo or org", init: init{ From 64831d821d9668e3740d580c7f0912b8652b5dbd Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 11:04:48 -0400 Subject: [PATCH 34/48] enable another test --- pkg/sources/github/github_integration_test.go | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 310298c3df44..68a5857ddccf 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -415,44 +415,42 @@ func TestSource_Scan(t *testing.T) { minRepo: 2, minOrg: 1, }, - /* - { - name: "token authenticated, no org or user (enum)", - // This configuration currently will only find gists from the user. No repos or orgs will be scanned. - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - Credential: &sourcespb.GitHub_Token{ - Token: githubToken, - }, - }, - }, - wantChunk: nil, - wantErr: false, - minRepo: 0, - minOrg: 0, - }, - { - name: "app authenticated (old), no repo or org (enum)", - init: init{ - name: "test source", - connection: &sourcespb.GitHub{ - ScanUsers: false, - Credential: &sourcespb.GitHub_GithubApp{ - GithubApp: &credentialspb.GitHubApp{ - PrivateKey: githubPrivateKey, - InstallationId: githubInstallationID, - AppId: githubAppID, - }, - }, + { + name: "token authenticated, no org or user (enum)", + // This configuration currently will only find gists from the user. No repos or orgs will be scanned. + init: init{ + name: "test source", + connection: &sourcespb.GitHub{ + Credential: &sourcespb.GitHub_Token{ + Token: githubToken, }, }, - wantChunk: nil, - wantErr: false, - minRepo: 3, - minOrg: 0, }, - */ + wantChunk: nil, + wantErr: false, + minRepo: 2, + minOrg: 0, + }, + //{ + // name: "app authenticated (old), no repo or org (enum)", + // init: init{ + // name: "test source", + // connection: &sourcespb.GitHub{ + // ScanUsers: false, + // Credential: &sourcespb.GitHub_GithubApp{ + // GithubApp: &credentialspb.GitHubApp{ + // PrivateKey: githubPrivateKey, + // InstallationId: githubInstallationID, + // AppId: githubAppID, + // }, + // }, + // }, + // }, + // wantChunk: nil, + // wantErr: false, + // minRepo: 3, + // minOrg: 0, + //}, { name: "unauthenticated, single org", init: init{ From c9f26f0de77393e5e86c150101b15b6d00897962 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 11:09:26 -0400 Subject: [PATCH 35/48] update test --- pkg/sources/github/github_integration_test.go | 22 +------------------ 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 68a5857ddccf..a5ea579fb312 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -431,26 +431,6 @@ func TestSource_Scan(t *testing.T) { minRepo: 2, minOrg: 0, }, - //{ - // name: "app authenticated (old), no repo or org (enum)", - // init: init{ - // name: "test source", - // connection: &sourcespb.GitHub{ - // ScanUsers: false, - // Credential: &sourcespb.GitHub_GithubApp{ - // GithubApp: &credentialspb.GitHubApp{ - // PrivateKey: githubPrivateKey, - // InstallationId: githubInstallationID, - // AppId: githubAppID, - // }, - // }, - // }, - // }, - // wantChunk: nil, - // wantErr: false, - // minRepo: 3, - // minOrg: 0, - //}, { name: "unauthenticated, single org", init: init{ @@ -506,7 +486,7 @@ func TestSource_Scan(t *testing.T) { }, wantChunk: nil, wantErr: false, - minRepo: 1, + minRepo: 32, minOrg: 0, }, { From dfa8a10528aa4871a20f42950e571f26edfc89a9 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 11:14:50 -0400 Subject: [PATCH 36/48] delete dead commented code --- pkg/sources/github/github_integration_test.go | 29 ------------------- 1 file changed, 29 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index a5ea579fb312..85b7318f860f 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -711,35 +711,6 @@ func githubCommentCheckFunc(gotChunk, wantChunk *sources.Chunk, i int, t *testin } } -// func TestSource_paginateRepos(t *testing.T) { -// type args struct { -// ctx context.Context -// apiClient *github.Client -// } -// tests := []struct { -// name string -// org string -// args args -// }{ -// { -// org: "fakeNetflix", -// args: args{ -// ctx: context.Background(), -// apiClient: github.NewClient(common.SaneHttpClient()), -// }, -// }, -// } -// for _, tt := range tests { -// t.Run(tt.name, func(t *testing.T) { -// s := &Source{httpClient: common.SaneHttpClient()} -// s.paginateRepos(tt.args.ctx, tt.args.apiClient, tt.org) -// if len(s.repos) < 101 { -// t.Errorf("expected > 100 repos, got %d", len(s.repos)) -// } -// }) -// } -// } - func TestSource_Chunks_TargetedScan(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*3000) defer cancel() From 7a344c7e57dc60891c8d6f350eea44431d07c33b Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 11:22:07 -0400 Subject: [PATCH 37/48] delete obsolete test code --- pkg/sources/github/github_integration_test.go | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index 85b7318f860f..a2f171eca8c9 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -68,18 +68,6 @@ func TestSource_Token(t *testing.T) { err = s.enumerateWithApp(ctx) assert.NoError(t, err) - //user, token, err := s.userAndToken(ctx, installationClient) - //assert.NotEmpty(t, token) - //assert.NoError(t, err) - // - //// user provided - //_, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", user) - //assert.NoError(t, err) - // - //// no user provided - //_, _, err = git.CloneRepoUsingToken(ctx, token, "https://github.com/truffle-test-integration-org/another-test-repo.git", "") - //assert.Error(t, err) - _, _, err = s.cloneRepo(ctx, "https://github.com/truffle-test-integration-org/another-test-repo.git") assert.NoError(t, err) } From 366446ae770bd69571ec4337431079ca3cf2b870 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 11:56:28 -0400 Subject: [PATCH 38/48] self-review --- pkg/sources/github/connector_app.go | 12 ++++++------ pkg/sources/github/connector_basicauth.go | 3 ++- pkg/sources/github/connector_token.go | 2 +- pkg/sources/github/connector_unauthenticated.go | 4 ++-- pkg/sources/github/github.go | 4 +++- pkg/sources/github/github_test.go | 2 -- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index d143eb568692..37bda49a21bc 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -28,12 +28,12 @@ var _ connector = (*appConnector)(nil) func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConnector, error) { installationID, err := strconv.ParseInt(app.InstallationId, 10, 64) if err != nil { - return nil, err + return nil, fmt.Errorf("could not parse app installation ID %q: %w", app.InstallationId, err) } appID, err := strconv.ParseInt(app.AppId, 10, 64) if err != nil { - return nil, err + return nil, fmt.Errorf("could not parse app ID %q: %w", appID, err) } httpClient := common.RetryableHTTPClientTimeout(60) @@ -43,7 +43,7 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn appID, []byte(app.PrivateKey)) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create installation client transport: %w", err) } installationTransport.BaseURL = apiEndpoint @@ -51,7 +51,7 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn installationHttpClient.Transport = installationTransport installationClient, err := github.NewClient(installationHttpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create installation client: %w", err) } apiTransport, err := ghinstallation.New( @@ -60,14 +60,14 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn installationID, []byte(app.PrivateKey)) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create API client transport: %w", err) } apiTransport.BaseURL = apiEndpoint httpClient.Transport = apiTransport apiClient, err := github.NewClient(httpClient).WithEnterpriseURLs(apiEndpoint, apiEndpoint) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create API client: %w", err) } return &appConnector{ diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 20942778c5c7..28c152e232ca 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -1,6 +1,7 @@ package github import ( + "fmt" "net/http" "strings" @@ -31,7 +32,7 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (* apiClient, err := createGitHubClient(httpClient, apiEndpoint) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create API client: %w", err) } return &basicAuthConnector{ diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 551f2a38c2a5..dfdf3b0d9376 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -36,7 +36,7 @@ func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(er apiClient, err := createGitHubClient(httpClient, apiEndpoint) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create API client: %w", err) } return &tokenConnector{ diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index b4715469444a..770a4dbd37d1 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -1,6 +1,7 @@ package github import ( + "fmt" "net/http" "strings" @@ -20,11 +21,10 @@ type unauthenticatedConnector struct { var _ connector = (*unauthenticatedConnector)(nil) func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { - httpClient := common.RetryableHTTPClientTimeout(60) apiClient, err := createGitHubClient(httpClient, apiEndpoint) if err != nil { - return nil, err + return nil, fmt.Errorf("could not create API client: %w", err) } return &unauthenticatedConnector{ httpClient: httpClient, diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 40cbe552543f..05ab93875519 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -307,13 +307,15 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar err := s.enumerate(ctx) if err != nil { - return err + return fmt.Errorf("error enumerating: %w", err) } return s.scan(ctx, chunksChan) } func (s *Source) enumerate(ctx context.Context) error { + // I'm not wild about switching on the connector type here (as opposed to dispatching to the connector itself) but + // this felt like a compromise that allowed me to isolate connection logic without rewriting the entire source. switch s.connector.(type) { case *appConnector: if err := s.enumerateWithApp(ctx); err != nil { diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 4df5c5e0ae4c..2507346db040 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -44,8 +44,6 @@ func initTestSource(src *sourcespb.GitHub) *Source { if err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1); err != nil { panic(err) } - //s.apiClient = github.NewClient(s.httpClient) - //gock.InterceptClient(s.httpClient) gock.InterceptClient(s.connector.HttpClient()) gock.Intercept() gock.InterceptClient(s.connector.ApiClient().Client()) From e019a1ccec4f35edc20c6b5b53b6821192ef1301 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 12:08:22 -0400 Subject: [PATCH 39/48] clean up regular tests file --- pkg/sources/github/github_test.go | 62 ++++++++++--------------------- 1 file changed, 19 insertions(+), 43 deletions(-) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 2507346db040..87a8f90266d9 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -30,6 +30,23 @@ import ( "github.com/trufflesecurity/trufflehog/v3/pkg/sources" ) +func createPrivateKey() string { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + panic(err) + } + + data := x509.MarshalPKCS1PrivateKey(key) + var pemKey bytes.Buffer + if err := pem.Encode(&pemKey, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: data, + }); err != nil { + panic(err) + } + return pemKey.String() +} + func createTestSource(src *sourcespb.GitHub) (*Source, *anypb.Any) { s := &Source{} conn, err := anypb.New(src) @@ -196,23 +213,7 @@ func TestAddMembersByOrg(t *testing.T) { func TestAddMembersByApp(t *testing.T) { defer gock.Off() - // generate a private key (it just needs to be in the right format) - privateKey := func() string { - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - panic(err) - } - - data := x509.MarshalPKCS1PrivateKey(key) - var pemKey bytes.Buffer - if err := pem.Encode(&pemKey, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: data, - }); err != nil { - panic(err) - } - return pemKey.String() - }() + privateKey := createPrivateKey() gock.New("https://api.github.com"). Get("/app/installations"). @@ -679,23 +680,7 @@ func TestEnumerateWithToken_IncludeRepos(t *testing.T) { func TestEnumerateWithApp(t *testing.T) { defer gock.Off() - // generate a private key (it just needs to be in the right format) - privateKey := func() string { - key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - panic(err) - } - - data := x509.MarshalPKCS1PrivateKey(key) - var pemKey bytes.Buffer - if err := pem.Encode(&pemKey, &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: data, - }); err != nil { - panic(err) - } - return pemKey.String() - }() + privateKey := createPrivateKey() gock.New("https://api.github.com"). Post("/app/installations/1337/access_tokens"). @@ -717,15 +702,6 @@ func TestEnumerateWithApp(t *testing.T) { }, }, }) - //_, err := s.enumerateWithApp( - // context.Background(), - // "https://api.github.com", - // &credentialspb.GitHubApp{ - // InstallationId: "1337", - // AppId: "4141", - // PrivateKey: privateKey, - // }, - //) err := s.enumerateWithApp(context.Background()) assert.Nil(t, err) assert.Equal(t, 0, len(s.repos)) From 872b16c6d5dbadc8e0dc7ea00c6a124b3099f356 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 12:14:40 -0400 Subject: [PATCH 40/48] remove HttpClient from connector --- pkg/sources/github/connector.go | 2 -- pkg/sources/github/connector_app.go | 7 ------- pkg/sources/github/connector_basicauth.go | 7 ------- pkg/sources/github/connector_token.go | 7 ------- pkg/sources/github/connector_unauthenticated.go | 7 ------- pkg/sources/github/github_test.go | 2 -- pkg/sources/github/repo.go | 2 +- 7 files changed, 1 insertion(+), 33 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index f05d6faebf79..a116c49a3828 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -2,7 +2,6 @@ package github import ( "fmt" - "net/http" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v63/github" @@ -15,7 +14,6 @@ const cloudEndpoint = "https://api.github.com" type connector interface { ApiClient() *github.Client Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) - HttpClient() *http.Client IsGithubEnterprise() bool InstallationClient() *github.Client } diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 37bda49a21bc..1fd5d40008a0 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -2,7 +2,6 @@ package github import ( "fmt" - "net/http" "strconv" "strings" @@ -16,7 +15,6 @@ import ( ) type appConnector struct { - httpClient *http.Client apiClient *github.Client installationClient *github.Client installationID int64 @@ -71,7 +69,6 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn } return &appConnector{ - httpClient: httpClient, apiClient: apiClient, installationClient: installationClient, installationID: installationID, @@ -100,10 +97,6 @@ func (c *appConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } -func (c *appConnector) HttpClient() *http.Client { - return c.httpClient -} - func (c *appConnector) InstallationClient() *github.Client { return c.installationClient } diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 28c152e232ca..e548646cd747 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -2,7 +2,6 @@ package github import ( "fmt" - "net/http" "strings" gogit "github.com/go-git/go-git/v5" @@ -14,7 +13,6 @@ import ( ) type basicAuthConnector struct { - httpClient *http.Client apiClient *github.Client username string password string @@ -36,7 +34,6 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (* } return &basicAuthConnector{ - httpClient: httpClient, apiClient: apiClient, username: cred.Username, password: cred.Password, @@ -56,10 +53,6 @@ func (c *basicAuthConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } -func (c *basicAuthConnector) HttpClient() *http.Client { - return c.httpClient -} - func (c *basicAuthConnector) InstallationClient() *github.Client { return nil } diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index dfdf3b0d9376..9d955e763bd7 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -2,7 +2,6 @@ package github import ( "fmt" - "net/http" "strings" "sync" @@ -15,7 +14,6 @@ import ( ) type tokenConnector struct { - httpClient *http.Client apiClient *github.Client token string isGitHubEnterprise bool @@ -40,7 +38,6 @@ func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(er } return &tokenConnector{ - httpClient: httpClient, apiClient: apiClient, token: token, isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), @@ -63,10 +60,6 @@ func (c *tokenConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } -func (c *tokenConnector) HttpClient() *http.Client { - return c.httpClient -} - func (c *tokenConnector) InstallationClient() *github.Client { return nil } diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 770a4dbd37d1..9180073cfc6f 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -2,7 +2,6 @@ package github import ( "fmt" - "net/http" "strings" gogit "github.com/go-git/go-git/v5" @@ -13,7 +12,6 @@ import ( ) type unauthenticatedConnector struct { - httpClient *http.Client apiClient *github.Client isGitHubEnterprise bool } @@ -27,7 +25,6 @@ func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, return nil, fmt.Errorf("could not create API client: %w", err) } return &unauthenticatedConnector{ - httpClient: httpClient, apiClient: apiClient, isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), }, nil @@ -45,10 +42,6 @@ func (c *unauthenticatedConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } -func (c *unauthenticatedConnector) HttpClient() *http.Client { - return c.httpClient -} - func (c *unauthenticatedConnector) InstallationClient() *github.Client { return nil } diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 87a8f90266d9..553d7f779c40 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -61,8 +61,6 @@ func initTestSource(src *sourcespb.GitHub) *Source { if err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1); err != nil { panic(err) } - gock.InterceptClient(s.connector.HttpClient()) - gock.Intercept() gock.InterceptClient(s.connector.ApiClient().Client()) if installationClient := s.connector.InstallationClient(); installationClient != nil { gock.InterceptClient(installationClient.Client()) diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index cec89c1d48d2..5e56df481c96 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -270,7 +270,7 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool { return false } - res, err := s.connector.HttpClient().Do(req) + res, err := s.connector.ApiClient().Client().Do(req) if err != nil { return false } From 0fe6175a5b65181d01151536b678e097400924d4 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 12:22:09 -0400 Subject: [PATCH 41/48] add comments --- pkg/sources/github/connector.go | 5 +++++ pkg/sources/github/connector_app.go | 3 +++ pkg/sources/github/connector_basicauth.go | 3 +++ pkg/sources/github/connector_unauthenticated.go | 3 +++ 4 files changed, 14 insertions(+) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index a116c49a3828..f7142a8fbc07 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -12,9 +12,14 @@ import ( const cloudEndpoint = "https://api.github.com" type connector interface { + // ApiClient returns a configured GitHub client that can be used for GitHub API operations. ApiClient() *github.Client + // Clone clones a repository using the configured authentication information. Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) + // IsGithubEnterprise returns whether the connector is for a GitHub Enterprise endpoint. IsGithubEnterprise() bool + // InstallationClient returns a GitHub client that can be used to get information about all the configured GitHub + // app's installations. If no GitHub app is configured, nil is returned. InstallationClient() *github.Client } diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 1fd5d40008a0..dd87c8224bb5 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -94,6 +94,9 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogi } func (c *appConnector) IsGithubEnterprise() bool { + // At the time of this writing, this method is not called anywhere, so if you start calling it and something looks + // wrong don't assume that this implementation is correct. (It is implemented here because the interface requires + // it, but the only code path that checks for GHE uses a different implementation of this interface.) return c.isGitHubEnterprise } diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index e548646cd747..f80953a9564e 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -50,6 +50,9 @@ func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, } func (c *basicAuthConnector) IsGithubEnterprise() bool { + // At the time of this writing, this method is not called anywhere, so if you start calling it and something looks + // wrong don't assume that this implementation is correct. (It is implemented here because the interface requires + // it, but the only code path that checks for GHE uses a different implementation of this interface.) return c.isGitHubEnterprise } diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 9180073cfc6f..2bc3e8dde1a3 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -39,6 +39,9 @@ func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (s } func (c *unauthenticatedConnector) IsGithubEnterprise() bool { + // At the time of this writing, this method is not called anywhere, so if you start calling it and something looks + // wrong don't assume that this implementation is correct. (It is implemented here because the interface requires + // it, but the only code path that checks for GHE uses a different implementation of this interface.) return c.isGitHubEnterprise } From 7b376bc974592835e406a5a56d7c0bd8fa2139ed Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 12:26:54 -0400 Subject: [PATCH 42/48] remove stuff from interface --- pkg/sources/github/connector.go | 5 ---- pkg/sources/github/connector_app.go | 10 ------- pkg/sources/github/connector_basicauth.go | 26 +++++-------------- pkg/sources/github/connector_token.go | 4 --- .../github/connector_unauthenticated.go | 18 ++----------- pkg/sources/github/github.go | 15 +++++++++-- pkg/sources/github/github_test.go | 6 ++--- 7 files changed, 24 insertions(+), 60 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index f7142a8fbc07..79867147f43b 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -16,11 +16,6 @@ type connector interface { ApiClient() *github.Client // Clone clones a repository using the configured authentication information. Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) - // IsGithubEnterprise returns whether the connector is for a GitHub Enterprise endpoint. - IsGithubEnterprise() bool - // InstallationClient returns a GitHub client that can be used to get information about all the configured GitHub - // app's installations. If no GitHub app is configured, nil is returned. - InstallationClient() *github.Client } func newConnector(source *Source) (connector, error) { diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index dd87c8224bb5..5e864eb1d4eb 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -3,7 +3,6 @@ package github import ( "fmt" "strconv" - "strings" "github.com/bradleyfalzon/ghinstallation/v2" gogit "github.com/go-git/go-git/v5" @@ -18,7 +17,6 @@ type appConnector struct { apiClient *github.Client installationClient *github.Client installationID int64 - isGitHubEnterprise bool } var _ connector = (*appConnector)(nil) @@ -72,7 +70,6 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn apiClient: apiClient, installationClient: installationClient, installationID: installationID, - isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), }, nil } @@ -93,13 +90,6 @@ func (c *appConnector) Clone(ctx context.Context, repoURL string) (string, *gogi return git.CloneRepoUsingToken(ctx, token.GetToken(), repoURL, "x-access-token") } -func (c *appConnector) IsGithubEnterprise() bool { - // At the time of this writing, this method is not called anywhere, so if you start calling it and something looks - // wrong don't assume that this implementation is correct. (It is implemented here because the interface requires - // it, but the only code path that checks for GHE uses a different implementation of this interface.) - return c.isGitHubEnterprise -} - func (c *appConnector) InstallationClient() *github.Client { return c.installationClient } diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index f80953a9564e..ce31e04470da 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -2,7 +2,6 @@ package github import ( "fmt" - "strings" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v63/github" @@ -13,10 +12,9 @@ import ( ) type basicAuthConnector struct { - apiClient *github.Client - username string - password string - isGitHubEnterprise bool + apiClient *github.Client + username string + password string } var _ connector = (*basicAuthConnector)(nil) @@ -34,10 +32,9 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (* } return &basicAuthConnector{ - apiClient: apiClient, - username: cred.Username, - password: cred.Password, - isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), + apiClient: apiClient, + username: cred.Username, + password: cred.Password, }, nil } @@ -48,14 +45,3 @@ func (c *basicAuthConnector) ApiClient() *github.Client { func (c *basicAuthConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { return git.CloneRepoUsingToken(ctx, c.password, repoURL, c.username) } - -func (c *basicAuthConnector) IsGithubEnterprise() bool { - // At the time of this writing, this method is not called anywhere, so if you start calling it and something looks - // wrong don't assume that this implementation is correct. (It is implemented here because the interface requires - // it, but the only code path that checks for GHE uses a different implementation of this interface.) - return c.isGitHubEnterprise -} - -func (c *basicAuthConnector) InstallationClient() *github.Client { - return nil -} diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 9d955e763bd7..00e2ae904351 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -60,10 +60,6 @@ func (c *tokenConnector) IsGithubEnterprise() bool { return c.isGitHubEnterprise } -func (c *tokenConnector) InstallationClient() *github.Client { - return nil -} - func (c *tokenConnector) getUser(ctx context.Context) (string, error) { var ( user *github.User diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 2bc3e8dde1a3..f5f814f1b63a 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -2,7 +2,6 @@ package github import ( "fmt" - "strings" gogit "github.com/go-git/go-git/v5" "github.com/google/go-github/v63/github" @@ -12,8 +11,7 @@ import ( ) type unauthenticatedConnector struct { - apiClient *github.Client - isGitHubEnterprise bool + apiClient *github.Client } var _ connector = (*unauthenticatedConnector)(nil) @@ -25,8 +23,7 @@ func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, return nil, fmt.Errorf("could not create API client: %w", err) } return &unauthenticatedConnector{ - apiClient: apiClient, - isGitHubEnterprise: !strings.EqualFold(apiEndpoint, cloudEndpoint), + apiClient: apiClient, }, nil } @@ -37,14 +34,3 @@ func (c *unauthenticatedConnector) ApiClient() *github.Client { func (c *unauthenticatedConnector) Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) { return git.CloneRepoUsingUnauthenticated(ctx, repoURL) } - -func (c *unauthenticatedConnector) IsGithubEnterprise() bool { - // At the time of this writing, this method is not called anywhere, so if you start calling it and something looks - // wrong don't assume that this implementation is correct. (It is implemented here because the interface requires - // it, but the only code path that checks for GHE uses a different implementation of this interface.) - return c.isGitHubEnterprise -} - -func (c *unauthenticatedConnector) InstallationClient() *github.Client { - return nil -} diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 05ab93875519..0918a671656c 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -435,6 +435,12 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context) { func (s *Source) enumerateWithToken(ctx context.Context) error { ctx.Logger().V(1).Info("Enumerating with token") + + connector, ok := s.connector.(*tokenConnector) + if !ok { + return fmt.Errorf("cannot enumerate repositories using a token because no token is configured") + } + var ghUser *github.User var err error for { @@ -458,7 +464,7 @@ func (s *Source) enumerateWithToken(ctx context.Context) error { s.log.Error(err, "Unable to fetch gists for the current user", "user", ghUser.GetLogin()) } - if s.connector.IsGithubEnterprise() { + if connector.IsGithubEnterprise() { s.addAllVisibleOrgs(ctx) } else { // Scan for orgs is default with a token. @@ -493,6 +499,11 @@ func (s *Source) enumerateWithToken(ctx context.Context) error { } func (s *Source) enumerateWithApp(ctx context.Context) error { + connector, ok := s.connector.(*appConnector) + if !ok { + return fmt.Errorf("cannot enumerate app repositories because no app is configured") + } + // If no repos were provided, enumerate them. if len(s.repos) == 0 { if err := s.getReposByApp(ctx); err != nil { @@ -501,7 +512,7 @@ func (s *Source) enumerateWithApp(ctx context.Context) error { // Check if we need to find user repos. if s.conn.ScanUsers { - err := s.addMembersByApp(ctx, s.connector.InstallationClient()) + err := s.addMembersByApp(ctx, connector.InstallationClient()) if err != nil { return err } diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 553d7f779c40..419a84064f63 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -62,8 +62,8 @@ func initTestSource(src *sourcespb.GitHub) *Source { panic(err) } gock.InterceptClient(s.connector.ApiClient().Client()) - if installationClient := s.connector.InstallationClient(); installationClient != nil { - gock.InterceptClient(installationClient.Client()) + if appConnector, ok := s.connector.(*appConnector); ok { + gock.InterceptClient(appConnector.InstallationClient().Client()) } return s } @@ -241,7 +241,7 @@ func TestAddMembersByApp(t *testing.T) { AppId: "4141", }, }}) - err := s.addMembersByApp(context.Background(), s.connector.InstallationClient()) + err := s.addMembersByApp(context.Background(), s.connector.(*appConnector).InstallationClient()) assert.Nil(t, err) assert.Equal(t, 3, len(s.memberCache)) _, ok := s.memberCache["ssm1"] From c7a14acdc21e285746d1012d9a45ce949dd6ae61 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 12:45:43 -0400 Subject: [PATCH 43/48] remove some cruft --- pkg/sources/github/github_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 419a84064f63..54f8efa172e1 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -100,7 +100,6 @@ func TestAddReposByOrg(t *testing.T) { Repositories: nil, IgnoreRepos: []string{"secret/super-*-repo2"}, }) - // gock works here because github.NewClient is using the default HTTP Transport err := s.getReposByOrg(context.Background(), "super-secret-org") assert.Nil(t, err) assert.Equal(t, 1, s.filteredRepoCache.Count()) @@ -129,7 +128,6 @@ func TestAddReposByOrg_IncludeRepos(t *testing.T) { IncludeRepos: []string{"super-secret-org/super*"}, Organizations: []string{"super-secret-org"}, }) - // gock works here because github.NewClient is using the default HTTP Transport err := s.getReposByOrg(context.Background(), "super-secret-org") assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) @@ -645,7 +643,6 @@ func BenchmarkEnumerate(b *testing.B) { setupMocks(b) b.StartTimer() - //_, _ = s.enumerate(context.Background(), "https://api.github.com") _ = s.enumerate(context.Background()) } } @@ -666,7 +663,6 @@ func TestEnumerateWithToken_IncludeRepos(t *testing.T) { }) s.repos = []string{"some-special-repo"} - //err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token") err := s.enumerateWithToken(context.Background()) assert.Nil(t, err) assert.Equal(t, 1, len(s.repos)) From 778fa1abba3488c3ff62ae52260bdc38b47f4fdd Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 12:46:24 -0400 Subject: [PATCH 44/48] remove more cruft --- pkg/sources/github/github_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 54f8efa172e1..23ecd15d7d27 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -458,7 +458,6 @@ func TestEnumerateWithToken(t *testing.T) { Token: "token", }, }) - //err := s.enumerateWithToken(context.Background(), "https://api.github.com", "token") err := s.enumerateWithToken(context.Background()) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) From 33b982ea620c0cc551717398a65957d715f59fc9 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Thu, 1 Aug 2024 12:49:22 -0400 Subject: [PATCH 45/48] remove even more cruft --- pkg/sources/github/github_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 23ecd15d7d27..61474c9bd41e 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -502,7 +502,6 @@ func BenchmarkEnumerateWithToken(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - //_ = s.enumerateWithToken(context.Background(), "https://api.github.com", "token") _ = s.enumerateWithToken(context.Background()) } } @@ -568,7 +567,6 @@ func TestEnumerate(t *testing.T) { s.filteredRepoCache.Set(repo.GetFullName(), repo.GetCloneURL()) // Act - //_, err := s.enumerate(context.Background(), "https://api.github.com") err := s.enumerate(context.Background()) // Assert From 2eda83e8801394a3dfcc49450ea096b824ff7cc6 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Fri, 2 Aug 2024 09:05:45 -0400 Subject: [PATCH 46/48] review feedback --- pkg/sources/github/connector.go | 6 ++--- pkg/sources/github/connector_app.go | 2 +- pkg/sources/github/connector_basicauth.go | 2 +- pkg/sources/github/connector_token.go | 16 +++++++----- .../github/connector_unauthenticated.go | 2 +- pkg/sources/github/github.go | 26 +++++++++---------- pkg/sources/github/github_test.go | 2 +- pkg/sources/github/repo.go | 10 +++---- 8 files changed, 34 insertions(+), 32 deletions(-) diff --git a/pkg/sources/github/connector.go b/pkg/sources/github/connector.go index 79867147f43b..707c2fa7f062 100644 --- a/pkg/sources/github/connector.go +++ b/pkg/sources/github/connector.go @@ -12,15 +12,15 @@ import ( const cloudEndpoint = "https://api.github.com" type connector interface { - // ApiClient returns a configured GitHub client that can be used for GitHub API operations. - ApiClient() *github.Client + // APIClient returns a configured GitHub client that can be used for GitHub API operations. + APIClient() *github.Client // Clone clones a repository using the configured authentication information. Clone(ctx context.Context, repoURL string) (string, *gogit.Repository, error) } func newConnector(source *Source) (connector, error) { apiEndpoint := source.conn.Endpoint - if len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) { + if apiEndpoint == "" || endsWithGithub.MatchString(apiEndpoint) { apiEndpoint = cloudEndpoint } diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 5e864eb1d4eb..209935fe5731 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -73,7 +73,7 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn }, nil } -func (c *appConnector) ApiClient() *github.Client { +func (c *appConnector) APIClient() *github.Client { return c.apiClient } diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index ce31e04470da..6d3a6d36c1d8 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -38,7 +38,7 @@ func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (* }, nil } -func (c *basicAuthConnector) ApiClient() *github.Client { +func (c *basicAuthConnector) APIClient() *github.Client { return c.apiClient } diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index 00e2ae904351..d0d1d373a8c8 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -45,7 +45,7 @@ func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(er }, nil } -func (c *tokenConnector) ApiClient() *github.Client { +func (c *tokenConnector) APIClient() *github.Client { return c.apiClient } @@ -82,13 +82,15 @@ func (c *tokenConnector) setUserIfUnset(ctx context.Context) error { c.userMu.Lock() defer c.userMu.Unlock() - if c.user == "" { - if user, err := c.getUser(ctx); err != nil { - return err - } else { - c.user = user - } + if c.user != "" { + return nil + } + + user, err := c.getUser(ctx) + if err != nil { + return err } + c.user = user return nil } diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index f5f814f1b63a..3a902be66878 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -27,7 +27,7 @@ func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, }, nil } -func (c *unauthenticatedConnector) ApiClient() *github.Client { +func (c *unauthenticatedConnector) APIClient() *github.Client { return c.apiClient } diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 0918a671656c..00f4e39bd8a5 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -265,7 +265,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobID sources.JobID, so // Validate is used by enterprise CLI to validate the GitHub config file. func (s *Source) Validate(ctx context.Context) []error { - if _, _, err := s.connector.ApiClient().Users.Get(ctx, ""); err != nil { + if _, _, err := s.connector.APIClient().Users.Get(ctx, ""); err != nil { return []error{err} } @@ -355,7 +355,7 @@ RepoLoop: // Cache gist info. for { gistID := extractGistID(urlParts) - gist, _, err := s.connector.ApiClient().Gists.Get(repoCtx, gistID) + gist, _, err := s.connector.APIClient().Gists.Get(repoCtx, gistID) // Normalize the URL to the Gist's pull URL. // See https://github.com/trufflesecurity/trufflehog/pull/2625#issuecomment-2025507937 repo = gist.GetGitPullURL() @@ -372,7 +372,7 @@ RepoLoop: } else { // Cache repository info. for { - ghRepo, _, err := s.connector.ApiClient().Repositories.Get(repoCtx, urlParts[1], urlParts[2]) + ghRepo, _, err := s.connector.APIClient().Repositories.Get(repoCtx, urlParts[1], urlParts[2]) if s.handleRateLimit(err) { continue } @@ -444,7 +444,7 @@ func (s *Source) enumerateWithToken(ctx context.Context) error { var ghUser *github.User var err error for { - ghUser, _, err = s.connector.ApiClient().Users.Get(ctx, "") + ghUser, _, err = s.connector.APIClient().Users.Get(ctx, "") if s.handleRateLimit(err) { continue } @@ -751,7 +751,7 @@ func (s *Source) addUserGistsToCache(ctx context.Context, user string) error { gistOpts := &github.GistListOptions{} logger := s.log.WithValues("user", user) for { - gists, res, err := s.connector.ApiClient().Gists.List(ctx, user, gistOpts) + gists, res, err := s.connector.APIClient().Gists.List(ctx, user, gistOpts) if s.handleRateLimit(err) { continue } @@ -808,7 +808,7 @@ func (s *Source) addAllVisibleOrgs(ctx context.Context) { }, } for { - orgs, _, err := s.connector.ApiClient().Organizations.ListAll(ctx, orgOpts) + orgs, _, err := s.connector.APIClient().Organizations.ListAll(ctx, orgOpts) if s.handleRateLimit(err) { continue } @@ -847,7 +847,7 @@ func (s *Source) addOrgsByUser(ctx context.Context, user string) { } logger := s.log.WithValues("user", user) for { - orgs, resp, err := s.connector.ApiClient().Organizations.List(ctx, "", orgOpts) + orgs, resp, err := s.connector.APIClient().Organizations.List(ctx, "", orgOpts) if s.handleRateLimit(err) { continue } @@ -880,7 +880,7 @@ func (s *Source) addMembersByOrg(ctx context.Context, org string) error { logger := s.log.WithValues("org", org) for { - members, res, err := s.connector.ApiClient().Organizations.ListMembers(ctx, org, opts) + members, res, err := s.connector.APIClient().Organizations.ListMembers(ctx, org, opts) if s.handleRateLimit(err) { continue } @@ -1002,7 +1002,7 @@ func (s *Source) processGistComments(ctx context.Context, gistURL string, urlPar Page: initialPage, } for { - comments, _, err := s.connector.ApiClient().Gists.ListComments(ctx, gistID, options) + comments, _, err := s.connector.APIClient().Gists.ListComments(ctx, gistID, options) if s.handleRateLimit(err) { continue } @@ -1115,7 +1115,7 @@ func (s *Source) processIssues(ctx context.Context, repoInfo repoInfo, chunksCha } for { - issues, _, err := s.connector.ApiClient().Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts) + issues, _, err := s.connector.APIClient().Issues.ListByRepo(ctx, repoInfo.owner, repoInfo.name, bodyTextsOpts) if s.handleRateLimit(err) { continue } @@ -1187,7 +1187,7 @@ func (s *Source) processIssueComments(ctx context.Context, repoInfo repoInfo, ch } for { - issueComments, _, err := s.connector.ApiClient().Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts) + issueComments, _, err := s.connector.APIClient().Issues.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, issueOpts) if s.handleRateLimit(err) { continue } @@ -1252,7 +1252,7 @@ func (s *Source) processPRs(ctx context.Context, repoInfo repoInfo, chunksChan c } for { - prs, _, err := s.connector.ApiClient().PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts) + prs, _, err := s.connector.APIClient().PullRequests.List(ctx, repoInfo.owner, repoInfo.name, prOpts) if s.handleRateLimit(err) { continue } @@ -1284,7 +1284,7 @@ func (s *Source) processPRComments(ctx context.Context, repoInfo repoInfo, chunk } for { - prComments, _, err := s.connector.ApiClient().PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts) + prComments, _, err := s.connector.APIClient().PullRequests.ListComments(ctx, repoInfo.owner, repoInfo.name, allComments, prOpts) if s.handleRateLimit(err) { continue } diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index 61474c9bd41e..df89ec41aa33 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -61,7 +61,7 @@ func initTestSource(src *sourcespb.GitHub) *Source { if err := s.Init(context.Background(), "test - github", 0, 1337, false, conn, 1); err != nil { panic(err) } - gock.InterceptClient(s.connector.ApiClient().Client()) + gock.InterceptClient(s.connector.APIClient().Client()) if appConnector, ok := s.connector.(*appConnector); ok { gock.InterceptClient(appConnector.InstallationClient().Client()) } diff --git a/pkg/sources/github/repo.go b/pkg/sources/github/repo.go index 5e56df481c96..67e7728816c7 100644 --- a/pkg/sources/github/repo.go +++ b/pkg/sources/github/repo.go @@ -69,7 +69,7 @@ func (a *appListOptions) getListOptions() *github.ListOptions { } func (s *Source) appListReposWrapper(ctx context.Context, _ string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { - someRepos, res, err := s.connector.ApiClient().Apps.ListRepos(ctx, opts.getListOptions()) + someRepos, res, err := s.connector.APIClient().Apps.ListRepos(ctx, opts.getListOptions()) if someRepos != nil { return someRepos.Repositories, res, err } @@ -93,7 +93,7 @@ func (u *userListOptions) getListOptions() *github.ListOptions { } func (s *Source) userListReposWrapper(ctx context.Context, user string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { - return s.connector.ApiClient().Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions) + return s.connector.APIClient().Repositories.ListByUser(ctx, user, &opts.(*userListOptions).RepositoryListByUserOptions) } func (s *Source) getReposByUser(ctx context.Context, user string) error { @@ -116,7 +116,7 @@ func (o *orgListOptions) getListOptions() *github.ListOptions { func (s *Source) orgListReposWrapper(ctx context.Context, org string, opts repoListOptions) ([]*github.Repository, *github.Response, error) { // TODO: It's possible to exclude forks when making the API request rather than doing post-request filtering - return s.connector.ApiClient().Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions) + return s.connector.APIClient().Repositories.ListByOrg(ctx, org, &opts.(*orgListOptions).RepositoryListByOrgOptions) } func (s *Source) getReposByOrg(ctx context.Context, org string) error { @@ -270,7 +270,7 @@ func (s *Source) wikiIsReachable(ctx context.Context, repoURL string) bool { return false } - res, err := s.connector.ApiClient().Client().Do(req) + res, err := s.connector.APIClient().Client().Do(req) if err != nil { return false } @@ -297,7 +297,7 @@ func (s *Source) getDiffForFileInCommit(ctx context.Context, query commitQuery) err error ) for { - commit, _, err = s.connector.ApiClient().Repositories.GetCommit(ctx, query.owner, query.repo, query.sha, nil) + commit, _, err = s.connector.APIClient().Repositories.GetCommit(ctx, query.owner, query.repo, query.sha, nil) if s.handleRateLimit(err) { continue } From 5b8d602b9c06e035f0256347d6628b1845fbcc44 Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Fri, 2 Aug 2024 09:06:59 -0400 Subject: [PATCH 47/48] use consts for http timeout --- pkg/sources/github/connector_app.go | 3 ++- pkg/sources/github/connector_basicauth.go | 3 ++- pkg/sources/github/connector_token.go | 3 ++- pkg/sources/github/connector_unauthenticated.go | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pkg/sources/github/connector_app.go b/pkg/sources/github/connector_app.go index 209935fe5731..925c7748fce9 100644 --- a/pkg/sources/github/connector_app.go +++ b/pkg/sources/github/connector_app.go @@ -32,7 +32,8 @@ func newAppConnector(apiEndpoint string, app *credentialspb.GitHubApp) (*appConn return nil, fmt.Errorf("could not parse app ID %q: %w", appID, err) } - httpClient := common.RetryableHTTPClientTimeout(60) + const httpTimeoutSeconds = 60 + httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) installationTransport, err := ghinstallation.NewAppsTransport( httpClient.Transport, diff --git a/pkg/sources/github/connector_basicauth.go b/pkg/sources/github/connector_basicauth.go index 6d3a6d36c1d8..eda81fa02753 100644 --- a/pkg/sources/github/connector_basicauth.go +++ b/pkg/sources/github/connector_basicauth.go @@ -20,7 +20,8 @@ type basicAuthConnector struct { var _ connector = (*basicAuthConnector)(nil) func newBasicAuthConnector(apiEndpoint string, cred *credentialspb.BasicAuth) (*basicAuthConnector, error) { - httpClient := common.RetryableHTTPClientTimeout(60) + const httpTimeoutSeconds = 60 + httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) httpClient.Transport = &github.BasicAuthTransport{ Username: cred.Username, Password: cred.Password, diff --git a/pkg/sources/github/connector_token.go b/pkg/sources/github/connector_token.go index d0d1d373a8c8..6fedac31f7f1 100644 --- a/pkg/sources/github/connector_token.go +++ b/pkg/sources/github/connector_token.go @@ -25,8 +25,9 @@ type tokenConnector struct { var _ connector = (*tokenConnector)(nil) func newTokenConnector(apiEndpoint string, token string, handleRateLimit func(error) bool) (*tokenConnector, error) { + const httpTimeoutSeconds = 60 + httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) - httpClient := common.RetryableHTTPClientTimeout(60) httpClient.Transport = &oauth2.Transport{ Base: httpClient.Transport, Source: oauth2.ReuseTokenSource(nil, tokenSource), diff --git a/pkg/sources/github/connector_unauthenticated.go b/pkg/sources/github/connector_unauthenticated.go index 3a902be66878..fb2b867154fa 100644 --- a/pkg/sources/github/connector_unauthenticated.go +++ b/pkg/sources/github/connector_unauthenticated.go @@ -17,7 +17,8 @@ type unauthenticatedConnector struct { var _ connector = (*unauthenticatedConnector)(nil) func newUnauthenticatedConnector(apiEndpoint string) (*unauthenticatedConnector, error) { - httpClient := common.RetryableHTTPClientTimeout(60) + const httpTimeoutSeconds = 60 + httpClient := common.RetryableHTTPClientTimeout(int64(httpTimeoutSeconds)) apiClient, err := createGitHubClient(httpClient, apiEndpoint) if err != nil { return nil, fmt.Errorf("could not create API client: %w", err) From f0e37e33c50bfe743345cf72c0c2243bd287d53d Mon Sep 17 00:00:00 2001 From: Cody Rose Date: Fri, 2 Aug 2024 11:14:52 -0400 Subject: [PATCH 48/48] typecheck in only one place --- pkg/sources/github/github.go | 24 ++++++------------- pkg/sources/github/github_integration_test.go | 2 +- pkg/sources/github/github_test.go | 8 +++---- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.go index 00f4e39bd8a5..4222200ca624 100644 --- a/pkg/sources/github/github.go +++ b/pkg/sources/github/github.go @@ -316,9 +316,9 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, tar func (s *Source) enumerate(ctx context.Context) error { // I'm not wild about switching on the connector type here (as opposed to dispatching to the connector itself) but // this felt like a compromise that allowed me to isolate connection logic without rewriting the entire source. - switch s.connector.(type) { + switch c := s.connector.(type) { case *appConnector: - if err := s.enumerateWithApp(ctx); err != nil { + if err := s.enumerateWithApp(ctx, c.InstallationClient()); err != nil { return err } case *basicAuthConnector: @@ -326,7 +326,7 @@ func (s *Source) enumerate(ctx context.Context) error { return err } case *tokenConnector: - if err := s.enumerateWithToken(ctx); err != nil { + if err := s.enumerateWithToken(ctx, c.IsGithubEnterprise()); err != nil { return err } case *unauthenticatedConnector: @@ -433,14 +433,9 @@ func (s *Source) enumerateUnauthenticated(ctx context.Context) { } } -func (s *Source) enumerateWithToken(ctx context.Context) error { +func (s *Source) enumerateWithToken(ctx context.Context, isGithubEnterprise bool) error { ctx.Logger().V(1).Info("Enumerating with token") - connector, ok := s.connector.(*tokenConnector) - if !ok { - return fmt.Errorf("cannot enumerate repositories using a token because no token is configured") - } - var ghUser *github.User var err error for { @@ -464,7 +459,7 @@ func (s *Source) enumerateWithToken(ctx context.Context) error { s.log.Error(err, "Unable to fetch gists for the current user", "user", ghUser.GetLogin()) } - if connector.IsGithubEnterprise() { + if isGithubEnterprise { s.addAllVisibleOrgs(ctx) } else { // Scan for orgs is default with a token. @@ -498,12 +493,7 @@ func (s *Source) enumerateWithToken(ctx context.Context) error { return nil } -func (s *Source) enumerateWithApp(ctx context.Context) error { - connector, ok := s.connector.(*appConnector) - if !ok { - return fmt.Errorf("cannot enumerate app repositories because no app is configured") - } - +func (s *Source) enumerateWithApp(ctx context.Context, installationClient *github.Client) error { // If no repos were provided, enumerate them. if len(s.repos) == 0 { if err := s.getReposByApp(ctx); err != nil { @@ -512,7 +502,7 @@ func (s *Source) enumerateWithApp(ctx context.Context) error { // Check if we need to find user repos. if s.conn.ScanUsers { - err := s.addMembersByApp(ctx, connector.InstallationClient()) + err := s.addMembersByApp(ctx, installationClient) if err != nil { return err } diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.go index a2f171eca8c9..cfc6fe01703b 100644 --- a/pkg/sources/github/github_integration_test.go +++ b/pkg/sources/github/github_integration_test.go @@ -65,7 +65,7 @@ func TestSource_Token(t *testing.T) { s.Init(ctx, "github integration test source", 0, 0, false, conn, 1) s.filteredRepoCache = s.newFilteredRepoCache(memory.New[string](), nil, nil) - err = s.enumerateWithApp(ctx) + err = s.enumerateWithApp(ctx, s.connector.(*appConnector).InstallationClient()) assert.NoError(t, err) _, _, err = s.cloneRepo(ctx, "https://github.com/truffle-test-integration-org/another-test-repo.git") diff --git a/pkg/sources/github/github_test.go b/pkg/sources/github/github_test.go index df89ec41aa33..729f5bf0a5df 100644 --- a/pkg/sources/github/github_test.go +++ b/pkg/sources/github/github_test.go @@ -458,7 +458,7 @@ func TestEnumerateWithToken(t *testing.T) { Token: "token", }, }) - err := s.enumerateWithToken(context.Background()) + err := s.enumerateWithToken(context.Background(), false) assert.Nil(t, err) assert.Equal(t, 2, s.filteredRepoCache.Count()) ok := s.filteredRepoCache.Exists("super-secret-user/super-secret-repo") @@ -502,7 +502,7 @@ func BenchmarkEnumerateWithToken(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - _ = s.enumerateWithToken(context.Background()) + _ = s.enumerateWithToken(context.Background(), false) } } @@ -660,7 +660,7 @@ func TestEnumerateWithToken_IncludeRepos(t *testing.T) { }) s.repos = []string{"some-special-repo"} - err := s.enumerateWithToken(context.Background()) + err := s.enumerateWithToken(context.Background(), false) assert.Nil(t, err) assert.Equal(t, 1, len(s.repos)) assert.Equal(t, []string{"some-special-repo"}, s.repos) @@ -693,7 +693,7 @@ func TestEnumerateWithApp(t *testing.T) { }, }, }) - err := s.enumerateWithApp(context.Background()) + err := s.enumerateWithApp(context.Background(), s.connector.(*appConnector).InstallationClient()) assert.Nil(t, err) assert.Equal(t, 0, len(s.repos)) assert.False(t, gock.HasUnmatchedRequest())