Merge pull request #326 from carlosms/git-auth

Use github auth to fetch git contents

cmd/lookoutd/common.go

@@ -326,8 +326,17 @@ func (c *queueConsumerCommand) initDataHandler() (*lookout.DataServerHandler, er
 		return nil, err
 	}
 
+	var authProvider git.AuthProvider
+	if c.Provider == github.Provider {
+		if c.pool == nil {
+			return nil, fmt.Errorf("pool must be initialized with initProvider")
+		}
+
+		authProvider = c.pool
+	}
+
 	lib := git.NewLibrary(osfs.New(c.Library))
-	sync := git.NewSyncer(lib)
+	sync := git.NewSyncer(lib, authProvider)
 	loader := git.NewLibraryCommitLoader(lib, sync)
 
 	gitService := git.NewService(loader)

cmd/lookoutd/serve.go

@@ -28,6 +28,11 @@ func (c *ServeCommand) Execute(args []string) error {
 		return err
 	}
 
+	err = c.initProvider(conf)
+	if err != nil {
+		return err
+	}
+
 	dataHandler, err := c.initDataHandler()
 	if err != nil {
 		return err
@@ -49,11 +54,6 @@ func (c *ServeCommand) Execute(args []string) error {
 		return err
 	}
 
-	err = c.initProvider(conf)
-	if err != nil {
-		return err
-	}
-
 	poster, err := c.initPoster(conf)
 	if err != nil {
 		return err

cmd/lookoutd/work.go

@@ -28,6 +28,11 @@ func (c *WorkCommand) Execute(args []string) error {
 		return err
 	}
 
+	err = c.initProvider(conf)
+	if err != nil {
+		return err
+	}
+
 	dataHandler, err := c.initDataHandler()
 	if err != nil {
 		return err
@@ -49,11 +54,6 @@ func (c *WorkCommand) Execute(args []string) error {
 		return err
 	}
 
-	err = c.initProvider(conf)
-	if err != nil {
-		return err
-	}
-
 	poster, err := c.initPoster(conf)
 	if err != nil {
 		return err

dummy/dummy_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/stretchr/testify/suite"
 	"google.golang.org/grpc"
 	"gopkg.in/src-d/go-git-fixtures.v3"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 	log "gopkg.in/src-d/go-log.v1"
 )
@@ -42,8 +43,7 @@ func (s *DummySuite) SetupSuite() {
 	fixture := fixtures.Basic().One()
 	s.Basic = fixture
 	fs := fixture.DotGit()
-	sto, err := filesystem.NewStorage(fs)
-	require.NoError(err)
+	sto := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize))
 
 	s.apiServer = grpc.NewServer()
 	server := &lookout.DataServerHandler{

provider/github/client.go

@@ -1,18 +1,21 @@
 package github
 
 import (
+	"context"
 	"net/http"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/src-d/lookout"
+	"github.com/src-d/lookout/service/git"
 	"github.com/src-d/lookout/util/cache"
 	"github.com/src-d/lookout/util/ctxlog"
 
 	"github.com/google/go-github/github"
 	"github.com/gregjones/httpcache"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport"
 	log "gopkg.in/src-d/go-log.v1"
 )
 
@@ -214,16 +217,36 @@ func (p *ClientPool) notify(e ClientPoolEvent) {
 	}
 }
 
+var _ git.AuthProvider = &ClientPool{}
+
+// GitAuth returns a go-git auth method for a repo
+func (p *ClientPool) GitAuth(ctx context.Context, repoInfo *lookout.RepositoryInfo) transport.AuthMethod {
+	c, ok := p.Client(repoInfo.Username, repoInfo.Name)
+	if !ok {
+		return nil
+	}
+
+	return c.gitAuth(ctx)
+}
+
+type gitAuthFn = func(ctx context.Context) transport.AuthMethod
+
 // Client is a wrapper for github.Client that supports cache and provides rate limit information
 type Client struct {
 	*github.Client
 	cache            *cache.ValidableCache
 	limitRT          *limitRoundTripper
 	watchMinInterval time.Duration
+	gitAuth          gitAuthFn
 }
 
 // NewClient creates new Client
-func NewClient(t http.RoundTripper, cache *cache.ValidableCache, watchMinInterval string) *Client {
+func NewClient(
+	t http.RoundTripper,
+	cache *cache.ValidableCache,
+	watchMinInterval string,
+	gitAuth gitAuthFn,
+) *Client {
 	limitRT := &limitRoundTripper{
 		Base: t,
 	}
@@ -247,6 +270,7 @@ func NewClient(t http.RoundTripper, cache *cache.ValidableCache, watchMinInterva
 		cache:            cache,
 		limitRT:          limitRT,
 		watchMinInterval: interval,
+		gitAuth:          gitAuth,
 	}
 }
 

provider/github/installations.go

@@ -8,7 +8,10 @@ import (
 	"github.com/google/go-github/github"
 	"github.com/src-d/lookout"
 	"github.com/src-d/lookout/util/cache"
+	"github.com/src-d/lookout/util/ctxlog"
 	vcsurl "gopkg.in/sourcegraph/go-vcsurl.v1"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport"
+	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 	log "gopkg.in/src-d/go-log.v1"
 )
 
@@ -123,9 +126,24 @@ func (t *Installations) createClient(installationID int64) (*Client, error) {
 		return nil, err
 	}
 
+	// Auth must be: https://x-access-token:<token>@github.com/owner/repo.git
+	// Reference: https://developer.github.com/apps/building-github-apps/authenticating-with-github-apps/#http-based-git-access-by-an-installation
+	gitAuth := func(ctx context.Context) transport.AuthMethod {
+		token, err := itr.Token()
+		if err != nil {
+			ctxlog.Get(ctx).Errorf(err, "failed to get an installation access token")
+			return nil
+		}
+
+		return &githttp.BasicAuth{
+			Username: "x-access-token",
+			Password: token,
+		}
+	}
+
 	// TODO (carlosms): hardcoded, take from config
 	watchMinInterval := ""
-	return NewClient(itr, t.cache, watchMinInterval), nil
+	return NewClient(itr, t.cache, watchMinInterval, gitAuth), nil
 }
 
 func (t *Installations) getRepos(iClient *Client) ([]*lookout.RepositoryInfo, error) {

provider/github/poster.go

@@ -3,6 +3,7 @@ package github
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"strings"
 
 	"github.com/src-d/lookout"
@@ -66,7 +67,7 @@ func (p *Poster) Post(ctx context.Context, e lookout.Event,
 
 		return p.postPR(ctx, ev, aCommentsList)
 	default:
-		return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type"))
+		return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type %s", reflect.TypeOf(e)))
 	}
 }
 
@@ -302,7 +303,7 @@ func (p *Poster) Status(ctx context.Context, e lookout.Event, status lookout.Ana
 
 		return p.statusPR(ctx, ev, status)
 	default:
-		return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type"))
+		return ErrEventNotSupported.Wrap(fmt.Errorf("unsupported event type %s", reflect.TypeOf(e)))
 	}
 }
 

provider/github/token_transport.go

@@ -1,13 +1,16 @@
 package github
 
 import (
+	"context"
 	"net/http"
 
 	"github.com/src-d/lookout"
 	"github.com/src-d/lookout/util/cache"
 	"github.com/src-d/lookout/util/ctxlog"
 
 	vcsurl "gopkg.in/sourcegraph/go-vcsurl.v1"
+	"gopkg.in/src-d/go-git.v4/plumbing/transport"
+	githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
 	log "gopkg.in/src-d/go-log.v1"
 )
 
@@ -42,10 +45,21 @@ func NewClientPoolFromTokens(urlToConfig map[string]ClientConfig, cache *cache.V
 	byClients := make(map[*Client][]*lookout.RepositoryInfo, len(byConfig))
 	byRepo := make(map[string]*Client, len(urlToConfig))
 	for conf, repos := range byConfig {
-		client := NewClient(&roundTripper{
+		rt := &roundTripper{
 			User:     conf.User,
 			Password: conf.Token,
-		}, cache, conf.MinInterval)
+		}
+
+		// Auth must be: https://<token>@github.com/owner/repo.git
+		// Reference: https://blog.github.com/2012-09-21-easier-builds-and-deployments-using-git-over-https-and-oauth/
+		gitAuth := func(ctx context.Context) transport.AuthMethod {
+			return &githttp.BasicAuth{
+				Username: conf.Token,
+				Password: "",
+			}
+		}
+
+		client := NewClient(rt, cache, conf.MinInterval, gitAuth)
 
 		if _, ok := byClients[client]; !ok {
 			byClients[client] = []*lookout.RepositoryInfo{}

provider/github/utils.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/go-github/github"
 	"gopkg.in/sourcegraph/go-vcsurl.v1"
 	"gopkg.in/src-d/go-git.v4/plumbing"
+	log "gopkg.in/src-d/go-log.v1"
 )
 
 func castEvent(r *lookout.RepositoryInfo, e *github.Event) (lookout.Event, error) {
@@ -100,7 +101,10 @@ func castPullRequestBranch(ctx context.Context, b *github.PullRequestBranch) loo
 
 	r, err := vcsurl.Parse(b.GetRepo().GetCloneURL())
 	if err != nil {
-		ctxlog.Get(ctx).Warningf("malformed repository URL on pull request branch")
+		ctxlog.Get(ctx).With(log.Fields{
+			"url": b.GetRepo().GetCloneURL()},
+		).Warningf("malformed repository URL on pull request branch")
+
 		return lookout.ReferencePointer{}
 	}
 

provider/github/watcher_test.go

@@ -296,7 +296,7 @@ func (s *WatcherTestSuite) TestCustomMinInterval() {
 	s.mux.HandleFunc("/repos/mock/test/events", eventsHandler(&eventCalls))
 
 	clientMinInterval := 200 * time.Millisecond
-	client := NewClient(nil, s.cache, clientMinInterval.String())
+	client := NewClient(nil, s.cache, clientMinInterval.String(), nil)
 	client.BaseURL = s.githubURL
 	client.UploadURL = s.githubURL
 
@@ -483,7 +483,7 @@ func (t *NoopTransport) Get(repo string) http.RoundTripper {
 }
 
 func newClient(githubURL *url.URL, cache *cache.ValidableCache) *Client {
-	client := NewClient(nil, cache, "")
+	client := NewClient(nil, cache, "", nil)
 	client.BaseURL = githubURL
 	client.UploadURL = githubURL
 	return client

server/server.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"sync"
 
 	"github.com/src-d/lookout"
@@ -40,7 +41,7 @@ func NewServer(p lookout.Poster, fileGetter lookout.FileGetter,
 // HandleEvent processes the event calling the analyzers, and posting the results
 func (s *Server) HandleEvent(ctx context.Context, e lookout.Event) error {
 	ctx, logger := ctxlog.WithLogFields(ctx, log.Fields{
-		"event-type": e.Type(),
+		"event-type": reflect.TypeOf(e),
 		"event-id":   e.ID().String(),
 		"repo":       e.Revision().Head.InternalRepositoryURL,
 		"head":       e.Revision().Head.ReferenceName,

service/git/examples_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/src-d/lookout"
 
 	"gopkg.in/src-d/go-git-fixtures.v3"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 )
 
@@ -18,10 +19,7 @@ func Example() {
 
 	fixture := fixtures.Basic().One()
 	fs := fixture.DotGit()
-	storer, err := filesystem.NewStorage(fs)
-	if err != nil {
-		panic(err)
-	}
+	storer := filesystem.NewStorage(fs, cache.NewObjectLRU(cache.DefaultMaxSize))
 
 	// Create the git service with a repository loader that allows it to find
 	// a repository by ID.

service/git/library.go

@@ -14,6 +14,7 @@ import (
 	"gopkg.in/src-d/go-errors.v1"
 	"gopkg.in/src-d/go-git.v4"
 	"gopkg.in/src-d/go-git.v4/config"
+	"gopkg.in/src-d/go-git.v4/plumbing/cache"
 	"gopkg.in/src-d/go-git.v4/storage"
 	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 )
@@ -142,7 +143,9 @@ func (l *Library) repositoryStorer(url *lookout.RepositoryInfo) (
 		return nil, err
 	}
 
-	return filesystem.NewStorage(fs)
+	// TODO(carlosms) take cache size from config
+	cache := cache.NewObjectLRU(cache.DefaultMaxSize)
+	return filesystem.NewStorage(fs, cache), nil
 }
 
 func (l *Library) repositoryPath(url *lookout.RepositoryInfo) string {