Merge pull request #164 from carlosms/integration-gotest

Integration tests with `go test`

.travis.yml

@@ -49,15 +49,13 @@ jobs:
       - psql -c 'create database lookout;' -U postgres
       - make build
       - ./build/bin/lookout migrate
-      - cp config.yml.tpl config.yml
       - make test-json
     - name: "Lookout serve Integration Tests macOS"
       script:
       - make prepare-services
       - psql -c 'create database lookout;' -U postgres
       - make build
       - ./build/lookout_darwin_amd64/lookout migrate
-      - cp config.yml.tpl config.yml
       - make test-json
       os: osx
       osx_image: xcode9.4

Makefile

@@ -95,7 +95,9 @@ test-sdk-short: clean-sdk build-sdk
 # Integration test for lookout serve
 .PHONY: test-json
 test-json: clean-sdk build-sdk
-	DUMMY_BIN=$(PWD)/$(DUMMY_BIN) LOOKOUT_BIN=$(PWD)/$(LOOKOUT_BIN) $(GOCMD) run cmd/server-test/main.go
+	DUMMY_BIN=$(PWD)/$(DUMMY_BIN) \
+	LOOKOUT_BIN=$(PWD)/$(LOOKOUT_BIN) \
+	$(GOTEST_INTEGRATION) github.com/src-d/lookout/cmd/server-test
 
 # Build sdk client and dummy analyzer
 .PHONY: build-sdk

cmd/sdk-test/bblfsh_test.go

@@ -3,7 +3,6 @@
 package sdk_test
 
 import (
-	"context"
 	"io"
 	"testing"
 
@@ -13,34 +12,32 @@ import (
 )
 
 type BblfshIntegrationSuite struct {
-	suite.Suite
-	ctx  context.Context
-	stop func()
+	cmdtest.IntegrationSuite
 }
 
 func (suite *BblfshIntegrationSuite) SetupSuite() {
-	suite.ctx, suite.stop = cmdtest.StoppableCtx()
-	cmdtest.StartDummy(suite.ctx, "--uast", "--files")
+	suite.StoppableCtx()
+	suite.StartDummy("--uast", "--files")
 }
 
 func (suite *BblfshIntegrationSuite) TearDownSuite() {
-	suite.stop()
+	suite.Stop()
 }
 
 func (suite *BblfshIntegrationSuite) RunReview() io.Reader {
-	return cmdtest.RunCli(suite.ctx, "review", "ipv4://localhost:10302",
+	return suite.RunCli("review", "ipv4://localhost:10302",
 		"--from=66924f49aa9987273a137857c979ee5f0e709e30",
 		"--to=2c9f56bcb55be47cf35d40d024ec755399f699c7")
 }
 
 func (suite *BblfshIntegrationSuite) RunPush() io.Reader {
-	return cmdtest.RunCli(suite.ctx, "push", "ipv4://localhost:10302",
+	return suite.RunCli("push", "ipv4://localhost:10302",
 		"--from=66924f49aa9987273a137857c979ee5f0e709e30",
 		"--to=2c9f56bcb55be47cf35d40d024ec755399f699c7")
 }
 
 func (suite *BblfshIntegrationSuite) TestReviewNoBblfshError() {
-	r := cmdtest.RunCli(suite.ctx, "review", "ipv4://localhost:10302",
+	r := suite.RunCli("review", "ipv4://localhost:10302",
 		"--bblfshd=ipv4://localhost:0000",
 		"--from=66924f49aa9987273a137857c979ee5f0e709e30",
 		"--to=2c9f56bcb55be47cf35d40d024ec755399f699c7")
@@ -63,7 +60,7 @@ func (suite *BblfshIntegrationSuite) TestReviewLanguage() {
 }
 
 func (suite *BblfshIntegrationSuite) TestPushNoBblfshError() {
-	r := cmdtest.RunCli(suite.ctx, "push", "ipv4://localhost:10302",
+	r := suite.RunCli("push", "ipv4://localhost:10302",
 		"--bblfshd=ipv4://localhost:0000",
 		"--from=66924f49aa9987273a137857c979ee5f0e709e30",
 		"--to=2c9f56bcb55be47cf35d40d024ec755399f699c7")

cmd/sdk-test/dummy_test.go

@@ -3,55 +3,52 @@
 package sdk_test
 
 import (
-	"context"
 	"testing"
 
 	"github.com/src-d/lookout/util/cmdtest"
 
 	"github.com/stretchr/testify/suite"
 )
 
-type IntegrationSuite struct {
-	suite.Suite
-	ctx  context.Context
-	stop func()
+type SDKDummyTestSuite struct {
+	cmdtest.IntegrationSuite
 }
 
-func (suite *IntegrationSuite) SetupTest() {
-	suite.ctx, suite.stop = cmdtest.StoppableCtx()
+func (suite *SDKDummyTestSuite) SetupTest() {
+	suite.StoppableCtx()
 }
 
-func (suite *IntegrationSuite) TearDownTest() {
-	suite.stop()
+func (suite *SDKDummyTestSuite) TearDownTest() {
+	suite.Stop()
 }
 
-func (suite *IntegrationSuite) TestReview() {
-	cmdtest.StartDummy(suite.ctx, "--files")
+func (suite *SDKDummyTestSuite) TestReview() {
+	suite.StartDummy("--files")
 
-	r := cmdtest.RunCli(suite.ctx, "review", "ipv4://localhost:10302",
+	r := suite.RunCli("review", "ipv4://localhost:10302",
 		"--from=66924f49aa9987273a137857c979ee5f0e709e30",
 		"--to=2c9f56bcb55be47cf35d40d024ec755399f699c7")
 	cmdtest.GrepTrue(r, "posting analysis")
 }
 
-func (suite *IntegrationSuite) TestPush() {
-	cmdtest.StartDummy(suite.ctx, "--files")
+func (suite *SDKDummyTestSuite) TestPush() {
+	suite.StartDummy("--files")
 
-	r := cmdtest.RunCli(suite.ctx, "push", "ipv4://localhost:10302",
+	r := suite.RunCli("push", "ipv4://localhost:10302",
 		"--from=66924f49aa9987273a137857c979ee5f0e709e30",
 		"--to=2c9f56bcb55be47cf35d40d024ec755399f699c7")
 	cmdtest.GrepTrue(r, "posting analysis")
 }
 
-func (suite *IntegrationSuite) TestPushNoComments() {
-	cmdtest.StartDummy(suite.ctx)
+func (suite *SDKDummyTestSuite) TestPushNoComments() {
+	suite.StartDummy()
 
-	r := cmdtest.RunCli(suite.ctx, "push", "ipv4://localhost:10302",
+	r := suite.RunCli("push", "ipv4://localhost:10302",
 		"--from=66924f49aa9987273a137857c979ee5f0e709e30",
 		"--to=2c9f56bcb55be47cf35d40d024ec755399f699c7")
 	cmdtest.GrepTrue(r, "no comments were produced")
 }
 
-func TestIntegrationSuite(t *testing.T) {
-	suite.Run(t, new(IntegrationSuite))
+func TestSDKDummyTestSuite(t *testing.T) {
+	suite.Run(t, new(SDKDummyTestSuite))
 }

cmd/server-test/common_test.go

@@ -0,0 +1,21 @@
+// +build integration
+
+package server_test
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/src-d/lookout/util/cmdtest"
+)
+
+type IntegrationSuite struct {
+	cmdtest.IntegrationSuite
+	r io.Reader
+	w io.WriteCloser
+}
+
+func (suite *IntegrationSuite) sendEvent(json string) {
+	_, err := fmt.Fprintln(suite.w, json)
+	suite.Require().NoError(err)
+}

cmd/server-test/dummy_analyzer_test.go

@@ -0,0 +1,76 @@
+// +build integration
+
+package server_test
+
+import (
+	"testing"
+
+	"github.com/src-d/lookout/util/cmdtest"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type DummyIntegrationSuite struct {
+	IntegrationSuite
+}
+
+func (suite *DummyIntegrationSuite) SetupTest() {
+	suite.ResetDB()
+
+	suite.StoppableCtx()
+	suite.StartDummy("--files")
+	suite.r, suite.w = suite.StartServe("--provider", "json",
+		"-c", "../../fixtures/dummy_config.yml", "dummy-repo-url")
+
+	// make sure server started correctly
+	cmdtest.GrepTrue(suite.r, "Starting watcher")
+}
+
+func (suite *DummyIntegrationSuite) TearDownTest() {
+	suite.Stop()
+}
+
+const successJSON = `{"event":"review", "internal_id": "1", "number": 1, "commit_revision":{"base":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"66924f49aa9987273a137857c979ee5f0e709e30"},"head":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"2c9f56bcb55be47cf35d40d024ec755399f699c7"}}}`
+
+func (suite *DummyIntegrationSuite) TestSuccessReview() {
+	suite.sendEvent(successJSON)
+	cmdtest.GrepTrue(suite.r, "processing pull request")
+	cmdtest.GrepTrue(suite.r, `{"analyzer-name":"Dummy","file":"cmd/lookout/push.go","line":13,"text":"This line exceeded 80 bytes."}`)
+	cmdtest.GrepTrue(suite.r, `status=success`)
+}
+
+func (suite *DummyIntegrationSuite) TestSkipReview() {
+	suite.sendEvent(successJSON)
+	cmdtest.GrepTrue(suite.r, `status=success`)
+
+	suite.sendEvent(successJSON)
+	cmdtest.GrepTrue(suite.r, `event successfully processed, skipping...`)
+}
+
+func (suite *DummyIntegrationSuite) TestReviewDontPost() {
+	suite.sendEvent(successJSON)
+	cmdtest.GrepTrue(suite.r, `status=success`)
+
+	json := `{"event":"review", "internal_id": "2", "number": 1, "commit_revision":{"base":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"66924f49aa9987273a137857c979ee5f0e709e30"},"head":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"2c9f56bcb55be47cf35d40d024ec755399f699c7"}}}`
+	suite.sendEvent(json)
+	cmdtest.GrepTrue(suite.r, "processing pull request")
+	cmdtest.GrepAndNot(suite.r, `status=success`, `posting analysis`)
+}
+
+func (suite *DummyIntegrationSuite) TestWrongRevision() {
+	json := `{"event":"review", "internal_id": "3", "number": 3, "commit_revision": {"base":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"0000000000000000000000000000000000000000"},"head":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"0000000000000000000000000000000000000000"}}}`
+	suite.sendEvent(json)
+	cmdtest.GrepTrue(suite.r, `event processing failed`)
+}
+
+func (suite *DummyIntegrationSuite) TestSuccessPush() {
+	successPushJSON := `{"event":"push", "internal_id": "1", "commit_revision":{"base":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"66924f49aa9987273a137857c979ee5f0e709e30"},"head":{"internal_repository_url":"https://github.com/src-d/lookout.git","reference_name":"refs/heads/master","hash":"2c9f56bcb55be47cf35d40d024ec755399f699c7"}}}`
+	suite.sendEvent(successPushJSON)
+	cmdtest.GrepTrue(suite.r, "processing push")
+	cmdtest.GrepTrue(suite.r, "comments can belong only to review event but 1 is given")
+	cmdtest.GrepTrue(suite.r, `status=success`)
+}
+
+func TestDummyIntegrationSuite(t *testing.T) {
+	suite.Run(t, new(DummyIntegrationSuite))
+}

cmd/server-test/error_analyzer_test.go

@@ -0,0 +1,78 @@
+// +build integration
+
+package server_test
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"github.com/src-d/lookout"
+	"github.com/src-d/lookout/util/cmdtest"
+	"github.com/src-d/lookout/util/grpchelper"
+	log "gopkg.in/src-d/go-log.v1"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type errAnalyzer struct{}
+
+func (a *errAnalyzer) NotifyReviewEvent(ctx context.Context, e *lookout.ReviewEvent) (*lookout.EventResponse, error) {
+	return nil, errors.New("review error")
+}
+
+func (a *errAnalyzer) NotifyPushEvent(ctx context.Context, e *lookout.PushEvent) (*lookout.EventResponse, error) {
+	return nil, errors.New("push error")
+}
+
+type ErrorAnalyzerIntegrationSuite struct {
+	IntegrationSuite
+}
+
+func (suite *ErrorAnalyzerIntegrationSuite) startAnalyzer(ctx context.Context, a lookout.AnalyzerServer) error {
+	log.DefaultFactory = &log.LoggerFactory{
+		Level: log.ErrorLevel,
+	}
+	log.DefaultLogger = log.New(log.Fields{"app": "test"})
+
+	server := grpchelper.NewServer()
+	lookout.RegisterAnalyzerServer(server, a)
+
+	lis, err := grpchelper.Listen("ipv4://localhost:10302")
+	if err != nil {
+		return err
+	}
+
+	go server.Serve(lis)
+	go func() {
+		<-ctx.Done()
+		server.Stop()
+	}()
+	return nil
+}
+
+func (suite *ErrorAnalyzerIntegrationSuite) SetupTest() {
+	suite.ResetDB()
+
+	suite.StoppableCtx()
+	suite.startAnalyzer(suite.Ctx, &errAnalyzer{})
+	suite.r, suite.w = suite.StartServe("--provider", "json",
+		"-c", "../../fixtures/dummy_config.yml", "dummy-repo-url")
+
+	// make sure server started correctly
+	cmdtest.GrepTrue(suite.r, "Starting watcher")
+}
+
+func (suite *ErrorAnalyzerIntegrationSuite) TearDownTest() {
+	suite.Stop()
+}
+
+func (suite *ErrorAnalyzerIntegrationSuite) TestAnalyzerErr() {
+	suite.sendEvent(successJSON)
+
+	cmdtest.GrepTrue(suite.r, `msg="analysis failed" analyzer=Dummy app=lookout error="rpc error: code = Unknown desc = review error"`)
+}
+
+func TestErrorAnalyzerIntegrationSuite(t *testing.T) {
+	suite.Run(t, new(ErrorAnalyzerIntegrationSuite))
+}