From 935d275f6dfeaf642706e6a23dd1b699d5739115 Mon Sep 17 00:00:00 2001 From: Qingyang Hu Date: Tue, 25 Jul 2023 17:08:59 -0400 Subject: [PATCH] e2e wip --- .evergreen/config.yml | 58 +++++++ internal/test/searchindex/Makefile | 8 + internal/test/searchindex/mongodb/go.mod | 27 +++ internal/test/searchindex/mongodb/go.sum | 57 +++++++ internal/test/searchindex/mongodb/main.go | 194 ++++++++++++++++++++++ internal/test/searchindex/template.yaml | 58 +++++++ 6 files changed, 402 insertions(+) create mode 100644 internal/test/searchindex/Makefile create mode 100644 internal/test/searchindex/mongodb/go.mod create mode 100644 internal/test/searchindex/mongodb/go.sum create mode 100644 internal/test/searchindex/mongodb/main.go create mode 100644 internal/test/searchindex/template.yaml diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ab6289576a..a7557f9ae9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -2271,6 +2271,25 @@ tasks: ${PREPARE_SHELL} ./.evergreen/run-deployed-lambda-aws-tests.sh + - name: "test-aws-searchindex-deployed" + commands: + - command: ec2.assume_role + params: + role_arn: ${LAMBDA_AWS_ROLE_ARN} + duration_seconds: 3600 + - command: shell.exec + params: + working_dir: src/go.mongodb.org/mongo-driver + shell: bash + add_expansions_to_env: true + env: + TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/internal/test/searchindex + LAMBDA_STACK_NAME: dbx-go-lambda + AWS_REGION: us-east-1 + script: | + ${PREPARE_SHELL} + ./.evergreen/run-deployed-lambda-aws-tests.sh + axes: - id: version display_name: MongoDB Version @@ -2628,6 +2647,39 @@ task_groups: tasks: - test-aws-lambda-deployed + - name: test-search-index-task-group + setup_group: + - func: fetch-source + - func: prepare-resources + - command: subprocess.exec + params: + working_dir: src/go.mongodb.org/mongo-driver + binary: bash + add_expansions_to_env: true + env: + LAMBDA_STACK_NAME: dbx-go-lambda + AWS_REGION: us-east-1 + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml + teardown_group: + - command: subprocess.exec + params: + working_dir: src/go.mongodb.org/mongo-driver + binary: bash + add_expansions_to_env: true + env: + LAMBDA_STACK_NAME: dbx-go-lambda + AWS_REGION: us-east-1 + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + setup_group_can_fail_task: true + setup_group_timeout_secs: 1800 + tasks: + - test-aws-searchindex-deployed + buildvariants: - name: static-analysis display_name: "Static Analysis" @@ -2777,6 +2829,12 @@ buildvariants: tasks: - test-aws-lambda-task-group + - matrix_name: "searchindex-test" + matrix_spec: { version: ["7.0"], os-faas-80: ["rhel80-large-go-1-20"] } + display_name: "Search Index ${version} ${os-faas-80}" + tasks: + - test-search-index-task-group + - name: testgcpkms-variant display_name: "GCP KMS" run_on: diff --git a/internal/test/searchindex/Makefile b/internal/test/searchindex/Makefile new file mode 100644 index 0000000000..b374dc97b8 --- /dev/null +++ b/internal/test/searchindex/Makefile @@ -0,0 +1,8 @@ +.PHONY: default +default: + sam build --debug + # Cannot connect to a local mongo server on non x86_64 architectures. + # These cases require either building a mongodb docker container or + # using a serverless service such as Atlas. + sam local invoke --parameter-overrides "MongoDbUri=${MONGODB_URI}" + diff --git a/internal/test/searchindex/mongodb/go.mod b/internal/test/searchindex/mongodb/go.mod new file mode 100644 index 0000000000..bf9115938e --- /dev/null +++ b/internal/test/searchindex/mongodb/go.mod @@ -0,0 +1,27 @@ +module go.mongodb.go/mongo-driver/internal/test/mongodb + +go 1.20 + +replace go.mongodb.org/mongo-driver => ../../../../ + +require ( + github.com/aws/aws-lambda-go v1.41.0 + // Note that the Go driver version is replaced with the local Go driver code + // by the replace directive above. + go.mongodb.org/mongo-driver v1.11.7 +) + +require ( + github.com/golang/snappy v0.0.1 // indirect + github.com/klauspost/compress v1.13.6 // indirect + github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/scram v1.1.2 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/text v0.7.0 // indirect +) + +replace gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.2.8 diff --git a/internal/test/searchindex/mongodb/go.sum b/internal/test/searchindex/mongodb/go.sum new file mode 100644 index 0000000000..e808493120 --- /dev/null +++ b/internal/test/searchindex/mongodb/go.sum @@ -0,0 +1,57 @@ +github.com/aws/aws-lambda-go v1.41.0 h1:l/5fyVb6Ud9uYd411xdHZzSf2n86TakxzpvIoz7l+3Y= +github.com/aws/aws-lambda-go v1.41.0/go.mod h1:jwFe2KmMsHmffA1X2R09hH6lFzJQxzI8qK17ewzbQMM= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/test/searchindex/mongodb/main.go b/internal/test/searchindex/mongodb/main.go new file mode 100644 index 0000000000..632d1ab332 --- /dev/null +++ b/internal/test/searchindex/mongodb/main.go @@ -0,0 +1,194 @@ +// Copyright (C) MongoDB, Inc. 2023-present. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "time" + + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-lambda-go/lambda" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/event" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +const timeout = 10 * time.Second + +// eventListener supports command, heartbeat, and pool event handlers to record +// event durations, as well as the number of heartbeats, commands, and open +// conections. +type eventListener struct { + commandCount int + commandDuration int64 + heartbeatCount int + heartbeatDuration int64 + openConnections int +} + +// commandMonitor initializes an event.CommandMonitor that will count the number +// of successful or failed command events and record a running duration of these +// events. +func (listener *eventListener) commandMonitor() *event.CommandMonitor { + succeeded := func(_ context.Context, e *event.CommandSucceededEvent) { + listener.commandCount++ + listener.commandDuration += e.DurationNanos + } + + failed := func(_ context.Context, e *event.CommandFailedEvent) { + listener.commandCount++ + listener.commandDuration += e.DurationNanos + } + + return &event.CommandMonitor{ + Succeeded: succeeded, + Failed: failed, + } +} + +// severMonitor initializes an event.ServerMonitor that will count the number +// of successful or failed heartbeat events and record a running duration of +// these events. +func (listener *eventListener) serverMonitor() *event.ServerMonitor { + succeeded := func(e *event.ServerHeartbeatSucceededEvent) { + listener.heartbeatCount++ + listener.heartbeatDuration += e.DurationNanos + } + + failed := func(e *event.ServerHeartbeatFailedEvent) { + listener.heartbeatCount++ + listener.heartbeatDuration += e.DurationNanos + } + + return &event.ServerMonitor{ + ServerHeartbeatSucceeded: succeeded, + ServerHeartbeatFailed: failed, + } +} + +// poolMonitor initialize an event.PoolMonitor that will increment each time a +// new conneciton is created and decrement each time a connection is closed. +func (listener *eventListener) poolMonitor() *event.PoolMonitor { + poolEvent := func(e *event.PoolEvent) { + switch e.Type { + case event.ConnectionCreated: + listener.openConnections++ + case event.ConnectionClosed: + listener.openConnections-- + } + } + + return &event.PoolMonitor{Event: poolEvent} +} + +// response is the data we return in the body of the API Gateway response. +type response struct { + AvgCommandDuration float64 `json:"averageCommandDuration"` + AvgHeartbeatDuration float64 `json:"averageHeartbeatDuration"` + OpenConnections int `json:"openConnections"` + HeartbeatCount int `json:"heartbeatCount"` +} + +// gateway500 is a convenience function for constructing a gateway response with +// a 500 status code, indicating an internal server error. +func gateway500() events.APIGatewayProxyResponse { + return events.APIGatewayProxyResponse{ + StatusCode: http.StatusInternalServerError, + Body: http.StatusText(http.StatusInternalServerError), + } + +} + +// handler is the AWS Lambda handler, executing at runtime. +func handler(ctx context.Context, request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + listener := new(eventListener) + + clientOptions := options.Client().ApplyURI(os.Getenv("MONGODB_URI")). + SetMonitor(listener.commandMonitor()). + SetServerMonitor(listener.serverMonitor()). + SetPoolMonitor(listener.poolMonitor()). + SetTimeout(time.Duration(5 * time.Minute)). + SetServerSelectionTimeout(time.Duration(5 * time.Minute)) + + // Create a MongoClient that points to MONGODB_URI and listens to the + // ComandMonitor, ServerMonitor, and PoolMonitor events. + client, err := mongo.NewClient(clientOptions) + if err != nil { + return gateway500(), fmt.Errorf("failed to create client: %w", err) + } + + // Attempt to connect to the client with a timeout. + if err = client.Connect(ctx); err != nil { + return gateway500(), fmt.Errorf("failed to connect: %w", err) + } + + defer client.Disconnect(ctx) + + var commandResult bson.M + err = client.Database("test").RunCommand(context.TODO(), bson.D{{"serverStatus", 1}}).Decode(&commandResult) + if err != nil { + return gateway500(), fmt.Errorf("failed to run command: %w", err) + } + + collection := client.Database("test").Collection("coll0") + + view := collection.SearchIndexes() + searchName := "test-search-index" + index, err := view.CreateOne(ctx, mongo.SearchIndexModel{ + Definition: bson.D{{"mappings", bson.D{{"dynamic", false}}}}, + Name: &searchName, + }) + if err != nil { + return gateway500(), fmt.Errorf("failed to create index: %+v, %w", commandResult["version"], err) + } + + cursor, err := view.List(ctx, &index, nil) + if err != nil { + return gateway500(), fmt.Errorf("failed to list: %w", err) + } + + var avgCmdDur float64 + if count := listener.commandCount; count != 0 { + avgCmdDur = float64(listener.commandDuration) / float64(count) + } + + var avgHBDur float64 + if count := listener.heartbeatCount; count != 0 { + avgHBDur = float64(listener.heartbeatDuration) / float64(count) + } + + rsp := &response{ + AvgCommandDuration: avgCmdDur, + AvgHeartbeatDuration: avgHBDur, + OpenConnections: listener.openConnections, + HeartbeatCount: listener.heartbeatCount, + } + + _, err = json.Marshal(rsp) + if err != nil { + return gateway500(), fmt.Errorf("failed to marshal: %w", err) + } + + return events.APIGatewayProxyResponse{ + Body: cursor.Current.String(), + StatusCode: http.StatusOK, + }, nil +} + +func main() { + ctx := context.Background() + + lambda.StartWithOptions(handler, lambda.WithContext(ctx)) +} diff --git a/internal/test/searchindex/template.yaml b/internal/test/searchindex/template.yaml new file mode 100644 index 0000000000..4750c1a680 --- /dev/null +++ b/internal/test/searchindex/template.yaml @@ -0,0 +1,58 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: > + Go Driver lambda function test + +Parameters: + MongoDbUri: + Type: String + Description: The MongoDB connection string. + +Globals: + Function: + Timeout: 30 + +Resources: + MongoDBFunction: + Type: AWS::Serverless::Function + Properties: + CodeUri: mongodb/ + Handler: mongodb + Runtime: go1.x + Architectures: + - x86_64 + Events: + MongoDB: + Type: Api + Properties: + Path: /mongodb + Method: GET + Environment: + Variables: + MONGODB_URI: !Ref MongoDbUri + + ApplicationResourceGroup: + Type: AWS::ResourceGroups::Group + Properties: + Name: + Fn::Sub: ApplicationInsights-SAM-${AWS::StackName} + ResourceQuery: + Type: CLOUDFORMATION_STACK_1_0 + ApplicationInsightsMonitoring: + Type: AWS::ApplicationInsights::Application + Properties: + ResourceGroupName: + Ref: ApplicationResourceGroup + AutoConfigurationEnabled: 'true' + +Outputs: + MongoDBApi: + Description: "API Gateway endpoint URL for Prod stage for MongoDB function" + Value: !Sub "https://${ServerlessRestApi}.execute-api.${AWS::Region}.amazonaws.com/Prod/mongodb/" + MongoDBFunction: + Description: "MongoDB Lambda Function ARN" + Value: !GetAtt MongoDBFunction.Arn + MongoDBFunctionIamRole: + Description: "Implicit IAM Role created for MongoDB function" + Value: !GetAtt MongoDBFunctionRole.Arn +