diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 5d256e9b0a..8785c40530 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -305,7 +305,7 @@ functions: script: | ${PREPARE_SHELL} # Clean up cse servers - bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop_servers.sh + bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh - command: shell.exec params: shell: "bash" @@ -366,10 +366,10 @@ functions: params: shell: "bash" working_dir: src/go.mongodb.org/mongo-driver - add_expansions_to_env: true + include_expansions_in_env: ["TOPOLOGY", "AUTH", "SSL", "MONGODB_URI", "CRYPT_SHARED_LIB_PATH", "SKIP_CRYPT_SHARED_LIB", "RACE", "MONGO_GO_DRIVER_COMPRESSOR", "REQUIRE_API_VERSION", "LOAD_BALANCER"] script: | ${PREPARE_SHELL} - sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh create-api-report: - command: shell.exec @@ -389,90 +389,32 @@ functions: file: src/go.mongodb.org/mongo-driver/perf.json run-enterprise-auth-tests: + - command: ec2.assume_role + params: + role_arn: "${aws_test_secrets_role}" - command: shell.exec type: test params: shell: "bash" working_dir: src/go.mongodb.org/mongo-driver + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | - # DO NOT ECHO WITH XTRACE - if [ "Windows_NT" = "$OS" ]; then - export GOPATH=$(cygpath -w $(dirname $(dirname $(dirname `pwd`)))) - export GOCACHE=$(cygpath -w "$(pwd)/.cache") - else - export GOPATH=$(dirname $(dirname $(dirname `pwd`))) - export GOCACHE="$(pwd)/.cache" - fi; - export GOPATH="$GOPATH" - export GOROOT="${GO_DIST}" - export GOCACHE="$GOCACHE" - export PATH="${GCC_PATH}:${GO_DIST}/bin:$PATH" - export MONGODB_URI="${MONGODB_URI}" - export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR}" - - make -s evg-test-enterprise-auth + ${PREPARE_SHELL} + bash ./etc/run-enterprise-plain-test.sh run-enterprise-gssapi-auth-tests: - - command: shell.exec - type: test + - command: ec2.assume_role params: - shell: "bash" - silent: true - working_dir: src/go.mongodb.org/mongo-driver - script: | - # DO NOT ECHO WITH XTRACE - if [ "Windows_NT" = "$OS" ]; then - export GOPATH=$(cygpath -w $(dirname $(dirname $(dirname `pwd`)))) - export GOCACHE=$(cygpath -w "$(pwd)/.cache") - export MONGODB_URI=${gssapi_auth_windows_mongodb_uri} - else - export GOPATH=$(dirname $(dirname $(dirname `pwd`))) - export GOCACHE="$(pwd)/.cache" - echo "${gssapi_auth_linux_keytab_base64}" > /tmp/drivers.keytab.base64 - base64 --decode /tmp/drivers.keytab.base64 > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab - mkdir -p ~/.krb5 - cat .evergreen/krb5.config | tee -a ~/.krb5/config - kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p "${gssapi_auth_username}" - export MONGODB_URI="${gssapi_auth_linux_mongodb_uri}" - fi; - export GOPATH="$GOPATH" - export GOROOT="${GO_DIST}" - export GOCACHE="$GOCACHE" - export PATH="${GCC_PATH}:${GO_DIST}/bin:$PATH" - export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR}" - - make -s evg-test-enterprise-auth - - run-enterprise-gssapi-service-host-auth-tests: + role_arn: "${aws_test_secrets_role}" - command: shell.exec type: test params: shell: "bash" - silent: true working_dir: src/go.mongodb.org/mongo-driver + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | - # DO NOT ECHO WITH XTRACE - if [ "Windows_NT" = "$OS" ]; then - export GOPATH=$(cygpath -w $(dirname $(dirname $(dirname `pwd`)))) - export GOCACHE=$(cygpath -w "$(pwd)/.cache") - export MONGODB_URI="${gssapi_service_host_auth_windows_mongodb_uri}" - else - export GOPATH=$(dirname $(dirname $(dirname `pwd`))) - export GOCACHE="$(pwd)/.cache" - echo "${gssapi_auth_linux_keytab_base64}" > /tmp/drivers.keytab.base64 - base64 --decode /tmp/drivers.keytab.base64 > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab - mkdir -p ~/.krb5 - cat .evergreen/krb5.config | tee -a ~/.krb5/config - kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p "${gssapi_auth_username}" - export MONGODB_URI="${gssapi_service_host_auth_linux_mongodb_uri}" - fi; - export GOPATH="$GOPATH" - export GOROOT="${GO_DIST}" - export GOCACHE="$GOCACHE" - export PATH="${GCC_PATH}:${GO_DIST}/bin:$PATH" - export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR}" - - make -s evg-test-enterprise-auth + ${PREPARE_SHELL} + bash ./etc/run-enterprise-gssapi-test.sh run-atlas-test: - command: ec2.assume_role @@ -513,7 +455,6 @@ functions: type: test params: shell: "bash" - silent: true working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} @@ -593,17 +534,9 @@ functions: params: shell: "bash" working_dir: src/go.mongodb.org/mongo-driver - add_expansions_to_env: true script: | ${PREPARE_SHELL} - - AUTH="auth" \ - SSL="ssl" \ - MONGODB_URI="${SERVERLESS_URI}" \ - SERVERLESS="serverless" \ - SERVERLESS_ATLAS_USER="${SERVERLESS_ATLAS_USER}" \ - MAKEFILE_TARGET=evg-test-serverless \ - sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + bash etc/run-serverless-test.sh run-atlas-data-lake-test: - command: shell.exec @@ -719,16 +652,13 @@ functions: - command: ec2.assume_role params: role_arn: ${aws_test_secrets_role} - - command: shell.exec + - command: subprocess.exec type: test params: include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - shell: "bash" - working_dir: "src" - script: | - ${PREPARE_SHELL} - cd $DRIVERS_TOOLS/.evergreen/auth_aws - ./setup_secrets.sh drivers/aws_auth + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh run-aws-auth-test-with-regular-aws-credentials: - command: shell.exec @@ -862,14 +792,13 @@ functions: working_dir: src/go.mongodb.org/mongo-driver binary: bash args: - - ${DRIVERS_TOOLS}/.evergreen/csfle/await_servers.sh + - ${DRIVERS_TOOLS}/.evergreen/csfle/await-servers.sh run-kms-tls-test: - command: shell.exec type: test params: shell: "bash" - silent: true working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} @@ -892,7 +821,6 @@ functions: type: test params: shell: "bash" - silent: true working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} @@ -1683,20 +1611,13 @@ tasks: tags: ["test", "enterprise-auth"] commands: - func: run-enterprise-auth-tests - vars: - MONGODB_URI: "${plain_auth_mongodb_uri}" - name: test-enterprise-auth-gssapi tags: ["test", "enterprise-auth"] commands: - func: run-enterprise-gssapi-auth-tests - - - name: test-enterprise-auth-gssapi-service-host - tags: ["test", "enterprise-auth"] - commands: - - func: run-enterprise-gssapi-service-host-auth-tests vars: - MONGO_GO_DRIVER_COMPRESSOR: "snappy" + MONGO_GO_DRIVER_COMPRESSOR: "snappy" # Build with the oldest supported version of Go. - name: go1.18-build @@ -1837,7 +1758,7 @@ tasks: PKG_CONFIG_PATH=$PKG_CONFIG_PATH \ make build-kms-test echo "Building build-kms-test ... end" - + source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh echo "Copying files ... begin" export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} @@ -1858,6 +1779,7 @@ tasks: working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} + source $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/secrets-export.sh export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} export GCPKMS_PROJECT=${GCPKMS_PROJECT} export GCPKMS_ZONE=${GCPKMS_ZONE} @@ -1892,23 +1814,10 @@ tasks: type: test params: shell: "bash" - silent: true working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} - echo "Building build-kms-test ... begin" - BUILD_TAGS="-tags=cse" \ - PKG_CONFIG_PATH=$PKG_CONFIG_PATH \ - make build-kms-test - echo "Building build-kms-test ... end" - - export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" - export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" - - LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ - MONGODB_URI='${atlas_free_tier_uri}' \ - PROVIDER='aws' \ - ./testkms + bash etc/run-awskms-test.sh - name: "testawskms-fail-task" # testawskms-fail-task runs without environment variables. @@ -1921,17 +1830,8 @@ tasks: working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} - echo "Building build-kms-test ... begin" - BUILD_TAGS="-tags=cse" \ - PKG_CONFIG_PATH=$PKG_CONFIG_PATH \ - make build-kms-test - echo "Building build-kms-test ... end" - - LD_LIBRARY_PATH=./install/libmongocrypt/lib64 \ - MONGODB_URI='${atlas_free_tier_uri}' \ - EXPECT_ERROR='status=400' \ - PROVIDER='aws' \ - ./testkms + export EXPECT_ERROR='status=400' + bash etc/run-awskms-test.sh - name: "testazurekms-task" commands: @@ -1966,8 +1866,7 @@ tasks: working_dir: src/go.mongodb.org/mongo-driver script: | ${PREPARE_SHELL} - # Get azurekms credentials from the vault. - . ./etc/get_aws_secrets.sh drivers/azurekms + source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey @@ -2011,14 +1910,13 @@ tasks: params: working_dir: src/go.mongodb.org/mongo-driver shell: bash - add_expansions_to_env: true env: TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/internal/test/faas/awslambda LAMBDA_STACK_NAME: dbx-go-lambda AWS_REGION: us-east-1 + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} - ls $TEST_LAMBDA_DIRECTORY ./.evergreen/run-deployed-lambda-aws-tests.sh - name: "test-search-index" @@ -2252,10 +2150,6 @@ task_groups: shell: "bash" script: | ${PREPARE_SHELL} - echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json - export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json - export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS - export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. - command: expansions.update @@ -2267,10 +2161,6 @@ task_groups: shell: "bash" script: | ${PREPARE_SHELL} - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh - func: handle-test-artifacts - func: cleanup @@ -2294,9 +2184,8 @@ task_groups: export AZUREKMS_VMNAME_PREFIX="GODRIVER" export AZUREKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS # Get azurekms credentials from the vault. - bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup_secrets.sh drivers/azurekms - source ./secrets-export.sh - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh + . ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/setup-secrets.sh + ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh - command: expansions.update params: file: testazurekms-expansions.yml @@ -2326,21 +2215,18 @@ task_groups: params: working_dir: src/go.mongodb.org/mongo-driver binary: bash - add_expansions_to_env: true env: LAMBDA_STACK_NAME: dbx-go-lambda AWS_REGION: us-east-1 + task_id: ${task_id} + execution: ${execution} args: - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: src/go.mongodb.org/mongo-driver/atlas-expansion.yml teardown_group: - command: subprocess.exec params: working_dir: src/go.mongodb.org/mongo-driver binary: bash - add_expansions_to_env: true env: LAMBDA_STACK_NAME: dbx-go-lambda AWS_REGION: us-east-1 @@ -2361,9 +2247,12 @@ task_groups: params: working_dir: src/go.mongodb.org/mongo-driver binary: bash - add_expansions_to_env: true env: MONGODB_VERSION: "7.0" + LAMBDA_STACK_NAME: dbx-go-lambda + AWS_REGION: us-east-1 + task_id: ${task_id} + execution: ${execution} args: - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - command: expansions.update @@ -2383,7 +2272,6 @@ task_groups: params: working_dir: src/go.mongodb.org/mongo-driver binary: bash - add_expansions_to_env: true args: - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh - func: handle-test-artifacts diff --git a/.evergreen/run-deployed-lambda-aws-tests.sh b/.evergreen/run-deployed-lambda-aws-tests.sh index 39b73516f4..bde083a732 100755 --- a/.evergreen/run-deployed-lambda-aws-tests.sh +++ b/.evergreen/run-deployed-lambda-aws-tests.sh @@ -2,6 +2,8 @@ # set -o errexit # Exit the script with error if any of the commands fail. +source ${DRIVERS_TOOLS}/.evergreen/atlas/secrets-export.sh + VARLIST=( AWS_REGION DRIVERS_TOOLS diff --git a/etc/get_aws_secrets.sh b/etc/get_aws_secrets.sh deleted file mode 100644 index 04475ded45..0000000000 --- a/etc/get_aws_secrets.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -# get-aws-secrets -# Gets AWS secrets from the vault -set -eu - -if [ -z "$DRIVERS_TOOLS" ]; then - echo "Please define DRIVERS_TOOLS variable" - exit 1 -fi - -bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup_secrets.sh "$@" -. ./secrets-export.sh diff --git a/etc/run-atlas-test.sh b/etc/run-atlas-test.sh index 9a1839993f..384c707732 100644 --- a/etc/run-atlas-test.sh +++ b/etc/run-atlas-test.sh @@ -5,7 +5,7 @@ set -eu set +x # Get the atlas secrets. -. etc/get_aws_secrets.sh drivers/atlas_connect +. ${DRIVERS_TOOLS}/.evergreen/atlas/setup-secrets.sh echo "Running cmd/testatlas/main.go" go run ./cmd/testatlas/main.go "$ATLAS_REPL" "$ATLAS_SHRD" "$ATLAS_FREE" "$ATLAS_TLS11" "$ATLAS_TLS12" "$ATLAS_SERVERLESS" "$ATLAS_SRV_REPL" "$ATLAS_SRV_SHRD" "$ATLAS_SRV_FREE" "$ATLAS_SRV_TLS11" "$ATLAS_SRV_TLS12" "$ATLAS_SRV_SERVERLESS" | tee test.suite diff --git a/etc/run-awskms-test.sh b/etc/run-awskms-test.sh new file mode 100644 index 0000000000..d3caef01c7 --- /dev/null +++ b/etc/run-awskms-test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# run-awskms-test +# Runs the awskms test. + +echo "Building build-kms-test ... begin" +BUILD_TAGS="-tags=cse" \ + PKG_CONFIG_PATH=$PKG_CONFIG_PATH \ + make build-kms-test +echo "Building build-kms-test ... end" + +. ${DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh drivers/atlas_connect +export MONGODB_URI="$ATLAS_FREE" + +if [ -z "${EXPECT_ERROR:-}" ]; then + . ${DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh + export AWS_SECRET_ACCESS_KEY=$FLE_AWS_SECRET_ACCESS_KEY + export AWS_ACCESS_KEY_ID=$FLE_AWS_ACCESS_KEY_ID +fi + +LD_LIBRARY_PATH=./install/libmongocrypt/lib64 PROVIDER='aws' ./testkms diff --git a/etc/run-enterprise-gssapi-test.sh b/etc/run-enterprise-gssapi-test.sh new file mode 100644 index 0000000000..d9bf57c7be --- /dev/null +++ b/etc/run-enterprise-gssapi-test.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# run-enterprise-gssapi-test +# Runs the enterprise auth tests with gssapi credentials. +set -eu + +. ${DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth +if [ "Windows_NT" = "${OS:-}" ]; then + export MONGODB_URI="mongodb://${PRINCIPAL/@/%40}:${SASL_PASS}@${SASL_HOST}:${SASL_PORT}/kerberos?authMechanism=GSSAPI" +else + echo "${KEYTAB_BASE64}" > /tmp/drivers.keytab.base64 + base64 --decode /tmp/drivers.keytab.base64 > ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab + mkdir -p ~/.krb5 + cat .evergreen/krb5.config | tee -a ~/.krb5/config + kinit -k -t ${PROJECT_DIRECTORY}/.evergreen/drivers.keytab -p "${PRINCIPAL}" + export MONGODB_URI="mongodb://${PRINCIPAL/@/%40}@${SASL_HOST}:${SASL_PORT}/kerberos?authMechanism=GSSAPI" +fi +export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR:-}" + +make -s evg-test-enterprise-auth diff --git a/etc/run-enterprise-plain-test.sh b/etc/run-enterprise-plain-test.sh new file mode 100644 index 0000000000..b508a8584d --- /dev/null +++ b/etc/run-enterprise-plain-test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# run-enterprise-gssapi-test +# Runs the enterprise auth tests with plain credentials. +set -eu +. ${DRIVERS_TOOLS}/.evergreen/secrets_handling/setup-secrets.sh drivers/enterprise_auth +export MONGODB_URI="mongodb://${SASL_USER}:${SASL_PASS}@${SASL_HOST}:${SASL_PORT}/ldap?authMechanism=PLAIN" +export MONGO_GO_DRIVER_COMPRESSOR="${MONGO_GO_DRIVER_COMPRESSOR:-}" +make -s evg-test-enterprise-auth diff --git a/etc/run-serverless-test.sh b/etc/run-serverless-test.sh new file mode 100644 index 0000000000..9d858a0610 --- /dev/null +++ b/etc/run-serverless-test.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# run-serverless-test +# Runs the serverless tests. +source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh +AUTH="auth" \ + SSL="ssl" \ + MONGODB_URI="${SERVERLESS_URI}" \ + SERVERLESS="serverless" \ + MAKEFILE_TARGET=evg-test-serverless \ + sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh diff --git a/etc/setup-encryption.sh b/etc/setup-encryption.sh index 8a049415f1..d8ab0e855f 100644 --- a/etc/setup-encryption.sh +++ b/etc/setup-encryption.sh @@ -16,5 +16,5 @@ export CSFLE_TLS_CA_FILE="${PARENT_DIR}/testdata/kmip-certs/ca-ec.pem" export CSFLE_TLS_CERT_FILE="${PARENT_DIR}/testdata/kmip-certs/server-ec.pem" export CSFLE_TLS_CLIENT_CERT_FILE="${PARENT_DIR}/testdata/kmip-certs/client-ec.pem" -bash $DRIVERS_TOOLS/.evergreen/csfle/setup_secrets.sh -bash $DRIVERS_TOOLS/.evergreen/csfle/start_servers.sh +bash $DRIVERS_TOOLS/.evergreen/csfle/setup-secrets.sh +bash $DRIVERS_TOOLS/.evergreen/csfle/start-servers.sh