GODRIVER-3100 [master] Use AWS Secrets Manager for CSFLE Tests (#1532)

mongodb · Feb 6, 2024 · e77ab70 · e77ab70
1 parent 6f1577f
commit e77ab70
Show file tree

Hide file tree

Showing 8 changed files with 129 additions and 240 deletions.
diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -123,7 +123,7 @@ functions:
              export UPLOAD_BUCKET="$UPLOAD_BUCKET"
              export PROJECT="$PROJECT"
              export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
-             export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
+             export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig
              export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
              export PATH="$PATH"
           EOT
@@ -299,6 +299,13 @@ functions:
           # Attempt to shut down a running load balancer. Ignore any errors that happen if the load
           # balancer is not running.
           DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop || echo "Ignoring load balancer stop error"
+    - command: shell.exec
+      params:
+        shell: "bash"
+        script: |
+          ${PREPARE_SHELL}
+          # Clean up cse servers
+          bash ${DRIVERS_TOOLS}/.evergreen/csfle/stop_servers.sh
     - command: shell.exec
       params:
         shell: "bash"
@@ -309,6 +316,7 @@ functions:
           cd -
           rm -rf $DRIVERS_TOOLS || true
 
+
   fix-absolute-paths:
     - command: shell.exec
       params:
@@ -509,27 +517,7 @@ functions:
         working_dir: src/go.mongodb.org/mongo-driver
         script: |
           ${PREPARE_SHELL}
-
-          # Set temp credentials for AWS.
-          export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
-          export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
-          export AWS_DEFAULT_REGION="us-east-1"
-
-          # Set client-side encryption credentials.
-          export CSFLE_TLS_CA_FILE="$PROJECT_DIRECTORY/testdata/kmip-certs/ca-ec.pem"
-          export CSFLE_TLS_CERTIFICATE_KEY_FILE="$PROJECT_DIRECTORY/testdata/kmip-certs/client-ec.pem"
-
-          ${PYTHON3_BINARY} -m venv ./venv
-          ./venv/${VENV_BIN_DIR|bin}/pip3 install boto3
-
-          # Set the PYTHON environment variable to point to the active python3 binary. This is used by the
-          # set-temp-creds.sh script.
-          if [ "Windows_NT" = "$OS" ]; then
-            export PYTHON="$(pwd)/venv/Scripts/python"
-          else
-            export PYTHON="$(pwd)/venv/bin/python"
-          fi
-          . ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh
+          source ./secrets-export.sh
 
           if [ "${SKIP_CRYPT_SHARED_LIB}" = "true" ]; then
             CRYPT_SHARED_LIB_PATH=""
@@ -547,17 +535,6 @@ functions:
           TOPOLOGY="${TOPOLOGY}" \
           MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
           BUILD_TAGS="-tags=cse" \
-          AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" \
-          AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" \
-          AWS_DEFAULT_REGION="us-east-1" \
-          CSFLE_AWS_TEMP_ACCESS_KEY_ID="$CSFLE_AWS_TEMP_ACCESS_KEY_ID" \
-          CSFLE_AWS_TEMP_SECRET_ACCESS_KEY="$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" \
-          CSFLE_AWS_TEMP_SESSION_TOKEN="$CSFLE_AWS_TEMP_SESSION_TOKEN" \
-          AZURE_TENANT_ID="${cse_azure_tenant_id}" \
-          AZURE_CLIENT_ID="${cse_azure_client_id}" \
-          AZURE_CLIENT_SECRET="${cse_azure_client_secret}" \
-          GCP_EMAIL="${cse_gcp_email}" \
-          GCP_PRIVATE_KEY="${cse_gcp_private_key}" \
           REQUIRE_API_VERSION="${REQUIRE_API_VERSION}" \
           CRYPT_SHARED_LIB_PATH="$CRYPT_SHARED_LIB_PATH" \
           make evg-test-versioned-api \
@@ -867,91 +844,24 @@ functions:
           export AWS_ROLE_SESSION_NAME="test"
           ${PROJECT_DIRECTORY}/.evergreen/run-mongodb-aws-test.sh web-identity
 
-  start-kms-mock-server:
-    - command: shell.exec
-      params:
-        shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-
-          cd ${DRIVERS_TOOLS}/.evergreen/csfle
-          . ./activate-kmstlsvenv.sh
-    - command: shell.exec
-      params:
-        shell: "bash"
-        background: true
-        script: |
-          cd ${DRIVERS_TOOLS}/.evergreen/csfle
-          ./kmstlsvenv/bin/python3 -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/${CERT_FILE} --port ${PORT}
-
-  start-kms-mock-server-require-client-cert:
-    - command: shell.exec
-      params:
-        shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-
-          cd ${DRIVERS_TOOLS}/.evergreen/csfle
-          . ./activate-kmstlsvenv.sh
-    - command: shell.exec
-      params:
-        shell: "bash"
-        background: true
-        script: |
-          cd ${DRIVERS_TOOLS}/.evergreen/csfle
-          ./kmstlsvenv/bin/python3 -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/${CERT_FILE} --port ${PORT} --require_client_cert
-
   start-cse-servers:
-    - command: shell.exec
-      params:
-        shell: "bash"
-        script: |
-          ${PREPARE_SHELL}
-
-          cd ${DRIVERS_TOOLS}/.evergreen/csfle
-          . ./activate-kmstlsvenv.sh
-
-    - command: shell.exec
+    - command: ec2.assume_role
       params:
-        shell: "bash"
-        background: true
-        script: |
-          cd ${DRIVERS_TOOLS}/.evergreen/csfle
-          . ./activate-kmstlsvenv.sh
-          python -u kms_kmip_server.py \
-              --port 5698 \
-              --ca_file "${PROJECT_DIRECTORY}/testdata/kmip-certs/ca-ec.pem" \
-              --cert_file "${PROJECT_DIRECTORY}/testdata/kmip-certs/server-ec.pem"
-
-    - command: shell.exec
+        role_arn: ${aws_test_secrets_role}
+    - command: subprocess.exec
       params:
-        shell: "bash"
+        working_dir: src/go.mongodb.org/mongo-driver
+        binary: bash
         background: true
-        script: |
-          cd ${DRIVERS_TOOLS}/.evergreen/csfle
-          . ./activate-kmstlsvenv.sh
-          python bottle.py fake_azure:imds
-
-    - command: shell.exec
+        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN", "DRIVERS_TOOLS"]
+        args:
+          - etc/setup-encryption.sh
+    - command: subprocess.exec
       params:
-        script: |
-          # Ensure mock servers are running before starting tests.
-          await_server() {
-            for i in $(seq 300); do
-                # Exit code 7: "Failed to connect to host".
-                if curl -s "localhost:$2"; test $? -ne 7; then
-                  return 0
-                else
-                  sleep 1
-                fi
-            done
-            echo "could not detect '$1' server on port $2"
-          }
-          # * List servers to await here ...
-          await_server "KMS", 5698
-          await_server "Azure", 8080
-
-          echo "finished awaiting servers"
+        working_dir: src/go.mongodb.org/mongo-driver
+        binary: bash
+        args:
+          - ${DRIVERS_TOOLS}/.evergreen/csfle/await_servers.sh
 
   run-kms-tls-test:
     - command: shell.exec
@@ -962,6 +872,7 @@ functions:
         working_dir: src/go.mongodb.org/mongo-driver
         script: |
           ${PREPARE_SHELL}
+          source ./secrets-export.sh
           export KMS_TLS_TESTCASE="${KMS_TLS_TESTCASE}"
 
           AUTH="${AUTH}" \
@@ -970,13 +881,6 @@ functions:
           TOPOLOGY="${TOPOLOGY}" \
           MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
           BUILD_TAGS="-tags=cse" \
-          AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" \
-          AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" \
-          AZURE_TENANT_ID="${cse_azure_tenant_id}" \
-          AZURE_CLIENT_ID="${cse_azure_client_id}" \
-          AZURE_CLIENT_SECRET="${cse_azure_client_secret}" \
-          GCP_EMAIL="${cse_gcp_email}" \
-          GCP_PRIVATE_KEY="${cse_gcp_private_key}" \
           make evg-test-kms \
           PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH
@@ -990,6 +894,7 @@ functions:
         working_dir: src/go.mongodb.org/mongo-driver
         script: |
           ${PREPARE_SHELL}
+          source ./secrets-export.sh
           export KMS_MOCK_SERVERS_RUNNING="true"
 
           AUTH="${AUTH}" \
@@ -998,15 +903,6 @@ functions:
           TOPOLOGY="${TOPOLOGY}" \
           MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
           BUILD_TAGS="-tags=cse" \
-          AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}" \
-          AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}" \
-          AZURE_TENANT_ID="${cse_azure_tenant_id}" \
-          AZURE_CLIENT_ID="${cse_azure_client_id}" \
-          AZURE_CLIENT_SECRET="${cse_azure_client_secret}" \
-          GCP_EMAIL="${cse_gcp_email}" \
-          GCP_PRIVATE_KEY="${cse_gcp_private_key}" \
-          CSFLE_TLS_CA_FILE="$PROJECT_DIRECTORY/testdata/kmip-certs/ca-ec.pem"
-          CSFLE_TLS_CERTIFICATE_KEY_FILE="$PROJECT_DIRECTORY/testdata/kmip-certs/client-ec.pem"
           make evg-test-kmip \
           PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH
@@ -1877,10 +1773,7 @@ tasks:
           TOPOLOGY: "server"
           AUTH: "noauth"
           SSL: "nossl"
-      - func: start-kms-mock-server
-        vars:
-          CERT_FILE: "expired.pem"
-          PORT: 8000
+      - func: start-cse-servers
       - func: run-kms-tls-test
         vars:
           KMS_TLS_TESTCASE: "INVALID_CERT"
@@ -1896,10 +1789,7 @@ tasks:
           TOPOLOGY: "server"
           AUTH: "noauth"
           SSL: "nossl"
-      - func: start-kms-mock-server
-        vars:
-          CERT_FILE: "wrong-host.pem"
-          PORT: 8000
+      - func: start-cse-servers
       - func: run-kms-tls-test
         vars:
           KMS_TLS_TESTCASE: "INVALID_HOSTNAME"
@@ -1915,18 +1805,7 @@ tasks:
           TOPOLOGY: "server"
           AUTH: "noauth"
           SSL: "nossl"
-      - func: start-kms-mock-server
-        vars:
-          CERT_FILE: "expired.pem"
-          PORT: 8000
-      - func: start-kms-mock-server
-        vars:
-          CERT_FILE: "wrong-host.pem"
-          PORT: 8001
-      - func: start-kms-mock-server-require-client-cert
-        vars:
-          CERT_FILE: "server.pem"
-          PORT: 8002
+      - func: start-cse-servers
       - func: run-kmip-tests
         vars:
           TOPOLOGY: "server"

diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh
@@ -10,7 +10,7 @@ if [ -z $DRIVERS_TOOLS ]; then
   export DRIVERS_TOOLS="$(dirname $(dirname $(dirname `pwd`)))/drivers-tools"
 fi
 
-if [ "Windows_NT" = "$OS" ]; then
+if [ "Windows_NT" = "${OS:-}" ]; then
     export GOPATH=$(cygpath -m $GOPATH)
     export GOCACHE=$(cygpath -m $GOCACHE)
     export DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
@@ -19,9 +19,17 @@ fi
 export GOROOT="${GOROOT}"
 export PATH="${GOROOT}/bin:${GCC_PATH}:$GOPATH/bin:$PATH"
 export PROJECT="${project}"
-export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig:$(pwd)/install/mongo-c-driver/lib/pkgconfig
+export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig
 export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
 
+if [ "$(uname -s)" = "Darwin" ]; then
+  export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib/pkgconfig
+  export DYLD_FALLBACK_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib
+else
+  export PKG_CONFIG_PATH=$(pwd)/install/libmongocrypt/lib64/pkgconfig
+  export LD_LIBRARY_PATH=$(pwd)/install/libmongocrypt/lib64
+fi
+
 SSL=${SSL:-nossl}
 if [ "$SSL" != "nossl" -a -z "${SERVERLESS+x}" ]; then
     export MONGO_GO_DRIVER_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
@@ -37,33 +45,8 @@ if [ "$SSL" != "nossl" -a -z "${SERVERLESS+x}" ]; then
     fi
 fi
 
-if [ -z ${AWS_ACCESS_KEY_ID+x} ]; then
-  export AWS_ACCESS_KEY_ID="${cse_aws_access_key_id}"
-  export AWS_SECRET_ACCESS_KEY="${cse_aws_secret_access_key}"
-fi
-
-# Set temp credentials for AWS if python3 is available.
-#
-# Using python3-venv in Ubuntu 14.04 (an OS required for legacy server version
-# tasks) requires the use of apt-get, which we wish to avoid. So, we do not set
-# a python3 binary on Ubuntu 14.04. Setting AWS temp credentials for legacy
-# server version tasks is unnecessary, as temp credentials are only needed on 4.2+.
-if [ ! -z ${PYTHON3_BINARY} ]; then
-  export AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}"
-  export AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}"
-  export AWS_DEFAULT_REGION="us-east-1"
-  ${PYTHON3_BINARY} -m venv ./venv
-
-  # Set the PYTHON environment variable to point to the active python3 binary. This is used by the
-  # set-temp-creds.sh script.
-  if [ "Windows_NT" = "$OS" ]; then
-    export PYTHON="$(pwd)/venv/Scripts/python"
-  else
-    export PYTHON="$(pwd)/venv/bin/python"
-  fi
-
-  ./venv/${VENV_BIN_DIR:-bin}/pip3 install boto3
-  . ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh
+if [ -f "secrets-export.sh" ]; then
+    source $(pwd)/secrets-export.sh
 fi
 
 # If GO_BUILD_TAGS is not set, set the default Go build tags to "cse" to enable
@@ -72,6 +55,17 @@ if [ -z ${GO_BUILD_TAGS+x} ]; then
   GO_BUILD_TAGS="cse"
 fi
 
+if [[ $GO_BUILD_TAGS == *"cse"* ]]; then
+  if [ "Windows_NT" = "$OS" ]; then
+    if [ ! -d /cygdrive/c/libmongocrypt/bin ]; then
+      bash $(pwd)/etc/install-libmongocrypt.sh
+    fi
+    export PATH=$PATH:/cygdrive/c/libmongocrypt/bin
+  elif [ ! -d "$PKG_CONFIG_PATH" ]; then
+    bash $(pwd)/etc/install-libmongocrypt.sh
+  fi
+fi
+
 if [ "${SKIP_CRYPT_SHARED_LIB}" = "true" ]; then
   CRYPT_SHARED_LIB_PATH=""
   echo "crypt_shared library is skipped"
@@ -82,14 +76,6 @@ else
   echo "crypt_shared library will be loaded from path: $CRYPT_SHARED_LIB_PATH"
 fi
 
-CSFLE_TLS_CA_FILE="$(pwd)/testdata/kmip-certs/ca-ec.pem"
-CSFLE_TLS_CERTIFICATE_KEY_FILE="$(pwd)/testdata/kmip-certs/client-ec.pem"
-
-if [ "Windows_NT" = "$OS" ]; then
-  CSFLE_TLS_CA_FILE=$(cygpath -m $CSFLE_TLS_CA_FILE)
-  CSFLE_TLS_CERTIFICATE_KEY_FILE=$(cygpath -m $CSFLE_TLS_CERTIFICATE_KEY_FILE)
-fi
-
 if [ -z ${MAKEFILE_TARGET+x} ]; then
   if [ "$(uname -s)" = "Darwin" ]; then
       # Run a subset of the tests on Darwin
@@ -109,20 +95,8 @@ MONGODB_URI="${MONGODB_URI}" \
 TOPOLOGY=${TOPOLOGY} \
 MONGO_GO_DRIVER_COMPRESSOR=${MONGO_GO_DRIVER_COMPRESSOR} \
 BUILD_TAGS="${RACE} -tags=${GO_BUILD_TAGS}" \
-AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
-AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
-AWS_DEFAULT_REGION="us-east-1" \
-CSFLE_AWS_TEMP_ACCESS_KEY_ID="$CSFLE_AWS_TEMP_ACCESS_KEY_ID" \
-CSFLE_AWS_TEMP_SECRET_ACCESS_KEY="$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY" \
-CSFLE_AWS_TEMP_SESSION_TOKEN="$CSFLE_AWS_TEMP_SESSION_TOKEN" \
-AZURE_TENANT_ID="${cse_azure_tenant_id}" \
-AZURE_CLIENT_ID="${cse_azure_client_id}" \
-AZURE_CLIENT_SECRET="${cse_azure_client_secret}" \
-GCP_EMAIL="${cse_gcp_email}" \
-GCP_PRIVATE_KEY="${cse_gcp_private_key}" \
-CSFLE_TLS_CA_FILE="$CSFLE_TLS_CA_FILE" \
-CSFLE_TLS_CERTIFICATE_KEY_FILE="$CSFLE_TLS_CERTIFICATE_KEY_FILE" \
 CRYPT_SHARED_LIB_PATH=$CRYPT_SHARED_LIB_PATH \
 PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
 LD_LIBRARY_PATH=$LD_LIBRARY_PATH \
+MACOS_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH \
 make $MAKEFILE_TARGET