GODRIVER-2924 Add docker file support

.evergreen/auth_oidc/Dockerfile

@@ -1,20 +1,6 @@
-FROM ubuntu:20.04
+FROM drivers-evergreen-tools
 
-RUN apt-get -qq update && apt-get -qqy -o DPkg::Lock::Timeout=-1 install --no-install-recommends \
-  git \
-  ca-certificates \
-  curl \
-  wget \
-  sudo \
-  gnupg \
-  python \
-  python3 \
-  python3-virtualenv \
-  lsof \
-  libsnmp35 \
-  net-tools \
- && rm -rf /var/lib/apt/lists/*
+COPY ./docker_entry.sh /root/docker_entry.sh
+COPY ./docker_entry_base.sh /root/docker_entry_base.sh
 
-COPY ./docker_entry.sh /home/root/docker_entry.sh
-
-ENTRYPOINT ["/bin/bash", "/home/root/docker_entry.sh"]
+ENV TOPOLOGY=replica_set

.evergreen/auth_oidc/docker_entry.sh

@@ -3,22 +3,6 @@
 # Entry point for Dockerfile for launching an oidc-enabled server.
 #
 set -eux
-export MONGODB_VERSION=latest
-export TOPOLOGY=replica_set
-export ORCHESTRATION_FILE=auth-oidc.json
-export DRIVERS_TOOLS=$HOME/drivers-evergreen-tools
-export PROJECT_ORCHESTRATION_HOME=$DRIVERS_TOOLS/.evergreen/orchestration
-export MONGO_ORCHESTRATION_HOME=$HOME
-export NO_IPV6=${NO_IPV6:-""}
 
-if [ ! -d $DRIVERS_TOOLS ]; then
-    git clone --branch DRIVERS-2415 https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS
-fi
-
-cd $DRIVERS_TOOLS/.evergreen/auth_oidc
-. ./activate-authoidcvenv.sh
-python oidc_write_orchestration.py
-
-bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh
-$DRIVERS_TOOLS/mongodb/bin/mongosh $DRIVERS_TOOLS/.evergreen/auth_oidc/setup_oidc.js
-tail -f $MONGO_ORCHESTRATION_HOME/server.log
+bash /root/docker_entry_base.sh
+tail -f $MONGO_ORCHESTRATION_HOME/server.log

.evergreen/auth_oidc/docker_entry_base.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# Entry point for Dockerfile for launching an oidc-enabled server.
+#
+set -eu
+export ORCHESTRATION_FILE=auth-oidc.json
+
+cd $DRIVERS_TOOLS/.evergreen/auth_oidc
+. ./activate-authoidcvenv.sh
+python oidc_write_orchestration.py
+
+bash $DRIVERS_TOOLS/.evergreen/run-orchestration.sh
+$DRIVERS_TOOLS/mongodb/bin/mongosh $DRIVERS_TOOLS/.evergreen/auth_oidc/setup_oidc.js
+tail -f $MONGO_ORCHESTRATION_HOME/server.log

.evergreen/auth_oidc/oidc_write_orchestration.py

@@ -53,7 +53,7 @@ def azure():
     orch_file = os.path.abspath(os.path.join(HERE, '..', 'orchestration', 'configs', 'servers', 'auth-oidc.json'))
     with open(orch_file, 'w') as fid:
         json.dump(data, fid, indent=4)
-
+    print(f"Wrote OIDC config to {orch_file}")
 
 def main():
     print("Bootstrapping OIDC config")
@@ -126,6 +126,7 @@ def main():
     orch_file = os.path.abspath(os.path.join(HERE, '..', 'orchestration', 'configs', 'replica_sets', 'auth-oidc.json'))
     with open(orch_file, 'w') as fid:
         json.dump(data, fid, indent=4)
+    print(f"Wrote OIDC config to {orch_file}")
 
 
 if __name__ == '__main__':

.evergreen/auth_oidc/start_local_server.sh

@@ -5,16 +5,30 @@
 # prequisites and usage.
 #
 set -eux
-if [[ -z "${AWS_ROLE_ARN}" ||  -z "${AWS_ACCESS_KEY_ID}" || -z "${AWS_SECRET_ACCESS_KEY}" ]]; then
-    echo "Missing AWS credentials"
-    exit 1
-fi
 
 DRIVERS_TOOLS=${DRIVERS_TOOLS:-$(readlink -f ../..)}
-echo "Drivers tools: $DRIVERS_TOOLS"
+ENTRYPOINT=${ENTRYPOINT:-/root/docker_entry.sh}
+USE_TTY=""
+VOL="-v ${DRIVERS_TOOLS}:/root/drivers-evergreen-tools"
+
+if [ -z "$AWS_PROFILE" ]; then
+    if [[ -z "${AWS_SESSION_TOKEN}" ||  -z "${AWS_ACCESS_KEY_ID}" || -z "${AWS_SECRET_ACCESS_KEY}" ]]; then
+        echo "Please set AWS_PROFILE or set AWS credentials environment variables"
+       exit 1
+    fi
+    ENV="-e AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID"
+    ENV="$ENV -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY"
+else
+    ENV="-e AWS_PROFILE=$AWS_PROFILE"
+    VOL="$VOL -v $HOME/.aws:/root/.aws"
+fi
+
 rm -rf $DRIVERS_TOOLS/.evergreen/auth_oidc/authoidcvenv
-rm -rf $DRIVERS_TOOLS/mongodb
-rm -rf $DRIVERS_TOOLS/legacy-shell-download
-rm -rf $DRIVERS_TOOLS/mongosh
+test -t 1 && USE_TTY="-t"
+
+echo "Drivers tools: $DRIVERS_TOOLS"
+pushd ../docker
+docker build -t drivers-evergreen-tools ./ubuntu20.04
+popd
 docker build -t oidc-test .
-docker run -it -v ${DRIVERS_TOOLS}:/home/root/drivers-evergreen-tools -p 27017:27017 -p 27018:27018 -e HOME=/home/root -e AWS_ROLE_ARN=${AWS_ROLE_ARN} -e AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} -e AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} -e NO_IPV6=true oidc-test
+docker run --rm -i $USE_TTY $VOL $ENV -p 27017:27017 -p 27018:27018 oidc-test $ENTRYPOINT

.evergreen/config.yml

@@ -363,6 +363,29 @@ functions:
           CRYPT_SHARED_LIB_PATH="${CRYPT_SHARED_LIB_PATH}" \
             sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
+  "run docker test":
+    - command: shell.exec
+      params:
+        shell: "bash"
+        script: |
+          ${PREPARE_SHELL}
+          cd $DRIVERS_TOOLS/.evergreen/docker
+          ENTRYPOINT=/root/test-entrypoint.sh bash run-local.sh
+
+  "run oidc test":
+    - command: ec2.assume_role
+      params:
+        role_arn: ${aws_test_secrets_role}
+    - command: shell.exec
+      params:
+        shell: "bash"
+        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
+        script: |
+          ${PREPARE_SHELL}
+          cd $DRIVERS_TOOLS/.evergreen/auth_oidc
+          export ENTRYPOINT=/root/docker_entry_base.sh
+          bash start_local_server.sh
+
   "cleanup":
     - command: shell.exec
       params:
@@ -722,6 +745,16 @@ tasks:
       commands:
         - func: "run serverless tests"
 
+    - name: "test-docker"
+      tags: ["latest"]
+      commands:
+        - func: "run docker test"
+
+    - name: "test-oidc"
+      tags: ["latest"]
+      command:
+        - func: "run oidc test"
+
 # }}}
 
 
@@ -943,6 +976,8 @@ buildvariants:
      - name: "test-3.6-replica_set"
      - name: "test-3.6-sharded_cluster"
      - name: "test-3.6-standalone"
+     - name: "test-docker"
+     - name: "test-oidc"
 
 - matrix_name: "tests-nossl"
   matrix_spec: {"os-nossl": "*", auth: "*", ssl: "nossl" }
@@ -1073,4 +1108,3 @@ buildvariants:
       # Debian 7.1 does not support MongoDB 2.4
       # SUSE12 x86_64 is only supported by MongoDB 3.2+
       # vim: set et sw=2 ts=2 :
-

.evergreen/docker/README.md

@@ -0,0 +1,43 @@
+# Drivers Evergreen Tools Dockerfile
+
+The ``Dockerfile`` and scripts in the subdirector(ies) serve dual purposes.
+
+- Run a local server in docker container.
+- Extend and run a driver test in a docker container.
+
+You will need Docker (or podman aliased to Docker) installed and running
+locally.
+
+# Run Local Server
+
+To run a local server, change to this directory and run:
+
+```bash
+bash ./run-local.sh
+```
+
+This will build the docker image and run it with appropriate settings.
+Note that any of the environment variables used by `run-orchestration`
+will be passed through to the container.
+The appropriate port(s) will be exposed, allowing you to run local test against
+the running docker container.
+
+The default image can be overridden with `IMAGE``, and the entrypoint with `ENTRYPOINT`.
+To use a specific architecture, use `PLATFORM`, e.g. `--platform linux/amd64`.
+
+## Driver Testing in Docker
+
+To extend this image and run against a driver test suite, first build the
+image locally.
+
+```bash
+docker build -t drivers-evergreen-tools .
+```
+
+Then, in your `Dockerfile`, use `FROM drivers-evergreen-tools`.
+
+When running your derived image, use `-v $DRIVERS_TOOLS:/root/drivers-evergreen-tools`
+to use the local checkout.
+
+In your entry point script, run `run-orchestration.sh` before running your test suite.
+Note that you will probably want to expose the environment variables as is done in `run-local.sh`.

.evergreen/docker/run-local.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+#
+# Run a local MongoDB orchestration inside a docker container
+#
+set -eu
+
+NAME=drivers-evergreen-tools
+ENTRYPOINT=${ENTRYPOINT:-/root/local-entrypoint.sh}
+IMAGE=${TARGET_IMAGE:-ubuntu20.04}
+PLATFORM=${DOCKER_PLATFORM:-}
+# e.g. --platform linux/amd64
+
+docker build $PLATFORM -t $NAME $IMAGE
+pushd ../..
+
+AUTH=${AUTH:-noauth}
+SSL=${SSL:-nossl}
+TOPOLOGY=${TOPOLOGY:-server}
+LOAD_BALANCER=${LOAD_BALANCER:-}
+STORAGE_ENGINE=${STORAGE_ENGINE:-}
+REQUIRE_API_VERSION=${REQUIRE_API_VERSION:-}
+DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-}
+MONGODB_VERSION=${MONGODB_VERSION:-latest}
+MONGODB_DOWNLOAD_URL=${MONGODB_DOWNLOAD_URL:-}
+ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-basic.json}
+
+ENV="-e MONGODB_VERSION=$MONGODB_VERSION"
+ENV="$ENV -e TOPOLOGY=$TOPOLOGY"
+ENV="$ENV -e AUTH=$AUTH"
+ENV="$ENV -e SSL=$SSL"
+ENV="$ENV -e ORCHESTRATION_FILE=$ORCHESTRATION_FILE"
+ENV="$ENV -e LOAD_BALANCER=$LOAD_BALANCER"
+ENV="$ENV -e STORAGE_ENGINE=$STORAGE_ENGINE"
+ENV="$ENV -e REQUIRE_API_VERSION=$REQUIRE_API_VERSION"
+ENV="$ENV -e DISABLE_TEST_COMMANDS=$DISABLE_TEST_COMMANDS"
+ENV="$ENV -e MONGODB_DOWNLOAD_URL=$MONGODB_DOWNLOAD_URL"
+
+if [ "$TOPOLOGY" == "server" ]; then
+    PORT="-p 27017:2017"
+else
+    PORT="-p 27017:2017 -p 27018:2018 -p 27019:2019"
+fi
+USE_TTY=""
+test -t 1 && USE_TTY="-t"
+VOL="-v `pwd`:/root/drivers-evergreen-tools"
+
+docker run $PLATFORM --rm $ENV $PORT $VOL -i $USE_TTY $NAME $ENTRYPOINT

.evergreen/docker/ubuntu20.04/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:20.04
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+  apt-get -qq update && apt-get -qqy install --no-install-recommends \
+  git \
+  ca-certificates \
+  curl \
+  wget \
+  sudo \
+  gnupg \
+  python \
+  python3 \
+  python3.8-venv \
+  lsof \
+  software-properties-common \
+  libsnmp35 \
+  net-tools \
+ && rm -rf /var/lib/apt/lists/*
+
+ENV DRIVERS_TOOLS=/root/drivers-evergreen-tools
+ENV PROJECT_ORCHESTRATION_HOME=/root/drivers-evergreen-tools/.evergreen/orchestration
+ENV MONGODB_BINARIES=/root/drivers-evergreen-tools/mongodb/bin
+ENV MONGO_ORCHESTRATION_HOME=/root
+ENV SKIP_LEGACY_SHELL=1
+ENV DOCKER_RUNNING=true
+
+COPY ./local-entrypoint.sh /root/local-entrypoint.sh
+COPY ./test-entrypoint.sh /root/test-entrypoint.sh

.evergreen/docker/ubuntu20.04/local-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -eu
+
+cd $DRIVERS_TOOLS/.evergreen
+bash run-orchestration.sh
+tail -f $MONGO_ORCHESTRATION_HOME/server.log

.evergreen/docker/ubuntu20.04/test-entrypoint.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+set -eu
+
+cd $DRIVERS_TOOLS/.evergreen
+bash run-orchestration.sh
+echo "Success!"

.evergreen/download-mongodb.sh

@@ -678,14 +678,18 @@ download_and_extract_package ()
    EXTRACT=$2
 
    cd $DRIVERS_TOOLS
+   echo "Installing server binaries..."
    curl_retry $MONGODB_DOWNLOAD_URL --output mongodb-binaries.tgz
+
    $EXTRACT mongodb-binaries.tgz
+   echo "Installing server binaries... done."
 
    rm -f mongodb-binaries.tgz
+   rm -rf mongodb
    mv mongodb* mongodb
    chmod -R +x mongodb
    find . -name vcredist_x64.exe -exec {} /install /quiet \;
-   ./mongodb/bin/mongod --version
+   echo "MongoDB server version: $(./mongodb/bin/mongod --version)"
    cd -
 }
 
@@ -695,15 +699,17 @@ download_and_extract_mongosh ()
    EXTRACT_MONGOSH=$2
 
    cd $DRIVERS_TOOLS
+   echo "Installing MongoDB shell..."
    curl_retry $MONGOSH_DOWNLOAD_URL --output mongosh.tgz
    $EXTRACT_MONGOSH mongosh.tgz
 
    rm -f mongosh.tgz
+   rm -rf mongosh
    mv mongosh-* mongosh
    mv mongosh/bin/* mongodb/bin
    chmod -R +x mongodb/bin
-   echo "MongoDB shell installed. Version:"
-   ./mongodb/bin/mongosh --version
+   echo "Installing MongoDB shell... done."
+   echo "MongoDB shell version: $(./mongodb/bin/mongosh --version)"
    cd -
 }
 
@@ -762,6 +768,7 @@ download_and_extract_crypt_shared ()
    MONGO_CRYPT_SHARED_DOWNLOAD_URL=$1
    EXTRACT=$2
    __CRYPT_SHARED_LIB_PATH=${3:-CRYPT_SHARED_LIB_PATH}
+   rm -rf crypt_shared_download
    mkdir crypt_shared_download
    cd crypt_shared_download
    curl_retry $MONGO_CRYPT_SHARED_DOWNLOAD_URL --output crypt_shared-binaries.tgz