Usability improvements (redux) (#477)

Co-authored-by: Ezra Chung <88335979+eramongodb@users.noreply.github.com>
mongodb-labs · Aug 27, 2024 · ff8d3fa · ff8d3fa
1 parent f2384ff
commit ff8d3fa
Show file tree

Hide file tree

Showing 17 changed files with 104 additions and 59 deletions.
diff --git a/.evergreen/atlas/atlas-utils.sh b/.evergreen/atlas/atlas-utils.sh
@@ -64,9 +64,7 @@ check_deployment ()
   ATLAS_BASE_URL=${ATLAS_BASE_URL:-"https://account-dev.mongodb.com/api/atlas/v1.0"}
   TYPE=${DEPLOYMENT_TYPE:-"clusters"}
 
-  echo "Finding Python3 binary..." 1>&2
-  PYTHON="$(find_python3 2>/dev/null)"
-  echo "Finding Python3 binary... done." 1>&2
+  PYTHON=$(ensure_python3)
 
   # Don't try longer than 20 minutes.
   echo "" 1>&2

diff --git a/.evergreen/auth_aws/activate-authawsvenv.sh b/.evergreen/auth_aws/activate-authawsvenv.sh
@@ -32,10 +32,7 @@ activate_authawsvenv() {
   else
     # shellcheck source=.evergreen/find-python3.sh
     . ../find-python3.sh || return
-
-    echo "Finding Python3 binary..."
-    PYTHON="$(find_python3 2>/dev/null)" || return
-    echo "Finding Python3 binary... done."
+    PYTHON=$(ensure_python3) || return
 
     echo "Creating virtual environment 'authawsvenv'..."
     venvcreate "${PYTHON:?}" authawsvenv || return

diff --git a/.evergreen/auth_oidc/activate-authoidcvenv.sh b/.evergreen/auth_oidc/activate-authoidcvenv.sh
@@ -32,10 +32,7 @@ activate_authoidcvenv() {
   else
     # shellcheck source=.evergreen/find-python3.sh
     . ../find-python3.sh || return
-
-    echo "Finding Python3 binary..."
-    PYTHON="$(find_python3 2>/dev/null)" || return
-    echo "Finding Python3 binary... done."
+    PYTHON=$(ensure_python3) || return
 
     echo "Creating virtual environment 'authoidcvenv'..."
     venvcreate "${PYTHON:?}" authoidcvenv || return

diff --git a/.evergreen/csfle/activate-kmstlsvenv.sh b/.evergreen/csfle/activate-kmstlsvenv.sh
@@ -32,10 +32,7 @@ activate_kmstlsvenv() {
   else
     # shellcheck source=.evergreen/find-python3.sh
     . ../find-python3.sh || return
-
-    echo "Finding Python3 binary..."
-    PYTHON="$(find_python3 2>/dev/null)" || return
-    echo "Finding Python3 binary... done."
+    PYTHON=$(ensure_python3) || return
 
     echo "Creating virtual environment 'kmstlsvenv'..."
     venvcreate "${PYTHON:?}" kmstlsvenv || return

diff --git a/.evergreen/docker/ubuntu18.04/Dockerfile b/.evergreen/docker/ubuntu18.04/Dockerfile
@@ -48,7 +48,6 @@ ARG GROUP_ID
 ENV DRIVERS_TOOLS=/root/drivers-evergreen-tools
 ENV PROJECT_ORCHESTRATION_HOME=/root/drivers-evergreen-tools/.evergreen/orchestration
 ENV MONGODB_BINARIES=/root/drivers-evergreen-tools/.evergreen/docker/ubuntu18.04/mongodb/bin
-ENV MONGODB_BINARY_ROOT=/root/drivers-evergreen-tools/.evergreen/docker/ubuntu18.04/
 ENV MONGO_ORCHESTRATION_HOME=/root/drivers-evergreen-tools/.evergreen/docker/ubuntu18.04/orchestration
 ENV DOCKER_RUNNING=true
 

diff --git a/.evergreen/docker/ubuntu20.04/Dockerfile b/.evergreen/docker/ubuntu20.04/Dockerfile
@@ -23,7 +23,6 @@ ARG GROUP_ID
 ENV DRIVERS_TOOLS=/root/drivers-evergreen-tools
 ENV PROJECT_ORCHESTRATION_HOME=/root/drivers-evergreen-tools/.evergreen/orchestration
 ENV MONGODB_BINARIES=/root/drivers-evergreen-tools/.evergreen/docker/ubuntu20.04/mongodb/bin
-ENV MONGODB_BINARY_ROOT=/root/drivers-evergreen-tools/.evergreen/docker/ubuntu20.04/
 ENV MONGO_ORCHESTRATION_HOME=/root/drivers-evergreen-tools/.evergreen/docker/ubuntu20.04/orchestration
 ENV DOCKER_RUNNING=true
 

diff --git a/.evergreen/download-mongodb.sh b/.evergreen/download-mongodb.sh
@@ -699,13 +699,14 @@ download_and_extract_package ()
    MONGODB_DOWNLOAD_URL=$1
    EXTRACT=$2
 
-   if [ -n "$MONGODB_BINARY_ROOT" ]; then
-      cd $MONGODB_BINARY_ROOT
+   if [ -n "${MONGODB_BINARIES:-}" ]; then
+      cd "$(dirname "$(dirname "${MONGODB_BINARIES:?}")")"
    else
       SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
       . $SCRIPT_DIR/handle-paths.sh
       cd $DRIVERS_TOOLS
    fi
+
    echo "Installing server binaries..."
    curl_retry $MONGODB_DOWNLOAD_URL --output mongodb-binaries.tgz
 
@@ -729,13 +730,14 @@ download_and_extract_mongosh ()
       get_mongodb_download_url_for $(get_distro) latest false
    fi
 
-   if [ -n "$MONGODB_BINARY_ROOT" ]; then
-      cd $MONGODB_BINARY_ROOT
+   if [ -n "${MONGODB_BINARIES:-}" ]; then
+      cd "$(dirname "$(dirname "${MONGODB_BINARIES:?}")")"
    else
       SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0})
       . $SCRIPT_DIR/handle-paths.sh
       cd $DRIVERS_TOOLS
    fi
+
    echo "Installing MongoDB shell..."
    curl_retry $MONGOSH_DOWNLOAD_URL --output mongosh.tgz
    $EXTRACT_MONGOSH mongosh.tgz

diff --git a/.evergreen/find-python3.sh b/.evergreen/find-python3.sh
@@ -291,3 +291,33 @@ find_python3() (
 
   return 0
 )
+
+#
+# Usage:
+#   ensure_python3
+#   PYTHON_BINARY=$(ensure_python3)
+#   PYTHON_BINARY=$(ensure_python3 2>/dev/null)
+#
+# If successful, print the name of the binary stdout (pipe 1).
+# Otherwise, no output is printed to stdout (pipe 1).
+#
+# Diagnostic messages may be printed to stderr (pipe 2). Redirect to /dev/null
+# with `2>/dev/null` to silence these messages.
+#
+# If PYTHON or PYTHON_BINARY are set, it will return that value.  Otherwise
+# it will use find_python3 to return a suitable value.
+#
+ensure_python3() {
+  declare python_binary
+  python_binary="${PYTHON:-"${PYTHON_BINARY:-""}"}"
+  {
+    if [ -z "${python_binary}" ]; then
+      echo "Finding Python3 binary..."
+      python_binary="$(find_python3 2>/dev/null)" || return
+      echo "Finding Python3 binary... done."
+    else
+      echo "Using Python binary $PYTHON"
+    fi
+  } 1>&2
+  echo "${python_binary:?}"
+}
diff --git a/.evergreen/handle-paths.sh b/.evergreen/handle-paths.sh
@@ -14,14 +14,11 @@ if [ -z "$SCRIPT_DIR" ]; then
   exit 1
 fi
 
-if command -v realpath &> /dev/null
+if command -v realpath >/dev/null 2>&1
 then
     SCRIPT_DIR=$(realpath $SCRIPT_DIR)
 else
-  SCRIPT_DIR="$( cd -- "$SCRIPT_DIR" &> /dev/null && pwd )"
-fi
-if [[ "$(uname -s)" == CYGWIN* ]]; then
-  SCRIPT_DIR=$(cygpath -m $SCRIPT_DIR)
+  SCRIPT_DIR="$( cd -- "$SCRIPT_DIR" > /dev/null 2>&1 && pwd )"
 fi
 
 # Find the DRIVERS_TOOLS by walking up the folder tree until there
@@ -37,25 +34,36 @@ if [ -z "${DRIVERS_TOOLS:-}" ]; then
   done
 fi
 
-if [[ "$(uname -s)" == CYGWIN* ]]; then
-    DRIVERS_TOOLS=$(cygpath -m $DRIVERS_TOOLS)
-fi
+case "$(uname -s)" in
+  CYGWIN*)
+    SCRIPT_DIR=$(cygpath -m "$SCRIPT_DIR")
+    DRIVERS_TOOLS=$(cygpath -m "$DRIVERS_TOOLS")
+    # USERPROFILE is required by Python for pathlib.Path().expanduser(~).
+    if [ -z "${USERPROFILE:-}" ]; then
+      USERPROFILE=$(cygpath -m "$HOME")
+    fi
+  ;;
+esac
 
 # Handle .env files
 if [ -f "$DRIVERS_TOOLS/.env" ]; then
   echo "Reading $DRIVERS_TOOLS/.env file"
-  export $(grep -v '^#' $DRIVERS_TOOLS/.env | xargs)
+  export $(grep -v '^#' "$DRIVERS_TOOLS/.env" | xargs)
 fi
 
 if [ -f "$SCRIPT_DIR/.env" ]; then
   echo "Reading $SCRIPT_DIR/.env file"
-  export $(grep -v '^#' $SCRIPT_DIR/.env | xargs)
+  export $(grep -v '^#' "$SCRIPT_DIR/.env" | xargs)
 fi
 
 MONGODB_BINARIES=${MONGODB_BINARIES:-${DRIVERS_TOOLS}/mongodb/bin}
 MONGO_ORCHESTRATION_HOME=${MONGO_ORCHESTRATION_HOME:-${DRIVERS_TOOLS}/.evergreen/orchestration}
 
 # Add the local .bin dir to the path.
-if [[ $PATH != *"$DRIVERS_TOOLS/.bin"* ]]; then
-  PATH=$PATH:$DRIVERS_TOOLS/.bin
-fi
+case "$PATH" in
+  *"$DRIVERS_TOOLS/.bin"*)
+  ;;
+  *)
+    PATH="$PATH:$DRIVERS_TOOLS/.bin"
+  ;;
+esac
diff --git a/.evergreen/ocsp/activate-ocspvenv.sh b/.evergreen/ocsp/activate-ocspvenv.sh
@@ -32,10 +32,7 @@ activate_ocspvenv() {
   else
     # shellcheck source=.evergreen/find-python3.sh
     . ../find-python3.sh || return
-
-    echo "Finding Python3 binary..."
-    PYTHON="$(find_python3 2>/dev/null)" || return
-    echo "Finding Python3 binary... done."
+    PYTHON=$(ensure_python3) || return
 
     echo "Creating virtual environment 'ocspvenv'..."
     venvcreate "${PYTHON:?}" ocspvenv || return

diff --git a/.evergreen/run-orchestration.sh b/.evergreen/run-orchestration.sh
@@ -14,6 +14,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #   ORCHESTRATION_FILE     Set to a non-empty string to use the <topology>/<orchestration_file>.json configuration.
 #   SKIP_CRYPT_SHARED      Set to a non-empty string to skip downloading crypt_shared
 #   MONGODB_BINARIES       Set to a non-empty string to set the path to the MONGODB_BINARIES for mongo orchestration.
+#   PYTHON                 Set to a non-empty string to set the Python binary to use.
 #   INSTALL_LEGACY_SHELL   Set to a non-empty string to install the legacy mongo shell.
 
 # See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505
@@ -31,19 +32,18 @@ DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS}
 MONGODB_VERSION=${MONGODB_VERSION:-latest}
 MONGODB_DOWNLOAD_URL=${MONGODB_DOWNLOAD_URL}
 ORCHESTRATION_FILE=${ORCHESTRATION_FILE}
-MONGODB_BINARIES=${MONGODB_BINARIES:-}
 INSTALL_LEGACY_SHELL=${INSTALL_LEGACY_SHELL:-}
+PYTHON=${PYTHON:-}
+# Note: MONGO_ORCHESTRATION_HOME and MONGODB_BINARIES defaults are handled in handle-paths.sh.
 
 DL_START=$(date +%s)
 
 # Functions to fetch MongoDB binaries.
 . $SCRIPT_DIR/download-mongodb.sh
 
 # To continue supporting `sh run-orchestration.sh` for backwards-compatibility,
-# explicitly invoke Bash as a subshell here when running `find_python3`.
-echo "Finding Python3 binary..."
-PYTHON="$(bash -c ". $SCRIPT_DIR/find-python3.sh && find_python3 2>/dev/null")"
-echo "Finding Python3 binary... done."
+# explicitly invoke Bash as a subshell here when running `ensure_python3`.
+PYTHON=$(bash -c ". $SCRIPT_DIR/find-python3.sh && ensure_python3 2>/dev/null")
 
 # Set up the mongo orchestration config.
 mkdir -p $MONGO_ORCHESTRATION_HOME

diff --git a/.evergreen/serverless/create-instance.sh b/.evergreen/serverless/create-instance.sh
@@ -106,9 +106,7 @@ if [ $SERVERLESS_URI = "null" ]; then
   exit 1
 fi
 
-echo "Finding Python3 binary..." 1>&2
-PYTHON="$(find_python3 2>/dev/null)"
-echo "Finding Python3 binary... done." 1>&2
+PYTHON=$(ensure_python3)
 
 RESP=$(curl -sS \
   --digest -u "${ATLAS_PUBLIC_API_KEY}:${ATLAS_PRIVATE_API_KEY}" \

diff --git a/.evergreen/start-orchestration.sh b/.evergreen/start-orchestration.sh
@@ -28,14 +28,7 @@ det_evergreen_dir=$SCRIPT_DIR
 
 cd "$MONGO_ORCHESTRATION_HOME"
 
-if [[ -z "${PYTHON:-}" ]]; then
-  echo "Finding Python3 binary..."
-  PYTHON="$(find_python3 2>/dev/null)"
-  echo "Finding Python3 binary... done."
-else
-  # May have already been found by run-orchestration.sh. Avoid redundant lookup.
-  echo "Using Python3 binary: $PYTHON"
-fi
+PYTHON=$(ensure_python3)
 
 echo "Creating virtual environment 'venv'..."
 venvcreate "${PYTHON:?}" venv
@@ -44,7 +37,7 @@ echo "Creating virtual environment 'venv'... done."
 # Install from github to get the latest mongo-orchestration.
 python -m pip install -q --upgrade 'https://github.com/mongodb/mongo-orchestration/archive/master.tar.gz'
 python -m pip list
-cd -
+cd $DRIVERS_TOOLS
 
 # Create default config file if it doesn't exist
 if [ ! -f $MONGO_ORCHESTRATION_HOME/orchestration.config ]; then

diff --git a/.evergreen/stop-orchestration.sh b/.evergreen/stop-orchestration.sh
@@ -1,14 +1,20 @@
 #!/usr/bin/env bash
 set -o errexit  # Exit the script with error if any of the commands fail
 
-cd "$MONGO_ORCHESTRATION_HOME"
+SCRIPT_DIR=$(dirname "${BASH_SOURCE:-"$0"}")
+. "$SCRIPT_DIR/handle-paths.sh"
+
+cd ${DRIVERS_TOOLS}
+
 # source the mongo-orchestration virtualenv if it exists
 if [ -f venv/bin/activate ]; then
-    . venv/bin/activate
+    . "$MONGO_ORCHESTRATION_HOME/venv/bin/activate"
     mongo-orchestration stop
 elif [ -f venv/Scripts/activate ]; then
-    . venv/Scripts/activate
+    . "$MONGO_ORCHESTRATION_HOME/venv/Scripts/activate"
     mongo-orchestration stop
 else
-    echo "No mongo orchestration to stop!"
+    echo "No virtualenv found!"
 fi
+
+cd -
diff --git a/.gitignore b/.gitignore
@@ -9,6 +9,7 @@ __pycache__/
 
 # C extensions
 *.so
+*.dylib
 
 # Distribution / packaging
 .Python

diff --git a/Makefile b/Makefile
@@ -3,6 +3,15 @@
 all:
 	@echo "Project successfully compiled"
 
+clean:
+	rm -rf mongodb
+
+run-server: clean
+	.evergreen/run-orchestration.sh
+
+stop-server:
+	.evergreen/stop-orchestration.sh
+
 test:
 	@echo "Running tests..."
 	@echo "All done, thank you and please come again"

diff --git a/README.md b/README.md
@@ -80,6 +80,20 @@ returned from the `setup-mongodb` workflow step when running tests:
           CRYPT_SHARED_LIB_PATH: ${{ steps.setup-mongodb.outputs.crypt-shared-lib-path }}
 ```
 
+### Starting and Stopping Servers Locally or on an Evergreen Host
+
+There are two options for running a MongoDB server configuration.
+One is to use [docker](./.evergreen/docker/README.md).
+The other is to run `./evergreen/run-orchestration.sh` locally.
+For convenience, you can run `make run-server` and `make stop-server` to start and stop the server(s).
+For example:
+
+```bash
+TOPOLOGY=replica_set MONGODB_VERSION=7.0 make run-server
+```
+
+See (run-orchestration.sh)[./evergreen/run-orchestration.sh] for the available environment variables.
+
 ## Linters and Formatters
 
 This repo uses [pre-commit](https://pre-commit.com/) for managing linting and formatting of the codebase.