diff --git a/.evergreen/atlas/atlas-utils.sh b/.evergreen/atlas/atlas-utils.sh index e8c325a7..fc77f34f 100755 --- a/.evergreen/atlas/atlas-utils.sh +++ b/.evergreen/atlas/atlas-utils.sh @@ -21,7 +21,7 @@ create_deployment () # Ensure that all variables required to run the test are set, otherwise throw # an error. - for VARNAME in ${VARLIST[*]}; do + for VARNAME in "${VARLIST[@]}"; do [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; done @@ -54,7 +54,7 @@ check_deployment () # Ensure that all variables required to run the test are set, otherwise throw # an error. - for VARNAME in ${VARLIST[*]}; do + for VARNAME in "${VARLIST[@]}"; do [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; done diff --git a/.evergreen/atlas/setup-atlas-cluster.sh b/.evergreen/atlas/setup-atlas-cluster.sh index 8ba00dd3..4fdd5ce6 100755 --- a/.evergreen/atlas/setup-atlas-cluster.sh +++ b/.evergreen/atlas/setup-atlas-cluster.sh @@ -37,6 +37,7 @@ fi # Backwards compatibility: map old names to new names. if [ -n "${LAMBDA_STACK_NAME:-}" ]; then + # shellcheck disable=SC2034 CLUSTER_PREFIX=$LAMBDA_STACK_NAME fi if [ -n "${DRIVERS_ATLAS_LAMBDA_USER:-}" ]; then @@ -57,7 +58,7 @@ CLUSTER_PREFIX # Ensure that all variables required to run the test are set, otherwise throw # an error. -for VARNAME in ${VARLIST[*]}; do +for VARNAME in "${VARLIST[@]}"; do [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; done @@ -71,7 +72,7 @@ done export VERSION="${MONGODB_VERSION:-6.0}" # Set the create cluster configuration. -export DEPLOYMENT_DATA=$(cat <> "secrets-export.sh" # Set the create cluster configuration. -export DEPLOYMENT_DATA=$(cat <> ./secrets-export.sh popd diff --git a/.evergreen/auth_oidc/gcp/setup-instance.sh b/.evergreen/auth_oidc/gcp/setup-instance.sh index 10916a5f..8f40165f 100755 --- a/.evergreen/auth_oidc/gcp/setup-instance.sh +++ b/.evergreen/auth_oidc/gcp/setup-instance.sh @@ -5,14 +5,18 @@ set -o errexit # Exit on first command error. SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/../../handle-paths.sh -if [ -z "$GCPKMS_GCLOUD" -o -z "$GCPKMS_PROJECT" -o -z "$GCPKMS_ZONE" -o -z "$GCPKMS_INSTANCENAME" ]; then - echo "Please set the following required environment variables" - echo " GCPKMS_GCLOUD to the path of the gcloud binary" - echo " GCPKMS_PROJECT to the GCP project" - echo " GCPKMS_ZONE to the GCP zone" - echo " GCPKMS_INSTANCENAME to the GCE instance name" - exit 1 -fi +VARLIST=( +GCPKMS_GCLOUD +GCPKMS_PROJECT +GCPKMS_ZONE +GCPKMS_INSTANCENAME +) + +# Ensure that all variables required to run the test are set, otherwise throw +# an error. +for VARNAME in "${VARLIST[@]}"; do + [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; +done echo "Copying setup-gce-instance.sh to GCE instance ($GCPKMS_INSTANCENAME) ... begin" # Copy files to test. Use "-p" to preserve execute mode. diff --git a/.evergreen/auth_oidc/gcp/setup.sh b/.evergreen/auth_oidc/gcp/setup.sh index b1c7a56d..eb9e7a0c 100755 --- a/.evergreen/auth_oidc/gcp/setup.sh +++ b/.evergreen/auth_oidc/gcp/setup.sh @@ -22,7 +22,7 @@ DEPLOYMENT_NAME="$RANDOM-DRIVERGCP" echo "export CLUSTER_NAME=$DEPLOYMENT_NAME" >> "$DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh" # Set the create cluster configuration. -export DEPLOYMENT_DATA=$(cat <> "$DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh" # Set the create cluster configuration. -export DEPLOYMENT_DATA=$(cat <> "secrets-export.sh" # Set the create cluster configuration. - export DEPLOYMENT_DATA=$(cat </dev/null -if [ $(uname -s) = "Darwin" ]; then +if [ "$(uname -s)" = "Darwin" ]; then SHUTDOWN_TIME=$(date -u -v+1H +"%H%M") else SHUTDOWN_TIME=$(date -u -d "$(date) + 1 hours" +"%H%M") diff --git a/.evergreen/csfle/azurekms/login.sh b/.evergreen/csfle/azurekms/login.sh index 946667b3..3e4bc567 100755 --- a/.evergreen/csfle/azurekms/login.sh +++ b/.evergreen/csfle/azurekms/login.sh @@ -3,15 +3,17 @@ set -o errexit set -o pipefail set -o nounset -if [ -z "$AZUREKMS_CLIENTID" -o \ - -z "$AZUREKMS_SECRET" -o \ - -z "$AZUREKMS_TENANTID" ]; then - echo "Please set the following required environment variables" - echo " AZUREKMS_CLIENTID" - echo " AZUREKMS_SECRET" - echo " AZUREKMS_TENANTID" - exit 1 -fi +VARLIST=( +AZUREKMS_CLIENTID +AZUREKMS_SECRET +AZUREKMS_TENANTID +) + +# Ensure that all variables required to run the test are set, otherwise throw +# an error. +for VARNAME in "${VARLIST[@]}"; do + [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; +done # Check for Azure Command-Line Interface (`az`) version 2.25.0 or newer. if ! command -v az &> /dev/null; then diff --git a/.evergreen/csfle/azurekms/run-command.sh b/.evergreen/csfle/azurekms/run-command.sh index eef9bcff..b66069ac 100755 --- a/.evergreen/csfle/azurekms/run-command.sh +++ b/.evergreen/csfle/azurekms/run-command.sh @@ -3,17 +3,18 @@ set -o errexit set -o pipefail set -o nounset -if [ -z "$AZUREKMS_RESOURCEGROUP" -o \ - -z "$AZUREKMS_VMNAME" -o \ - -z "$AZUREKMS_PRIVATEKEYPATH" -o \ - -z "$AZUREKMS_CMD" ]; then - echo "Please set the following required environment variables" - echo " AZUREKMS_RESOURCEGROUP" - echo " AZUREKMS_VMNAME" - echo " AZUREKMS_PRIVATEKEYPATH" - echo " AZUREKMS_CMD" - exit 1 -fi +VARLIST=( +AZUREKMS_RESOURCEGROUP +AZUREKMS_VMNAME +AZUREKMS_PRIVATEKEYPATH +AZUREKMS_CMD +) + +# Ensure that all variables required to run the test are set, otherwise throw +# an error. +for VARNAME in "${VARLIST[@]}"; do + [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; +done echo "Running '$AZUREKMS_CMD' on Azure Virtual Machine ... begin" IP=$(az vm show --show-details --resource-group $AZUREKMS_RESOURCEGROUP --name $AZUREKMS_VMNAME --query publicIps -o tsv) diff --git a/.evergreen/csfle/gcpkms/copy-file.sh b/.evergreen/csfle/gcpkms/copy-file.sh index f444ad05..42bf9f39 100755 --- a/.evergreen/csfle/gcpkms/copy-file.sh +++ b/.evergreen/csfle/gcpkms/copy-file.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash # Copy a file to or from a GCE instance. set -o errexit # Exit on first command error. -if [ -z "$GCPKMS_GCLOUD" -o -z "$GCPKMS_PROJECT" -o -z "$GCPKMS_ZONE" -o -z "$GCPKMS_SRC" -o -z "$GCPKMS_DST" ]; then - echo "Please set the following required environment variables" - echo " GCPKMS_GCLOUD to the path of the gcloud binary" - echo " GCPKMS_PROJECT to the GCP project" - echo " GCPKMS_ZONE to the GCP zone" - echo " GCPKMS_SRC to the source file" - echo " GCPKMS_DST to the destination file" - echo "To copy from or to a GCE host, use the host instance name" - echo "Example: GCPKMS_SRC=$GCPKMS_INSTANCENAME:src.txt GCPKMS_DST=. ./copy-file.sh" - exit 1 -fi + +VARLIST=( +GCPKMS_GCLOUD +GCPKMS_PROJECT +GCPKMS_ZONE +GCPKMS_SRC +GCPKMS_DST +) + +# Ensure that all variables required to run the test are set, otherwise throw +# an error. +for VARNAME in "${VARLIST[@]}"; do + [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; +done echo "Copying $GCPKMS_SRC to $GCPKMS_DST ... begin" # Copy files to test. Use "-p" to preserve execute mode. diff --git a/.evergreen/csfle/gcpkms/create-and-setup-instance.sh b/.evergreen/csfle/gcpkms/create-and-setup-instance.sh index ee12bdca..256009a2 100755 --- a/.evergreen/csfle/gcpkms/create-and-setup-instance.sh +++ b/.evergreen/csfle/gcpkms/create-and-setup-instance.sh @@ -15,6 +15,7 @@ GCPKMS_SECRETS_FILE=${GCPKMS_SECRETS_FILE:-./secrets-export.sh} # Handle secrets from vault. if [ -f "$GCPKMS_SECRETS_FILE" ]; then echo "Sourcing secrets" + # shellcheck source=secrets-export.sh source $GCPKMS_SECRETS_FILE fi if [ -z "${GCPKMS_SERVICEACCOUNT:-}" ]; then @@ -28,7 +29,7 @@ if [ -n "$GCPKMS_KEYFILE_CONTENT" ]; then echo ${GCPKMS_KEYFILE_CONTENT} | base64 --decode > $GCPKMS_KEYFILE fi -if [ -z "$GCPKMS_KEYFILE" -o -z "$GCPKMS_SERVICEACCOUNT" ]; then +if [ -z "$GCPKMS_KEYFILE" ] || [ -z "$GCPKMS_SERVICEACCOUNT" ]; then echo "Please set the following required environment variables" echo " GCPKMS_KEYFILE to the JSON file for the service account" echo " GCPKMS_SERVICEACCOUNT to a GCP service account used to create and attach to the GCE instance" @@ -74,7 +75,7 @@ fi # Wait for a maximum of five minutes for VM to finish booting. # Otherwise SSH may fail. See https://cloud.google.com/compute/docs/troubleshooting/troubleshooting-ssh. wait_for_server () { - for i in $(seq 300); do + for _ in $(seq 300); do # The first `gcloud compute ssh` creates an SSH key pair and stores the public key in the Google Account. # The public key is deleted from the Google Account in delete-instance.sh. if SSHOUTPUT=$($GCPKMS_GCLOUD compute ssh "$GCPKMS_INSTANCENAME" --zone $GCPKMS_ZONE --project $GCPKMS_PROJECT --command "echo 'ping' --ssh-flag='-o ConnectTimeout=10'" 2>&1); then @@ -98,5 +99,6 @@ echo "Adding expiration time to SSH key ... end" SETUP_INSTANCE=${GCPKMS_SETUP_INSTANCE:-$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/setup-instance.sh} echo "setup-instance.sh ... begin" +# shellcheck source=setup-instance.sh . $SETUP_INSTANCE echo "setup-instance.sh ... end" diff --git a/.evergreen/csfle/gcpkms/create-instance.sh b/.evergreen/csfle/gcpkms/create-instance.sh index 096aeab1..bc22725c 100755 --- a/.evergreen/csfle/gcpkms/create-instance.sh +++ b/.evergreen/csfle/gcpkms/create-instance.sh @@ -1,25 +1,24 @@ #!/usr/bin/env bash # Create a GCE instance. set -o errexit # Exit on first command error. -if [ -z "$GCPKMS_GCLOUD" -o \ - -z "$GCPKMS_PROJECT" -o \ - -z "$GCPKMS_ZONE" -o \ - -z "$GCPKMS_SERVICEACCOUNT" -o \ - -z "$GCPKMS_IMAGEPROJECT" -o \ - -z "$GCPKMS_IMAGEFAMILY" -o \ - -z "$GCPKMS_MACHINETYPE" -o \ - -z "$GCPKMS_DISKSIZE" ]; then - echo "Please set the following required environment variables" - echo " GCPKMS_GCLOUD to the path of the gcloud binary" - echo " GCPKMS_PROJECT to the GCP project" - echo " GCPKMS_ZONE to the GCP zone" - echo " GCPKMS_SERVICEACCOUNT to a GCP service account used to create and attach to the GCE instance" - echo " GCPKMS_IMAGEPROJECT to the GCE image project (e.g. debian-cloud)" - echo " GCPKMS_IMAGEFAMILY to the GCE image family (e.g. debian-11)" - echo " GCPKMS_MACHINETYPE to the GCE machine type (e.g. e2-micro)" - echo " GCPKMS_DISKSIZE to the GCE disk size (e.g. 20gb)" - exit 1 -fi + +VARLIST=( +GCPKMS_GCLOUD +GCPKMS_PROJECT +GCPKMS_ZONE +GCPKMS_SERVICEACCOUNT +GCPKMS_IMAGEPROJECT +GCPKMS_IMAGEFAMILY +GCPKMS_MACHINETYPE +GCPKMS_DISKSIZE +) + +# Ensure that all variables required to run the test are set, otherwise throw +# an error. +for VARNAME in "${VARLIST[@]}"; do + [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; +done + GCPKMS_INSTANCENAME="instancename-$RANDOM" SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) diff --git a/.evergreen/csfle/gcpkms/delete-instance.sh b/.evergreen/csfle/gcpkms/delete-instance.sh index 62134e91..71db1370 100755 --- a/.evergreen/csfle/gcpkms/delete-instance.sh +++ b/.evergreen/csfle/gcpkms/delete-instance.sh @@ -10,7 +10,7 @@ if [ -f $SCRIPT_DIR/secrets-export.sh ]; then source $SCRIPT_DIR/secrets-export.sh fi -if [ -z "$GCPKMS_GCLOUD" -o -z "$GCPKMS_PROJECT" -o -z "$GCPKMS_ZONE" -o -z "$GCPKMS_INSTANCENAME" ]; then +if [ -z "$GCPKMS_GCLOUD" ] || [ -z "$GCPKMS_PROJECT" ] || [ -z "$GCPKMS_ZONE" ] || [ -z "$GCPKMS_INSTANCENAME" ]; then echo "Please set the following required environment variables" echo " GCPKMS_GCLOUD to the path of the gcloud binary" echo " GCPKMS_PROJECT to the GCP project" diff --git a/.evergreen/csfle/gcpkms/download-gcloud.sh b/.evergreen/csfle/gcpkms/download-gcloud.sh index 0328d4b9..0aae338d 100755 --- a/.evergreen/csfle/gcpkms/download-gcloud.sh +++ b/.evergreen/csfle/gcpkms/download-gcloud.sh @@ -13,3 +13,4 @@ else GCPKMS_GCLOUD=$(pwd)/google-cloud-sdk/bin/gcloud echo "Download gcloud ... end" fi +export GCPKMS_GCLOUD diff --git a/.evergreen/csfle/gcpkms/remote-scripts/start-mongodb.sh b/.evergreen/csfle/gcpkms/remote-scripts/start-mongodb.sh index 4457493a..dfdca3da 100755 --- a/.evergreen/csfle/gcpkms/remote-scripts/start-mongodb.sh +++ b/.evergreen/csfle/gcpkms/remote-scripts/start-mongodb.sh @@ -5,7 +5,8 @@ set -o pipefail echo "Starting MongoDB server ... begin" git clone https://github.com/mongodb-labs/drivers-evergreen-tools -export DRIVERS_TOOLS=$(pwd)/drivers-evergreen-tools +DRIVERS_TOOLS="$(pwd)/drivers-evergreen-tools" +export DRIVERS_TOOLS export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config diff --git a/.evergreen/csfle/gcpkms/remote-scripts/startup.sh b/.evergreen/csfle/gcpkms/remote-scripts/startup.sh index 3969318f..72b262a6 100755 --- a/.evergreen/csfle/gcpkms/remote-scripts/startup.sh +++ b/.evergreen/csfle/gcpkms/remote-scripts/startup.sh @@ -3,6 +3,6 @@ # Delete the GCE instance after a period of time. # Refer: https://cloud.google.com/community/tutorials/create-a-self-deleting-virtual-machine sleep 7200 -export NAME=$(curl -X GET http://metadata.google.internal/computeMetadata/v1/instance/name -H 'Metadata-Flavor: Google') -export ZONE=$(curl -X GET http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google') +NAME=$(curl -X GET http://metadata.google.internal/computeMetadata/v1/instance/name -H 'Metadata-Flavor: Google') +ZONE=$(curl -X GET http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google') gcloud --quiet compute instances delete $NAME --zone=$ZONE diff --git a/.evergreen/csfle/gcpkms/run-command.sh b/.evergreen/csfle/gcpkms/run-command.sh index 99dc3198..3f3b8f73 100755 --- a/.evergreen/csfle/gcpkms/run-command.sh +++ b/.evergreen/csfle/gcpkms/run-command.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Run a command on a remote GCE instance. set -o errexit # Exit on first command error. -if [ -z "$GCPKMS_GCLOUD" -o -z "$GCPKMS_PROJECT" -o -z "$GCPKMS_ZONE" -o -z "$GCPKMS_INSTANCENAME" -o -z "$GCPKMS_CMD" ]; then +if [ -z "$GCPKMS_GCLOUD" ] || [ -z "$GCPKMS_PROJECT" ] || [ -z "$GCPKMS_ZONE" ] || [ -z "$GCPKMS_INSTANCENAME" ] || [ -z "$GCPKMS_CMD" ]; then echo "Please set the following required environment variables" echo " GCPKMS_GCLOUD to the path of the gcloud binary" echo " GCPKMS_PROJECT to the GCP project" diff --git a/.evergreen/csfle/gcpkms/setup-instance.sh b/.evergreen/csfle/gcpkms/setup-instance.sh index d3f925e6..aa045aad 100755 --- a/.evergreen/csfle/gcpkms/setup-instance.sh +++ b/.evergreen/csfle/gcpkms/setup-instance.sh @@ -5,7 +5,7 @@ set -o errexit # Exit on first command error. SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/../../handle-paths.sh -if [ -z "$GCPKMS_GCLOUD" -o -z "$GCPKMS_PROJECT" -o -z "$GCPKMS_ZONE" -o -z "$GCPKMS_INSTANCENAME" ]; then +if [ -z "$GCPKMS_GCLOUD" ] || [ -z "$GCPKMS_PROJECT" ] || [ -z "$GCPKMS_ZONE" ] || [ -z "$GCPKMS_INSTANCENAME" ]; then echo "Please set the following required environment variables" echo " GCPKMS_GCLOUD to the path of the gcloud binary" echo " GCPKMS_PROJECT to the GCP project" diff --git a/.evergreen/csfle/set-temp-creds.sh b/.evergreen/csfle/set-temp-creds.sh index 42013b7d..c48c7583 100755 --- a/.evergreen/csfle/set-temp-creds.sh +++ b/.evergreen/csfle/set-temp-creds.sh @@ -24,7 +24,7 @@ set +o xtrace # Disable tracing. get_creds() { - $PYTHON - "$@" << 'EOF' + $PYTHON - << 'EOF' import sys import boto3 @@ -37,6 +37,9 @@ EOF PYTHON=${PYTHON:-python} CREDS=$(get_creds) -export CSFLE_AWS_TEMP_ACCESS_KEY_ID=$(echo $CREDS | awk '{print $1}') -export CSFLE_AWS_TEMP_SECRET_ACCESS_KEY=$(echo $CREDS | awk '{print $2}') -export CSFLE_AWS_TEMP_SESSION_TOKEN=$(echo $CREDS | awk '{print $3}') +CSFLE_AWS_TEMP_ACCESS_KEY_ID=$(echo $CREDS | awk '{print $1}') +CSFLE_AWS_TEMP_SECRET_ACCESS_KEY=$(echo $CREDS | awk '{print $2}') +CSFLE_AWS_TEMP_SESSION_TOKEN=$(echo $CREDS | awk '{print $3}') +export CSFLE_AWS_TEMP_ACCESS_KEY_ID +export CSFLE_AWS_TEMP_SECRET_ACCESS_KEY +export CSFLE_AWS_TEMP_SESSION_TOKEN diff --git a/.evergreen/csfle/stop-servers.sh b/.evergreen/csfle/stop-servers.sh index ecfbc7f6..7f92bfec 100755 --- a/.evergreen/csfle/stop-servers.sh +++ b/.evergreen/csfle/stop-servers.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Clean up CSFLE kmip servers +set -e + SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/../handle-paths.sh pushd $SCRIPT_DIR diff --git a/.evergreen/download-mongodb.sh b/.evergreen/download-mongodb.sh index efa32cea..52025167 100755 --- a/.evergreen/download-mongodb.sh +++ b/.evergreen/download-mongodb.sh @@ -1,10 +1,9 @@ #!/usr/bin/env bash +# shellcheck shell=sh #For future use the feed to get full list of distros : http://downloads.mongodb.org/full.json set -o errexit # Exit the script with error if any of the commands fail -SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) -. $SCRIPT_DIR/handle-paths.sh get_distro () { @@ -23,12 +22,14 @@ get_distro () DISTRO="${name}-${version}" elif [ -f /etc/redhat-release ]; then release=$(cat /etc/redhat-release) - - if [[ "$release" =~ "Red Hat" ]]; then - name="rhel" - elif [[ "$release" =~ "Fedora" ]]; then - name="fedora" - fi + case $release in + *Red\ Hat*) + name="rhel" + ;; + Fedora*) + name="fedora" + ;; + esac version=$(echo $release | sed 's/.*\([[:digit:]]\).*/\1/g') DISTRO="${name}-${version}" elif [ -f /etc/lsb-release ]; then @@ -694,25 +695,29 @@ download_and_extract_package () MONGODB_DOWNLOAD_URL=$1 EXTRACT=$2 + # shellcheck disable=SC3028 + SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + . $SCRIPT_DIR/handle-paths.sh + if [ -n "${MONGODB_BINARIES:-}" ]; then cd "$(dirname "$(dirname "${MONGODB_BINARIES:?}")")" else - SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) - . $SCRIPT_DIR/handle-paths.sh cd $DRIVERS_TOOLS fi echo "Installing server binaries..." - . "$SCRIPT_DIR/retry-with-backoff.sh" - retry_with_backoff curl $MONGODB_DOWNLOAD_URL --output mongodb-binaries.tgz + "$SCRIPT_DIR/retry-with-backoff.sh" curl $MONGODB_DOWNLOAD_URL --output mongodb-binaries.tgz $EXTRACT mongodb-binaries.tgz echo "Installing server binaries... done." + set -x rm -f mongodb-binaries.tgz mv mongodb* mongodb chmod -R +x mongodb - find . -name vcredist_x64.exe -exec {} /install /quiet \; + # Clear the environment to avoid "find: The environment is too large for exec()" + # error on Windows. + env -i PATH="$PATH" find . -name vcredist_x64.exe -exec {} /install /quiet \; echo "MongoDB server version: $(./mongodb/bin/mongod --version)" cd - } @@ -722,21 +727,22 @@ download_and_extract_mongosh () MONGOSH_DOWNLOAD_URL=$1 EXTRACT_MONGOSH=${2:-"tar zxf"} + # shellcheck disable=SC3028 + SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + . $SCRIPT_DIR/handle-paths.sh + if [ -z "$MONGOSH_DOWNLOAD_URL" ]; then - get_mongodb_download_url_for $(get_distro) latest false + get_mongodb_download_url_for "$(get_distro)" latest false fi if [ -n "${MONGODB_BINARIES:-}" ]; then cd "$(dirname "$(dirname "${MONGODB_BINARIES:?}")")" else - SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) - . $SCRIPT_DIR/handle-paths.sh cd $DRIVERS_TOOLS fi echo "Installing MongoDB shell..." - . "$SCRIPT_DIR/retry-with-backoff.sh" - retry_with_backoff curl $MONGOSH_DOWNLOAD_URL --output mongosh.tgz + "$SCRIPT_DIR/retry-with-backoff.sh" curl $MONGOSH_DOWNLOAD_URL --output mongosh.tgz $EXTRACT_MONGOSH mongosh.tgz rm -f mongosh.tgz @@ -765,10 +771,11 @@ download_and_extract () download_and_extract_mongosh "$MONGOSH_DOWNLOAD_URL" "$EXTRACT_MONGOSH" fi + # shellcheck disable=SC3028 SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) . $SCRIPT_DIR/handle-paths.sh - if [ ! -z "${INSTALL_LEGACY_SHELL:-}" -a ! -e $DRIVERS_TOOLS/mongodb/bin/mongo -a ! -e $DRIVERS_TOOLS/mongodb/bin/mongo.exe ]; then + if [ ! -z "${INSTALL_LEGACY_SHELL:-}" ] && [ ! -e $DRIVERS_TOOLS/mongodb/bin/mongo ] && [ ! -e $DRIVERS_TOOLS/mongodb/bin/mongo.exe ]; then # The legacy mongo shell is not included in server downloads of 6.0.0-rc6 or later. Refer: SERVER-64352. # Some test scripts use the mongo shell for setup. # Download 5.0 package to get the legacy mongo shell as a workaround until DRIVERS-2328 is addressed. @@ -822,8 +829,10 @@ download_and_extract_crypt_shared () mkdir crypt_shared_download cd crypt_shared_download - . "$SCRIPT_DIR/retry-with-backoff.sh" - retry_with_backoff curl $MONGO_CRYPT_SHARED_DOWNLOAD_URL --output crypt_shared-binaries.tgz + # shellcheck disable=SC3028 + SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) + . $SCRIPT_DIR/handle-paths.sh + "$SCRIPT_DIR/retry-with-backoff.sh" curl $MONGO_CRYPT_SHARED_DOWNLOAD_URL --output crypt_shared-binaries.tgz $EXTRACT crypt_shared-binaries.tgz LIBRARY_NAME="mongo_crypt_v1" diff --git a/.evergreen/ensure-binary.sh b/.evergreen/ensure-binary.sh index 8fa2fd09..8321fea7 100755 --- a/.evergreen/ensure-binary.sh +++ b/.evergreen/ensure-binary.sh @@ -3,6 +3,7 @@ # Ensure the given binary is on the PATH. # Should be called as: # . $DRIVERS_TOOLS/.evergreen/ensure-binary.sh +set -eu SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/handle-paths.sh @@ -27,8 +28,6 @@ OS_NAME=$(uname -s | tr '[:upper:]' '[:lower:]') MARCH=$(uname -m | tr '[:upper:]' '[:lower:]') URL="" -. "$SCRIPT_DIR/retry-with-backoff.sh" - case $NAME in kubectl) VERSION=$(curl -L -s https://dl.k8s.io/release/stable.txt) @@ -76,7 +75,7 @@ echo "Installing $NAME..." if [ "$NAME" != "gcloud" ]; then mkdir -p ${DRIVERS_TOOLS}/.bin TARGET=${DRIVERS_TOOLS}/.bin/$NAME - retry_with_backoff curl -L -s $URL -o $TARGET + "$SCRIPT_DIR/retry-with-backoff.sh" curl -L -s $URL -o $TARGET chmod +x $TARGET else @@ -84,7 +83,7 @@ else pushd /tmp rm -rf google-cloud-sdk FNAME=/tmp/google-cloud-sdk.tgz - retry_with_backoff curl -L -s $URL -o $FNAME + "$SCRIPT_DIR/retry-with-backoff.sh" curl -L -s $URL -o $FNAME tar xfz $FNAME popd ln -s /tmp/google-cloud-sdk/bin/gcloud $DRIVERS_TOOLS/.bin/gcloud diff --git a/.evergreen/find-python3.sh b/.evergreen/find-python3.sh index 8d0a3f42..bcf56200 100755 --- a/.evergreen/find-python3.sh +++ b/.evergreen/find-python3.sh @@ -51,6 +51,7 @@ is_python3() ( # For diagnostic purposes. echo " - $bin: $version_output" + # shellcheck disable=SC2091 if ! $("$bin" -c "import sys; exit(sys.version_info[0] == 3 and sys.version_info[1] == 12)"); then echo "Detected Python 3.12. Skipping due to failures to start mock KMS server. Refer: DRIVERS-2743" return 1 diff --git a/.evergreen/handle-paths.sh b/.evergreen/handle-paths.sh index b01d7c0a..84bda2bf 100755 --- a/.evergreen/handle-paths.sh +++ b/.evergreen/handle-paths.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck shell=sh # # This script will handle the correct cross-platform absolute # paths for a script directory and DRIVERS_TOOLS. @@ -47,15 +48,10 @@ case "$(uname -s)" in esac # Handle .env files -if [ -f "$DRIVERS_TOOLS/.env" ]; then - echo "Reading $DRIVERS_TOOLS/.env file" - export $(grep -v '^#' "$DRIVERS_TOOLS/.env" | xargs) -fi - -if [ -f "$SCRIPT_DIR/.env" ]; then - echo "Reading $SCRIPT_DIR/.env file" - export $(grep -v '^#' "$SCRIPT_DIR/.env" | xargs) -fi +set +a +[ -f "$DRIVERS_TOOLS/.env" ] && . "$DRIVERS_TOOLS/.env" +[ -f "$SCRIPT_DIR/.env" ] && . "$SCRIPT_DIR/.env" +set -a MONGODB_BINARIES=${MONGODB_BINARIES:-${DRIVERS_TOOLS}/mongodb/bin} MONGO_ORCHESTRATION_HOME=${MONGO_ORCHESTRATION_HOME:-${DRIVERS_TOOLS}/.evergreen/orchestration} diff --git a/.evergreen/install-dependencies.sh b/.evergreen/install-dependencies.sh index bb1b411d..dde7fac1 100755 --- a/.evergreen/install-dependencies.sh +++ b/.evergreen/install-dependencies.sh @@ -6,7 +6,6 @@ SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) # Functions to fetch MongoDB binaries . $SCRIPT_DIR/download-mongodb.sh -OS=$(uname -s | tr '[:upper:]' '[:lower:]') get_distro diff --git a/.evergreen/install-node.sh b/.evergreen/install-node.sh index 893394fd..e287b840 100755 --- a/.evergreen/install-node.sh +++ b/.evergreen/install-node.sh @@ -11,7 +11,6 @@ NODE_LTS_VERSION=${NODE_LTS_VERSION:-18} NPM_VERSION=${NPM_VERSION:-latest} source "./init-node-and-npm-env.sh" -source "./retry-with-backoff.sh" if [[ -z "${npm_global_prefix}" ]]; then echo "npm_global_prefix is unset" && exit 1; fi if [[ -z "${NODE_ARTIFACTS_PATH}" ]]; then echo "NODE_ARTIFACTS_PATH is unset" && exit 1; fi @@ -34,13 +33,12 @@ shopt -s nocasematch # index.tab is a sorted tab separated values file with the following headers # 0 1 2 3 4 5 6 7 8 9 10 # version date files npm v8 uv zlib openssl modules lts security -retry_with_backoff curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab +"$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "https://nodejs.org/dist/index.tab" --output node_index.tab while IFS=$'\t' read -r -a row; do node_index_version="${row[0]}" node_index_major_version=$(echo $node_index_version | sed -E 's/^v([0-9]+).*$/\1/') node_index_date="${row[1]}" - node_index_lts="${row[9]}" [[ "$node_index_version" = "version" ]] && continue # skip tsv header [[ "$NODE_LTS_VERSION" = "latest" ]] && break # first line is latest [[ "$NODE_LTS_VERSION" = "$node_index_version" ]] && break # match full version if specified @@ -90,7 +88,7 @@ if [[ "$file_extension" = "zip" ]]; then if [[ -d "${NODE_ARTIFACTS_PATH}/nodejs/bin/${node_directory}" ]]; then echo "Node.js already installed!" else - retry_with_backoff curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" + "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}" mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs" # Windows "bins" are at the top level @@ -103,7 +101,7 @@ else if [[ -d "${NODE_ARTIFACTS_PATH}/nodejs/${node_directory}" ]]; then echo "Node.js already installed!" else - retry_with_backoff curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" + "$SCRIPT_DIR/retry-with-backoff.sh" curl "${CURL_FLAGS[@]}" "${node_download_url}" --output "$node_archive_path" tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}" mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs" fi diff --git a/.evergreen/k8s/aks/setup-cluster.sh b/.evergreen/k8s/aks/setup-cluster.sh index b84c8772..32759805 100755 --- a/.evergreen/k8s/aks/setup-cluster.sh +++ b/.evergreen/k8s/aks/setup-cluster.sh @@ -27,12 +27,14 @@ az aks create -g "${AKS_RESOURCE_GROUP}" -n "${AKS_CLUSTER_NAME}" \ --node-count 1 --enable-oidc-issuer --enable-workload-identity \ --enable-cluster-autoscaler --min-count 1 --max-count 3 \ --generate-ssh-keys -export AKS_OIDC_ISSUER="$(az aks show -n "${AKS_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --query "oidcIssuerProfile.issuerUrl" -otsv)" +AKS_OIDC_ISSUER="$(az aks show -n "${AKS_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" --query "oidcIssuerProfile.issuerUrl" -otsv)" +export AKS_OIDC_ISSUER az account set --subscription "${AKS_SUBSCRIPTION}" az identity create --name "${AKS_USER_ASSIGNED_IDENTITY_NAME}" --resource-group "${AKS_RESOURCE_GROUP}" \ --location "${AKS_LOCATION}" --subscription "${AKS_SUBSCRIPTION}" -export USER_ASSIGNED_CLIENT_ID="$(az identity show --resource-group "${AKS_RESOURCE_GROUP}" \ +USER_ASSIGNED_CLIENT_ID="$(az identity show --resource-group "${AKS_RESOURCE_GROUP}" \ --name "${AKS_USER_ASSIGNED_IDENTITY_NAME}" --query 'clientId' -otsv)" +export USER_ASSIGNED_CLIENT_ID az aks get-credentials --overwrite-existing -n "${AKS_CLUSTER_NAME}" -g "${AKS_RESOURCE_GROUP}" . $DRIVERS_TOOLS/.evergreen/ensure-binary.sh kubectl cat < 600) | .metadata.name' | xargs -I{} kubectl delete pod {} --force --grace-period=0 @@ -30,8 +31,7 @@ echo "Waiting for pod to be ready... done." echo "Configuring pod $POD_NAME..." set -x # Account for initial error in connecting to pod. -. "../retry-with-backoff.sh" -retry_with_backoff kubectl cp ./remote-scripts/setup-pod.sh ${POD_NAME}:/tmp/setup-pod.sh +../retry-with-backoff.sh kubectl cp ./remote-scripts/setup-pod.sh ${POD_NAME}:/tmp/setup-pod.sh kubectl exec ${POD_NAME} -- /tmp/setup-pod.sh kubectl exec ${POD_NAME} -- git --version set +x diff --git a/.evergreen/retry-with-backoff.sh b/.evergreen/retry-with-backoff.sh old mode 100644 new mode 100755 index fc3d141b..d929c2e5 --- a/.evergreen/retry-with-backoff.sh +++ b/.evergreen/retry-with-backoff.sh @@ -1,4 +1,4 @@ -# shellcheck shell=sh +#!/usr/bin/env bash # # (Copied over from Compass - https://github.com/mongodb-js/compass/blob/b6fec9cbbb2c6949e9ece3fffe861c3f52e30a4f/.evergreen/retry-with-backoff.sh) # Retries a command a with backoff. @@ -15,7 +15,7 @@ retry_with_backoff() { local attempt=0 local exitCode=0 - command="$@" + command="$*" while [[ $attempt -lt $max_attempts ]]; do attempt_prompt=$(( attempt + 1 )) echo "retry_with_backoff: running '${command}' - attempt n. ${attempt_prompt} ..." @@ -40,3 +40,4 @@ retry_with_backoff() { return $exitCode } +retry_with_backoff "$@" diff --git a/.evergreen/run-atlas-proxy.sh b/.evergreen/run-atlas-proxy.sh index d474e75f..81fe7a61 100755 --- a/.evergreen/run-atlas-proxy.sh +++ b/.evergreen/run-atlas-proxy.sh @@ -67,7 +67,8 @@ cd atlasproxy # This section copied from atlasproxy's .evergreen.yml: <<< export PATH="/opt/golang/go1.11/bin:$PATH" export GOROOT="/opt/golang/go1.11" -export GOPATH=`pwd`/.gopath +GOPATH=`pwd`/.gopath +export GOPATH go version ./gpm export MONGO_DIR="$DRIVERS_TOOLS/mongodb/bin" diff --git a/.evergreen/run-load-balancer.sh b/.evergreen/run-load-balancer.sh index 7c3577e7..98115b0b 100755 --- a/.evergreen/run-load-balancer.sh +++ b/.evergreen/run-load-balancer.sh @@ -80,7 +80,7 @@ EOF_HAPROXY_CONFIG stop() { if [[ -f "$DRIVERS_TOOLS/haproxy.pid" ]]; then echo "Stopping HAProxy..." - kill -USR1 $(cat $DRIVERS_TOOLS/haproxy.pid) + kill -USR1 "$(cat "$DRIVERS_TOOLS/haproxy.pid")" rm $DRIVERS_TOOLS/haproxy.conf $DRIVERS_TOOLS/haproxy.pid fi } diff --git a/.evergreen/run-orchestration.sh b/.evergreen/run-orchestration.sh index 1205f625..592a4bfc 100755 --- a/.evergreen/run-orchestration.sh +++ b/.evergreen/run-orchestration.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck shell=sh set -o errexit # Exit the script with error if any of the commands fail # Supported environment variables: @@ -19,6 +20,7 @@ set -o errexit # Exit the script with error if any of the commands fail # See https://stackoverflow.com/questions/35006457/choosing-between-0-and-bash-source/35006505#35006505 # Why we need this syntax when sh is not aliased to bash (this script must be able to be called from sh) +# shellcheck disable=SC3028 SCRIPT_DIR=$(dirname ${BASH_SOURCE:-$0}) . $SCRIPT_DIR/handle-paths.sh diff --git a/.evergreen/secrets_handling/setup-secrets.sh b/.evergreen/secrets_handling/setup-secrets.sh index af50b389..be20acef 100755 --- a/.evergreen/secrets_handling/setup-secrets.sh +++ b/.evergreen/secrets_handling/setup-secrets.sh @@ -18,10 +18,10 @@ pushd $SCRIPT_DIR/../auth_aws > /dev/null . ./activate-authawsvenv.sh popd > /dev/null -ALL_ARGS="$@" +ALL_ARGS="$*" echo "Getting secrets: ${ALL_ARGS}..." python $SCRIPT_DIR/setup_secrets.py $ALL_ARGS -source $(pwd)/secrets-export.sh +source "$(pwd)/secrets-export.sh" echo "Getting secrets: $ALL_ARGS... done." # Restore the script dir if we've overridden it. diff --git a/.evergreen/serverless/create-instance.sh b/.evergreen/serverless/create-instance.sh index 2a182ae8..3bb1c8a4 100755 --- a/.evergreen/serverless/create-instance.sh +++ b/.evergreen/serverless/create-instance.sh @@ -48,7 +48,7 @@ SERVERLESS_DRIVERS_GROUP # Ensure that all variables required to run the test are set, otherwise throw # an error. -for VARNAME in ${VARLIST[*]}; do +for VARNAME in "${VARLIST[@]}"; do [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; done @@ -70,7 +70,7 @@ export DEPLOYMENT_TYPE=serverless # Note: backingProviderName and regionName below should correspond to the # multi-tenant MongoDB (MTM) associated with $SERVERLESS_DRIVERS_GROUP. -export DEPLOYMENT_DATA=$(cat <> ./secrets-export.sh echo "SERVERLESS_INSTANCE_NAME: \"$SERVERLESS_INSTANCE_NAME\"" > $CURRENT_DIR/serverless-expansion.yml -export SERVERLESS_URI=$(check_deployment) +SERVERLESS_URI=$(check_deployment) +export SERVERLESS_URI + echo "SERVERLESS_URI=$SERVERLESS_URI" if [ $SERVERLESS_URI = "null" ]; then exit 1 diff --git a/.evergreen/serverless/delete-instance.sh b/.evergreen/serverless/delete-instance.sh index c3dd145f..b7bec0ce 100755 --- a/.evergreen/serverless/delete-instance.sh +++ b/.evergreen/serverless/delete-instance.sh @@ -27,7 +27,7 @@ SERVERLESS_API_PUBLIC_KEY # Ensure that all variables required to run the test are set, otherwise throw # an error. -for VARNAME in ${VARLIST[*]}; do +for VARNAME in "${VARLIST[@]}"; do [[ -z "${!VARNAME:-}" ]] && echo "ERROR: $VARNAME not set" && exit 1; done diff --git a/.evergreen/start-orchestration.sh b/.evergreen/start-orchestration.sh index fe7ad3be..0c1fa5e7 100755 --- a/.evergreen/start-orchestration.sh +++ b/.evergreen/start-orchestration.sh @@ -16,7 +16,7 @@ set -o errexit # Exit the script with error if any of the commands fail MONGO_ORCHESTRATION_HOME="$1" -echo From shell `date` > $MONGO_ORCHESTRATION_HOME/server.log +echo "From shell `date`" > $MONGO_ORCHESTRATION_HOME/server.log SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) . $SCRIPT_DIR/handle-paths.sh diff --git a/.evergreen/stop-orchestration.sh b/.evergreen/stop-orchestration.sh index 69068e1e..ccbb3121 100755 --- a/.evergreen/stop-orchestration.sh +++ b/.evergreen/stop-orchestration.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash +# shellcheck shell=sh + set -o errexit # Exit the script with error if any of the commands fail +# shellcheck disable=SC3028 SCRIPT_DIR=$(dirname "${BASH_SOURCE:-"$0"}") . "$SCRIPT_DIR/handle-paths.sh" diff --git a/.evergreen/teardown.sh b/.evergreen/teardown.sh index d00a7a54..9c73525b 100755 --- a/.evergreen/teardown.sh +++ b/.evergreen/teardown.sh @@ -28,9 +28,9 @@ fi # Clean up docker. if command -v docker &> /dev/null; then # Kill all containers. - docker rm $(docker ps -a -q) &> /dev/null || true + docker rm "$(docker ps -a -q)" &> /dev/null || true # Remove all images. - docker rmi -f $(docker images -a -q) &> /dev/null || true + docker rmi -f "$(docker images -a -q)" &> /dev/null || true # Remove all generated docker files pushd docker if command -v sudo &> /dev/null; then @@ -44,13 +44,13 @@ fi # Move all child log files into $DRIVERS_TOOLS/.evergreen/test_logs.tar.gz. LOG_DIR="$(mktemp -d)" # Prepend the parent directory name to the file name. -find "$(pwd -P)" -name \*.log -exec bash -c 'x="{}"; cp $x '"${LOG_DIR}"'/$(basename $(dirname $x))_$(basename $x)' \; +find "$(pwd -P)" -name \*.log -exec bash -c 'x="$1"; cp $x '"${LOG_DIR}"'/$(basename $(dirname $x))_$(basename $x)' shell {} \; # Handle files from the .evergreen directory. pushd $LOG_DIR find . -name .evergreen_\* -exec bash -c 'mv $0 ${0/.evergreen_/}' {} \; popd # Slurp into a tar file. -tar zcvf $(pwd -P)/test_logs.tar.gz -C $LOG_DIR/ . +tar zcvf "$(pwd -P)/test_logs.tar.gz" -C $LOG_DIR/ . rm -rf $LOG_DIR popd diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 887efbb8..eb00664e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,7 +26,7 @@ repos: hooks: - id: shellcheck name: shellcheck - args: ["--severity=error"] + args: ["--severity=warning"] - repo: https://github.com/sirosen/check-jsonschema rev: 0.27.0