From 1b864e4e9e0f971121a4494b98557f2fb15eb4b5 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 23 Sep 2023 18:24:01 -0500 Subject: [PATCH 01/17] DRIVERS-2543 Pull mongohouse image from ADL ECR repo --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index f0840dfce7..46d841aff5 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -97,7 +97,7 @@ functions: # If this was a patch build, doing a fresh clone would not actually test the patch cp -R ${PROJECT_DIRECTORY}/ $DRIVERS_TOOLS else - git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + git clone --branch DRIVERS-2543 https://github.com/blink1073/drivers-evergreen-tools.git $DRIVERS_TOOLS fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config From 9e969a7efa40f65f56d2e9f845d293e566f98d68 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 23 Sep 2023 18:29:03 -0500 Subject: [PATCH 02/17] fix secrets handling --- .evergreen/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 46d841aff5..d435709c48 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -312,9 +312,13 @@ functions: value: "1" "bootstrap data lake": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} - command: shell.exec type: setup params: + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} set -o xtrace From 341755b7d2c09289f67ead8baed7a16b41e987db Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 23 Sep 2023 18:31:38 -0500 Subject: [PATCH 03/17] Revert "fix secrets handling" This reverts commit 9e969a7efa40f65f56d2e9f845d293e566f98d68. --- .evergreen/config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d435709c48..46d841aff5 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -312,13 +312,9 @@ functions: value: "1" "bootstrap data lake": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - command: shell.exec type: setup params: - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} set -o xtrace From 1af2409159c6733f82f64d0d73c90b79b4077fca Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 23 Sep 2023 19:22:11 -0500 Subject: [PATCH 04/17] restore bootstrap --- .evergreen/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 46d841aff5..d435709c48 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -312,9 +312,13 @@ functions: value: "1" "bootstrap data lake": + - command: ec2.assume_role + params: + role_arn: ${aws_test_secrets_role} - command: shell.exec type: setup params: + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} set -o xtrace From fca2644dc909d75c1bd23fba1ea21e7a11660c94 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 23 Sep 2023 19:30:27 -0500 Subject: [PATCH 05/17] use podman --- .evergreen/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d435709c48..c910ec7b64 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -324,6 +324,7 @@ functions: set -o xtrace # The mongohouse build script needs to be passed the VARIANT variable, see # https://github.com/10gen/mongohouse/blob/973cc11/evergreen.yaml#L65 + alias docker=podman VARIANT=rhel84-small bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh - command: shell.exec type: setup @@ -332,6 +333,7 @@ functions: script: | ${PREPARE_SHELL} set -o xtrace + alias docker=podman bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh "stop mongo-orchestration": From 2ca06df55fdd68214fc6f91d37c0cb11d06d71e6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 23 Sep 2023 20:30:57 -0500 Subject: [PATCH 06/17] use podman --- .evergreen/config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index c910ec7b64..d435709c48 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -324,7 +324,6 @@ functions: set -o xtrace # The mongohouse build script needs to be passed the VARIANT variable, see # https://github.com/10gen/mongohouse/blob/973cc11/evergreen.yaml#L65 - alias docker=podman VARIANT=rhel84-small bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh - command: shell.exec type: setup @@ -333,7 +332,6 @@ functions: script: | ${PREPARE_SHELL} set -o xtrace - alias docker=podman bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh "stop mongo-orchestration": From dee47e7b72096c7ea463c5c339635660a73203e6 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 25 Sep 2023 10:06:31 -0500 Subject: [PATCH 07/17] try without extra creds --- .evergreen/config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d435709c48..46d841aff5 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -312,13 +312,9 @@ functions: value: "1" "bootstrap data lake": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - command: shell.exec type: setup params: - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | ${PREPARE_SHELL} set -o xtrace From 1c54cf3cd265f7580da1e3189e364cd58dcd7284 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 29 Sep 2023 12:58:10 -0500 Subject: [PATCH 08/17] debug --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 46d841aff5..02ef24d598 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -329,6 +329,7 @@ functions: ${PREPARE_SHELL} set -o xtrace bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh + podman ps "stop mongo-orchestration": - command: shell.exec From 1212d64e7a0a8674750795c35901c67d6a284e2d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 08:00:03 -0500 Subject: [PATCH 09/17] more debug --- .evergreen/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 02ef24d598..96ca85d14d 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -328,7 +328,9 @@ functions: script: | ${PREPARE_SHELL} set -o xtrace + echo "HI" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh + echo "HELLO" podman ps "stop mongo-orchestration": From 1a634be0f431196b49e723d4a7d438a16f033699 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 08:10:26 -0500 Subject: [PATCH 10/17] more debug --- .evergreen/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 96ca85d14d..55c29ab7f9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -330,6 +330,10 @@ functions: set -o xtrace echo "HI" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh + - command: shell.exec + type: setup + params: + sleep 5 echo "HELLO" podman ps From 1699d205a406fd19bbbb2c94883ef02fffa49a33 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 08:11:09 -0500 Subject: [PATCH 11/17] more debug --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 55c29ab7f9..9b47310686 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -333,6 +333,7 @@ functions: - command: shell.exec type: setup params: + script: | sleep 5 echo "HELLO" podman ps From d0b8e8a00586528f467ff11d5da5640e9bc02bc2 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 09:09:13 -0500 Subject: [PATCH 12/17] debug --- .evergreen/config.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 9b47310686..47964e8f33 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -324,17 +324,11 @@ functions: - command: shell.exec type: setup params: - background: true script: | ${PREPARE_SHELL} set -o xtrace echo "HI" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh - - command: shell.exec - type: setup - params: - script: | - sleep 5 echo "HELLO" podman ps From a9822248d1dc5931148626f4c9880c8858ac5ab3 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 09:27:34 -0500 Subject: [PATCH 13/17] try ubuntu --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 47964e8f33..fb9ba5fe65 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -3134,7 +3134,7 @@ buildvariants: - matrix_name: "data-lake-spec-tests" matrix_spec: - platform: rhel8 + platform: ubuntu-22.04 python-version: ["3.7", "3.10"] auth: "auth" c-extensions: "*" From 74bb7ad5753623830cf1d0fd7c85edd6579bc5d0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 09:32:53 -0500 Subject: [PATCH 14/17] use docker --- .evergreen/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index fb9ba5fe65..6da482add6 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -330,7 +330,7 @@ functions: echo "HI" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh echo "HELLO" - podman ps + docker ps "stop mongo-orchestration": - command: shell.exec From 932b9bc3ddde630d4f893356e8da2fe3fd8a7457 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 09:44:06 -0500 Subject: [PATCH 15/17] try with podman init --- .evergreen/config.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 6da482add6..74b20be1b9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -328,9 +328,10 @@ functions: ${PREPARE_SHELL} set -o xtrace echo "HI" + podman machine start bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh echo "HELLO" - docker ps + podman ps "stop mongo-orchestration": - command: shell.exec @@ -3134,7 +3135,7 @@ buildvariants: - matrix_name: "data-lake-spec-tests" matrix_spec: - platform: ubuntu-22.04 + platform: rhel8 python-version: ["3.7", "3.10"] auth: "auth" c-extensions: "*" From a59438866cd1e96acced978c6a9586688035f53a Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 30 Sep 2023 09:54:29 -0500 Subject: [PATCH 16/17] try with podman init --- .evergreen/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 74b20be1b9..0291631cb9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -328,6 +328,7 @@ functions: ${PREPARE_SHELL} set -o xtrace echo "HI" + podman machine init podman machine start bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh echo "HELLO" From c702e9a4bec760ce51118c01a459d816864d394b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 15 Oct 2023 13:52:59 -0500 Subject: [PATCH 17/17] use updated scripts --- .evergreen/config.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0291631cb9..598a6188f7 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -317,22 +317,15 @@ functions: params: script: | ${PREPARE_SHELL} - set -o xtrace - # The mongohouse build script needs to be passed the VARIANT variable, see - # https://github.com/10gen/mongohouse/blob/973cc11/evergreen.yaml#L65 - VARIANT=rhel84-small bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh + bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh - command: shell.exec type: setup params: script: | ${PREPARE_SHELL} - set -o xtrace - echo "HI" - podman machine init - podman machine start - bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh - echo "HELLO" - podman ps + bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh + sleep 1 + docker ps "stop mongo-orchestration": - command: shell.exec @@ -3136,7 +3129,7 @@ buildvariants: - matrix_name: "data-lake-spec-tests" matrix_spec: - platform: rhel8 + platform: ubuntu-22.04 python-version: ["3.7", "3.10"] auth: "auth" c-extensions: "*"