test(NODE-5738): update data lake test scripts (#3920)

mongodb · Nov 10, 2023 · c698918 · c698918
1 parent 3934465
commit c698918
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 27 deletions.
diff --git a/.evergreen/ci_matrix_constants.js b/.evergreen/ci_matrix_constants.js
@@ -19,6 +19,7 @@ const DEFAULT_OS = 'rhel80-large';
 const WINDOWS_OS = 'windows-vsCurrent-large';
 const MACOS_OS = 'macos-1100';
 const UBUNTU_OS = 'ubuntu1804-large';
+const UBUNTU_22_OS = 'ubuntu2204-large'
 const DEBIAN_OS = 'debian11-small';
 
 module.exports = {
@@ -34,5 +35,6 @@ module.exports = {
   WINDOWS_OS,
   MACOS_OS,
   UBUNTU_OS,
+  UBUNTU_22_OS,
   DEBIAN_OS
 };
diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml
@@ -76,13 +76,15 @@ functions:
       params:
         script: |
           ${PREPARE_SHELL}
-          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
     - command: shell.exec
       params:
         background: true
         script: |
           ${PREPARE_SHELL}
-          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
+          sleep 1
+          docker ps
 
   "bootstrap kms servers":
     - command: subprocess.exec
@@ -1032,6 +1034,12 @@ functions:
           - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
 
 tasks:
+  - name: 'test-atlas-data-lake'
+    tags: ["datalake", "mongohouse"]
+    commands:
+      - func: 'install dependencies'
+      - func: 'bootstrap mongohoused'
+      - func: 'run data lake tests'
   - name: "test-serverless"
     tags: ["serverless"]
     commands:

diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -51,13 +51,15 @@ functions:
       params:
         script: |
           ${PREPARE_SHELL}
-          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/build-mongohouse-local.sh
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/pull-mongohouse-image.sh
     - command: shell.exec
       params:
         background: true
         script: |
           ${PREPARE_SHELL}
-          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
+          DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
+          sleep 1
+          docker ps
   bootstrap kms servers:
     - command: subprocess.exec
       params:
@@ -970,6 +972,14 @@ functions:
         args:
           - ${PROJECT_DIRECTORY}/.evergreen/run-benchmarks.sh
 tasks:
+  - name: test-atlas-data-lake
+    tags:
+      - datalake
+      - mongohouse
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongohoused
+      - func: run data lake tests
   - name: test-serverless
     tags:
       - serverless
@@ -1454,11 +1464,6 @@ tasks:
     commands:
       - func: install dependencies
       - func: run atlas tests
-  - name: test-atlas-data-lake
-    commands:
-      - func: install dependencies
-      - func: bootstrap mongohoused
-      - func: run data lake tests
   - name: test-5.0-load-balanced
     tags:
       - latest
@@ -3515,7 +3520,6 @@ buildvariants:
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
       - test-atlas-connectivity
-      - test-atlas-data-lake
       - test-5.0-load-balanced
       - test-6.0-load-balanced
       - test-latest-load-balanced
@@ -3567,7 +3571,6 @@ buildvariants:
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
       - test-atlas-connectivity
-      - test-atlas-data-lake
       - test-5.0-load-balanced
       - test-6.0-load-balanced
       - test-latest-load-balanced
@@ -3619,7 +3622,6 @@ buildvariants:
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
       - test-atlas-connectivity
-      - test-atlas-data-lake
       - test-5.0-load-balanced
       - test-6.0-load-balanced
       - test-latest-load-balanced
@@ -3669,7 +3671,6 @@ buildvariants:
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
       - test-atlas-connectivity
-      - test-atlas-data-lake
       - test-5.0-load-balanced
       - test-6.0-load-balanced
       - test-latest-load-balanced
@@ -3718,7 +3719,6 @@ buildvariants:
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
       - test-atlas-connectivity
-      - test-atlas-data-lake
       - test-5.0-load-balanced
       - test-6.0-load-balanced
       - test-latest-load-balanced
@@ -3768,7 +3768,6 @@ buildvariants:
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
       - test-atlas-connectivity
-      - test-atlas-data-lake
       - test-5.0-load-balanced
       - test-6.0-load-balanced
       - test-latest-load-balanced
@@ -3814,7 +3813,6 @@ buildvariants:
       - test-3.6-replica_set
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
-      - test-atlas-data-lake
       - test-socks5
       - test-socks5-tls
       - test-zstd-compression
@@ -3859,7 +3857,6 @@ buildvariants:
       - test-3.6-replica_set
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
-      - test-atlas-data-lake
       - test-socks5
       - test-socks5-tls
       - test-tls-support-latest
@@ -3902,7 +3899,6 @@ buildvariants:
       - test-3.6-replica_set
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
-      - test-atlas-data-lake
       - test-socks5
       - test-socks5-tls
       - test-tls-support-latest
@@ -3945,7 +3941,6 @@ buildvariants:
       - test-3.6-replica_set
       - test-3.6-sharded_cluster
       - test-latest-server-v1-api
-      - test-atlas-data-lake
       - test-socks5
       - test-socks5-tls
       - test-zstd-compression
@@ -4073,6 +4068,13 @@ buildvariants:
       - run-custom-csfle-tests-5.0-pinned-commit
       - run-custom-csfle-tests-rapid-pinned-commit
       - run-custom-csfle-tests-latest-pinned-commit
+  - name: ubuntu2204-test-atlas-data-lake
+    display_name: Atlas Data Lake Tests
+    run_on: ubuntu2204-large
+    expansions:
+      NODE_LTS_VERSION: 20
+    tasks:
+      - test-atlas-data-lake
   - name: rhel8-test-serverless
     display_name: Serverless Test
     run_on: rhel80-large

diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js
@@ -15,6 +15,7 @@ const {
   WINDOWS_OS,
   MACOS_OS,
   UBUNTU_OS,
+  UBUNTU_22_OS,
   DEBIAN_OS
 } = require('./ci_matrix_constants');
 
@@ -122,14 +123,6 @@ TASKS.push(
       tags: ['atlas-connect'],
       commands: [{ func: 'install dependencies' }, { func: 'run atlas tests' }]
     },
-    {
-      name: 'test-atlas-data-lake',
-      commands: [
-        { func: 'install dependencies' },
-        { func: 'bootstrap mongohoused' },
-        { func: 'run data lake tests' }
-      ]
-    },
     {
       name: 'test-5.0-load-balanced',
       tags: ['latest', 'sharded_cluster', 'load_balancer'],
@@ -622,6 +615,16 @@ BUILD_VARIANTS.push({
   tasks: oneOffFuncAsTasks.map(({ name }) => name)
 });
 
+BUILD_VARIANTS.push({
+  name: 'ubuntu2204-test-atlas-data-lake',
+  display_name: 'Atlas Data Lake Tests',
+  run_on: UBUNTU_22_OS,
+  expansions: {
+    NODE_LTS_VERSION: LATEST_LTS
+  },
+  tasks: ['test-atlas-data-lake']
+});
+
 // special case for serverless testing
 BUILD_VARIANTS.push({
   name: 'rhel8-test-serverless',