Remove hardcoded module definitions in CI (#39506) (#39528)

For reasons described in [^1], we hard coded `aws` or `kubernetes` for the env var `MODULE`, when no file changes were detected under `modules/` per project.

This commit reverts this and allows all module tests to run if no changesets apply. As discussed in [^1], for the tests to run properly (and not go into an infinite loop)
if there are no module changes the `MODULE` env var should **not** be set, rather than set to an empty string.

This commit is an important fix as it covers a major delta in test coverage between Jenkins and Buildkite.

Closes https://github.com/elastic/ingest-dev/issues/2993

[^1]: https://github.com/elastic/ingest-dev/issues/2993#issuecomment-2104741977

(cherry picked from commit c91503b)

Co-authored-by: Dimitrios Liappis <dimitrios.liappis@gmail.com>

.buildkite/metricbeat/pipeline.yml

@@ -53,7 +53,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           # TODO move this section to base image / pre-command hook
           echo "~~~ Installing kind"
@@ -86,7 +86,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           # TODO move this section to base image / pre-command hook
           echo "~~~ Installing kind"

.buildkite/scripts/changesets.psm1

@@ -56,17 +56,9 @@ function DefineModuleFromTheChangeSet($projectPath) {
         }
     }
 
-    # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved
-    if(!$changedModules) {
-        if($Env:BUILDKITE_PIPELINE_SLUG -eq 'beats-xpack-metricbeat') {
-            $Env:MODULE = "aws"
-        }
-        else {
-            $Env:MODULE = "kubernetes"
-        }
-    }
-    else {
-        # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export
-        $Env:MODULE = $changedModules
+    if ($changedModules) {
+        $env:MODULE = $changedModules
+        Write-Output "~~~ Set env var MODULE to [$env:MODULE]"
+        Write-Output "~~~ Resuming commands"
     }
 }

.buildkite/scripts/changesets.sh

@@ -68,13 +68,10 @@ defineModuleFromTheChangeSet() {
     fi
   done
 
-  if [[ -z "$changed_modules" ]]; then # TODO: remove this conditional when issue https://github.com/elastic/ingest-dev/issues/2993 gets resolved
-    if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-metricbeat" ]]; then
-      export MODULE="aws"
-    else
-      export MODULE="kubernetes"
-    fi
-  else
-    export MODULE="${changed_modules}"  # TODO: once https://github.com/elastic/ingest-dev/issues/2993 gets resolved, this should be the only thing we export
+  # export MODULE="" leads to an infinite loop https://github.com/elastic/ingest-dev/issues/2993
+  if [[ ! -z $changed_modules ]]; then 
+    export MODULE="${changed_modules}"
+    echo "~~~ Set env var MODULE to [$MODULE]"
+    echo "~~~ Resuming commands"
   fi
 }

.buildkite/x-pack/pipeline.xpack.auditbeat.yml

@@ -33,7 +33,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/auditbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/auditbeat
           mage update build test
         retry:

.buildkite/x-pack/pipeline.xpack.dockerlogbeat.yml

@@ -52,11 +52,9 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/dockerlogbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/dockerlogbeat
           mage goIntegTest
-        env:
-          MODULE: $MODULE
         retry:
           automatic:
            - limit: 3

.buildkite/x-pack/pipeline.xpack.filebeat.yml

@@ -51,7 +51,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/filebeat && mage goIntegTest
         retry:
           automatic:
@@ -74,7 +74,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/filebeat && mage pythonIntegTest
         retry:
           automatic:
@@ -268,7 +268,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           .buildkite/scripts/cloud_tests.sh
         env:
@@ -297,7 +297,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/filebeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           .buildkite/scripts/cloud_tests.sh
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2

.buildkite/x-pack/pipeline.xpack.metricbeat.yml

@@ -51,7 +51,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Will run tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/metricbeat && mage goIntegTest
         retry:
           automatic:
@@ -74,7 +74,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           cd x-pack/metricbeat && mage pythonIntegTest
         retry:
           automatic:
@@ -253,7 +253,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
 
           .buildkite/scripts/cloud_tests.sh
         env:
@@ -284,7 +284,7 @@ steps:
           # defines the MODULE env var based on what's changed in a PR
           source .buildkite/scripts/changesets.sh
           defineModuleFromTheChangeSet x-pack/metricbeat
-          echo "~~~ Running tests with env var MODULE=$$MODULE"
+          echo "~~~ Running tests"
           .buildkite/scripts/cloud_tests.sh
         env:
           ASDF_TERRAFORM_VERSION: 1.0.2

.buildkite/x-pack/pipeline.xpack.winlogbeat.yml

@@ -24,11 +24,9 @@ steps:
         command: |
           Import-Module ./.buildkite/scripts/changesets.psm1
           defineModuleFromTheChangeSet 'x-pack/winlogbeat'
-          Write-Output "~~~ Will run tests with env var MODULE=$$Env:MODULE"
+          Write-Output "~~~ Running tests"
           Set-Location -Path x-pack/winlogbeat
           mage build unitTest
-        env:
-          MODULE: $MODULE
         retry:
           automatic:
            - limit: 3