Merge branch 'main' into main

elastic · Sep 30, 2024 · e6fcb56 · e6fcb56
2 parents b46d785 + 74faa2f
commit e6fcb56
Show file tree

Hide file tree

Showing 181 changed files with 4,730 additions and 4,992 deletions.
diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml
@@ -19,6 +19,8 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   # Other deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -40,14 +42,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "auditbeat: check/update"
 
+      - label: "Auditbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "auditbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests

diff --git a/.buildkite/aws-tests-pipeline.yml b/.buildkite/aws-tests-pipeline.yml
@@ -19,7 +19,7 @@ env:
 steps:
   - label: ":ubuntu: x-pack/filebeat: AWS Tests"
     key: "x-pack-filebeat-extended-cloud-test"
-    skip: "skipping as it was on Jenkins: elastic/ingest-dev#3467"
+    skip: "skipping:  elastic/ingest-dev#3467"
     # Related issue: https://github.com/elastic/ingest-dev/issues/3467
     env:
       MODULE_DIR: "x-pack/filebeat/input/awss3/_meta/terraform"

diff --git a/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml b/.buildkite/deploy/kubernetes/deploy-k8s-pipeline.yml
@@ -8,11 +8,24 @@ env:
   # Other deps
   ASDF_KIND_VERSION: "0.20.0"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
 steps:
   - group: "Deploy/K8S"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
 
     steps:
+    - label: "Deploy/K8S: Run pre-commit"
+      command: "pre-commit run --all-files"
+      agents:
+        image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+        cpu: "2000m"
+        memory: "4Gi"
+        useCustomGlobalHooks: true
+      notify:
+        - github_commit_status:
+            context: "deploy/k8s: pre-commit"
+
     - label: "Checks"
       command: |
         set -euo pipefail

diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml
@@ -18,6 +18,8 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   #Deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -32,6 +34,7 @@ env:
 steps:
   - group: "Check/Update"
     key: "filebeat-check-update"
+
     steps:
       - label: "Filebeat: Run check/update"
         command: |
@@ -42,14 +45,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "filebeat: check/update"
 
+      - label: "Filebeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "filebeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests

diff --git a/.buildkite/heartbeat/heartbeat-pipeline.yml b/.buildkite/heartbeat/heartbeat-pipeline.yml
@@ -18,9 +18,12 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   # Other deps
   ASDF_MAGE_VERSION: 1.15.0
   ASDF_NODEJS_VERSION: 18.17.1
+  ASDF_PYTHON_VERSION: 3.10.9
 
   # Unit tests
   RACE_DETECTOR: "true"
@@ -40,14 +43,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "hearbeat: check/update"
 
+      - label: "Heartbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "heartbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -135,17 +148,9 @@ steps:
           - github_commit_status:
               context: "heartbeat: Win 2022 Unit Tests"
 
+      # Heartbeat has no modules
       - label: ":ubuntu: Heartbeat: Go Integration Tests"
         command: |
-          set -euo pipefail
-          echo "~~~ Installing @elastic/synthetics"
-          npm install -g @elastic/synthetics
-
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet heartbeat
-
-          echo "~~~ Running tests"
           cd heartbeat
           mage goIntegTest
         retry:
@@ -162,14 +167,9 @@ steps:
           - github_commit_status:
               context: "heartbeat: Go Integration Tests"
 
+      # Heartbeat has no modules yet
       - label: ":ubuntu: Heartbeat: Python Integration Tests"
         command: |
-          set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet heartbeat
-
-          echo "~~~ Running tests"
           cd heartbeat
           mage pythonIntegTest
         retry:

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
@@ -6,6 +6,13 @@ set -euo pipefail
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 PRIVATE_CI_GCS_CREDENTIALS_PATH="kv/ci-shared/platform-ingest/gcp-platform-ingest-ci-service-account"
 
+if [[ "${BUILDKITE_LABEL:-}" == *"Pipeline upload"* || "${BUILDKITE_LABEL:-}" == *"Upload Pipeline"* ]]; then
+
+  if [[ "${GITHUB_PR_TRIGGER_COMMENT:-}" != "" ]]; then
+    buildkite-agent annotate "**No changes.** Triggered from comment: _${GITHUB_PR_TRIGGER_COMMENT}_" --style "info" --context 'ctx-info'
+  fi
+fi
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP_KEY" == *"system-tests"* ]]; then
     PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry -t 5 -- vault kv get -field plaintext -format=json ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET

diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml
@@ -9,6 +9,8 @@ env:
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   #Deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -30,14 +32,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "libbeat: check/update"
 
+      - label: "Libbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "libbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -67,7 +79,7 @@ steps:
           - "libbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "libbeat: Ununtu x86_64 Unit Tests"
+              context: "libbeat: Ubuntu x86_64 Unit Tests"
 
       - label: ":ubuntu: Libbeat: Go Integration Tests"
         key: "mandatory-int-test"

diff --git a/.buildkite/metricbeat/pipeline.yml b/.buildkite/metricbeat/pipeline.yml
@@ -18,10 +18,15 @@ env:
   IMAGE_WIN_2019: "family/platform-ingest-beats-windows-2019"
   IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
 
+  IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+
   # Integration Tests
   K8S_VERSION: "v1.29.0"
   ASDF_KIND_VERSION: "0.20.0"
 
+  # Module Tests
+  BEAT_PATH: "metricbeat"
+
   # Other deps
   ASDF_MAGE_VERSION: 1.15.0
 
@@ -42,14 +47,24 @@ steps:
           automatic:
             - limit: 1
         agents:
-          image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
           cpu: "4000m"
           memory: "8Gi"
           useCustomGlobalHooks: true
         notify:
           - github_commit_status:
               context: "metricbeat: check/update"
 
+      - label: "Metricbeat: Run pre-commit"
+        command: "pre-commit run --all-files"
+        agents:
+          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+          memory: "2Gi"
+          useCustomGlobalHooks: true
+        notify:
+          - github_commit_status:
+              context: "metricbeat: pre-commit"
+
   - wait: ~
     # with PRs, we want to run mandatory tests only if check/update step succeed
     # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
@@ -80,14 +95,10 @@ steps:
           - github_commit_status:
               context: "metricbeat: Ubuntu x86_64 Unit Tests"
 
-      - label: ":ubuntu: Metricbeat: Go Integration Tests"
+      - label: ":ubuntu: Metricbeat: Go Integration Tests (Module)"
         key: "mandatory-int-test"
         command: |
           set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet metricbeat
-
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat
@@ -104,16 +115,12 @@ steps:
           - "metricbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "metricbeat: Go Integration Tests"
+              context: "metricbeat: Go Integration Tests (Module)"
 
-      - label: ":ubuntu: Metricbeat: Python Integration Tests"
+      - label: ":ubuntu: Metricbeat: Python Integration Tests (Module)"
         key: "mandatory-python-int-test"
         command: |
           set -euo pipefail
-          # defines the MODULE env var based on what's changed in a PR
-          source .buildkite/scripts/changesets.sh
-          defineModuleFromTheChangeSet metricbeat
-
           echo "~~~ Running tests"
           export KUBECONFIG="$$PWD/kubecfg"
           cd metricbeat