Merge branch 'main' into herrBez-patch-3

elastic · May 28, 2024 · b9d4de7 · b9d4de7
2 parents ab77ac8 + 19499ae
commit b9d4de7
Show file tree

Hide file tree

Showing 17 changed files with 4,080 additions and 3,949 deletions.
diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml
@@ -238,8 +238,6 @@ steps:
               context: "x-pack/filebeat: macOS x86_64 Unit Tests"
 
       - label: ":mac: x-pack/filebeat macOS arm64 Unit Tests"
-        skip: "Skipping due to elastic/beats#33036"
-        # https://github.com/elastic/beats/issues/33036
         if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail

diff --git a/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml b/.buildkite/x-pack/pipeline.xpack.osquerybeat.yml
@@ -2,8 +2,6 @@
 name: "beats-xpack-osquerybeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
-
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
@@ -27,8 +25,9 @@ env:
 steps:
   - group: "x-pack/osquerybeat Mandatory Tests"
     key: "x-pack-osquerybeat-mandatory-tests"
+
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
+      - label: ":ubuntu: x-pack/osquerybeat Linux x86_64 Unit Tests"
         key: "mandatory-linux-unit-test"
         command: |
           cd x-pack/osquerybeat
@@ -45,9 +44,9 @@ steps:
           - "x-pack/osquerybeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/osquerybeat: Ubuntu Unit Tests"
+              context: "x-pack/osquerybeat: Linux x86_64 Unit Tests"
 
-      - label: ":go: Go Integration Tests"
+      - label: ":ubuntu: x-pack/osquerybeat Go Integration Tests"
         key: "mandatory-int-test"
         command: |
           cd x-pack/osquerybeat
@@ -66,7 +65,7 @@ steps:
           - github_commit_status:
               context: "x-pack/osquerybeat: Go Integration Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+      - label: ":windows: x-pack/osquerybeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
@@ -85,9 +84,9 @@ steps:
           - "x-pack/osquerybeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/osquerybeat: Windows 2016 Unit Tests"
+              context: "x-pack/osquerybeat: Win-2016 Unit Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+      - label: ":windows: x-pack/osquerybeat Win-2022 Unit Tests"
         command: |
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
@@ -106,13 +105,14 @@ steps:
           - "x-pack/osquerybeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/osquerybeat: Windows 2022 Unit Tests"
+              context: "x-pack/osquerybeat: Win-2022 Unit Tests"
 
   - group: "x-pack/osquerybeat Extended Windows Tests"
     key: "x-pack-osquerybeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
-      - label: ":windows: Windows 10 Unit Tests"
+      - label: ":windows: x-pack/osquerybeat Win 10 Unit Tests"
         command: |
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
@@ -131,9 +131,9 @@ steps:
           - "x-pack/osquerybeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/osquerybeat: Windows 10 Unit Tests"
+              context: "x-pack/osquerybeat: Win 10 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
+      - label: ":windows: x-pack/osquerybeat Win 11 Unit Tests"
         command: |
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
@@ -152,9 +152,9 @@ steps:
           - "x-pack/osquerybeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/osquerybeat: Windows 11 Unit Tests"
+              context: "x-pack/osquerybeat: Win 11 Unit Tests"
 
-      - label: ":windows: Windows 2019 Unit Tests"
+      - label: ":windows: x-pack/osquerybeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/osquerybeat
           mage build unitTest
@@ -178,8 +178,9 @@ steps:
   - group: "x-pack/osquerybeat Extended Tests"
     key: "x-pack-osquerybeat-extended-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false"  || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
+
     steps:
-      - label: ":mac: MacOS x86_64 Unit Tests"
+      - label: ":mac: x-pack/osquerybeat macOS x86_64 Unit Tests"
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -195,9 +196,9 @@ steps:
           - "x-pack/osquerybeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/osquerybeat: MacOS x86_64 Unit Tests"
+              context: "x-pack/osquerybeat: macOS x86_64 Unit Tests"
 
-      - label: ":mac: MacOS arm64 Unit Tests"
+      - label: ":mac: x-pack/osquerybeat macOS arm64 Unit Tests"
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -213,7 +214,7 @@ steps:
           - "x-pack/osquerybeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/osquerybeat: MacOS arm64 Unit Tests"
+              context: "x-pack/osquerybeat: macOS arm64 Unit Tests"
 
   - wait: ~
     # with PRs, we want to run packaging only if mandatory tests succeed
@@ -226,7 +227,7 @@ steps:
   - group: "x-pack/osquerybeat Packaging"
     key: "x-pack-querybeat-packaging"
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":ubuntu: Packaging Linux"
         key: "packaging-linux"
         command: |
           .buildkite/scripts/install-msitools.sh
@@ -247,4 +248,3 @@ steps:
         notify:
           - github_commit_status:
               context: "x-pack/osquerybeat: Packaging Linux"
-      # no packaging arm step since there wasn't one on Jenkins, see https://github.com/elastic/beats/blob/0f118b18d36ce1627af73104c98b109e5d5f91f4/x-pack/osquerybeat/Jenkinsfile.yml#L93
diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml
@@ -2,14 +2,16 @@
 name: "beats-xpack-packetbeat"
 
 env:
-  AWS_ARM_INSTANCE_TYPE: "t4g.xlarge"
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
   GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   GCP_WIN_MACHINE_TYPE: "n2-standard-8"
+
   IMAGE_MACOS_ARM: "generic-13-ventura-arm"
   IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
   IMAGE_RHEL9_X86_64: "family/platform-ingest-beats-rhel-9"
-  IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
   IMAGE_WIN_10: "family/platform-ingest-beats-windows-10"
   IMAGE_WIN_11: "family/platform-ingest-beats-windows-11"
@@ -27,8 +29,9 @@ env:
 steps:
   - group: "x-pack/packetbeat Mandatory Tests"
     key: "x-pack-packetbeat-mandatory-tests"
+
     steps:
-      - label: ":linux: Ubuntu Unit Tests"
+      - label: ":ubuntu: x-pack/packetbeat Linux x86_64 Unit Tests"
         key: "mandatory-linux-unit-test"
         command: |
           cd x-pack/packetbeat
@@ -45,9 +48,9 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Ubuntu Unit Tests"
+              context: "x-pack/packetbeat: Linux x86_64 Unit Tests"
 
-      - label: ":linux: Ubuntu System Tests"
+      - label: ":ubuntu: x-pack/packetbeat Linux x86_64 System Tests"
         key: "mandatory-linux-system-test"
         command: |
           cd x-pack/packetbeat
@@ -64,9 +67,9 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Ubuntu System Tests"
+              context: "x-pack/packetbeat: Linux x86_64 System Tests"
 
-      - label: ":rhel: RHEL9 Unit Tests"
+      - label: ":rhel: x-pack/packetbeat RHEL9 Unit Tests"
         key: "mandatory-rhel9-unit-test"
         command: |
           cd x-pack/packetbeat
@@ -85,7 +88,7 @@ steps:
           - github_commit_status:
               context: "x-pack/packetbeat: RHEL9 Unit Tests"
 
-      - label: ":windows: Windows 2016 Unit Tests"
+      - label: ":windows: x-pack/packetbeat Win-2016 Unit Tests"
         command: |
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
@@ -104,9 +107,9 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Windows 2016 Unit Tests"
+              context: "x-pack/packetbeat: Win-2016 Unit Tests"
 
-      - label: ":windows: Windows 2022 Unit Tests"
+      - label: ":windows: x-pack/packetbeat Win-2022 Unit Tests"
         command: |
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
@@ -125,9 +128,9 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Windows 2022 Unit Tests"
+              context: "x-pack/packetbeat: Win-2022 Unit Tests"
 
-      - label: ":windows: Windows 2022 System Tests"
+      - label: ":windows: x-pack/packetbeat Win-2022 System Tests"
         key: "mandatory-win-2022-system-tests"
         skip: "skipping due to elastic/beats#38142"
         command: |
@@ -147,13 +150,14 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Windows 2022 System Tests"
+              context: "x-pack/packetbeat: Win-2022 System Tests"
 
   - group: "x-pack/packetbeat Extended Windows Tests"
     key: "x-pack-packetbeat-extended-win-tests"
     if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/
+
     steps:
-      - label: ":windows: Windows 10 Unit Tests"
+      - label: ":windows: x-pack/packetbeat Win 10 Unit Tests"
         command: |
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
@@ -172,9 +176,9 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Windows 10 Unit Tests"
+              context: "x-pack/packetbeat: Win 10 Unit Tests"
 
-      - label: ":windows: Windows 11 Unit Tests"
+      - label: ":windows: x-pack/packetbeat Win 11 Unit Tests"
         command: |
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
@@ -193,9 +197,9 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Windows 11 Unit Tests"
+              context: "x-pack/packetbeat: Win 11 Unit Tests"
 
-      - label: ":windows: Windows 2019 Unit Tests"
+      - label: ":windows: x-pack/packetbeat Win-2019 Unit Tests"
         command: |
           Set-Location -Path x-pack/packetbeat
           mage build unitTest
@@ -214,9 +218,9 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Windows 2019 Unit Tests"
+              context: "x-pack/packetbeat: Win-2019 Unit Tests"
 
-      - label: ":windows: Windows 10 System Tests"
+      - label: ":windows: x-pack/packetbeat Win 10 System Tests"
         key: "extended-win-10-system-tests"
         skip: "skipping due to elastic/beats#38142"
         command: |
@@ -236,38 +240,36 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Windows 2022 System Tests"
+              context: "x-pack/packetbeat: Win 10 System Tests"
 
-  - group: "x-pack/packetbeat Linux arm Extended Tests"
+  - group: "x-pack/packetbeat Extended Tests"
     key: "x-pack-packetbeat-extended-linux-arm-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
+    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm).*/
+
     steps:
-      - label: ":linux: Ubuntu ARM Unit Tests"
+      - label: ":ubuntu: x-pack/packetbeat Linux arm64 Unit Tests"
         key: "extended-arm64-unit-test"
+        if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         command: |
           cd x-pack/packetbeat
           mage build unitTest
-        if: build.env("GITHUB_PR_LABELS") =~ /.*arm.*/
         retry:
           automatic:
             - limit: 3
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         artifact_paths:
           - "x-pack/packetbeat/build/*.xml"
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Ubuntu ARM Unit Tests"
+              context: "x-pack/packetbeat: Linux arm64 Unit Tests"
 
-  - group: "x-pack/packetbeat MacOS Extended Tests"
-    key: "x-pack-packetbeat-extended-macos-tests"
-    if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
-    steps:
-      - label: ":mac: MacOS Unit Tests"
+      - label: ":mac: x-pack/packetbeat macOS Unit Tests"
         key: "extended-macos-unit-tests"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -284,10 +286,11 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: MacOS Unit Tests"
+              context: "x-pack/packetbeat: macOS Unit Tests"
 
-      - label: ":mac: MacOS arm64 Unit Tests"
+      - label: ":mac: x-pack/packetbeat macOS arm64 Unit Tests"
         key: "macos-arm64-unit-tests-extended"
+        if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/
         command: |
           set -euo pipefail
           source .buildkite/scripts/install_macos_tools.sh
@@ -304,16 +307,17 @@ steps:
           - "x-pack/packetbeat/build/*.json"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: MacOS arm64 Unit Tests"
+              context: "x-pack/packetbeat: macOS arm64 Unit Tests"
 
   - group: "x-pack/packetbeat Packaging"
     key: "x-pack-packetbeat-packaging"
     if: build.env("BUILDKITE_PULL_REQUEST") != "false"
     depends_on:
       - step: "x-pack-packetbeat-mandatory-tests"
         allow_failure: false
+
     steps:
-      - label: ":linux: Packaging Linux"
+      - label: ":ubuntu: x-pack/packetbeat Packaging Linux"
         key: "packaging-linux"
         command: |
           cd x-pack/packetbeat
@@ -334,7 +338,7 @@ steps:
           - github_commit_status:
               context: "x-pack/packetbeat: Packaging Linux"
 
-      - label: ":linux: Packaging ARM"
+      - label: ":ubuntu: x-pack/packetbeat Packaging arm64"
         key: "packaging-arm"
         command: |
           cd x-pack/packetbeat
@@ -345,11 +349,11 @@ steps:
         timeout_in_minutes: 20
         agents:
           provider: "aws"
-          imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
           instanceType: "${AWS_ARM_INSTANCE_TYPE}"
         env:
           PLATFORMS: "linux/arm64"
           PACKAGES: "docker"
         notify:
           - github_commit_status:
-              context: "x-pack/packetbeat: Packaging Linux ARM"
+              context: "x-pack/packetbeat: Packaging Linux arm64"