From a52cafc7db653da5a820c21ee5b05d22ee654162 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Wed, 15 May 2024 17:08:37 +0300 Subject: [PATCH] Auditbeat: align Buildkite steps to match Jenkins (#39568) Updated auditbeat pipeline to match same steps & commands as in Jenkins on 7.17 branch and removed dynamic pipeline generation script --- .buildkite/auditbeat/auditbeat-pipeline.yml | 175 ++------------ .../auditbeat/generate_auditbeat_pipeline.sh | 216 ------------------ 2 files changed, 24 insertions(+), 367 deletions(-) delete mode 100644 .buildkite/auditbeat/generate_auditbeat_pipeline.sh diff --git a/.buildkite/auditbeat/auditbeat-pipeline.yml b/.buildkite/auditbeat/auditbeat-pipeline.yml index 2f43687dbb2..0e9838252de 100644 --- a/.buildkite/auditbeat/auditbeat-pipeline.yml +++ b/.buildkite/auditbeat/auditbeat-pipeline.yml @@ -31,28 +31,27 @@ steps: key: "auditbeat-mandatory-tests" steps: - - label: ":ubuntu: Auditbeat Unit Tests" + - label: ":linux: Auditbeat Ubuntu arm64 Unit Tests" + key: "auditbeat-extended-arm64-unit-tests" command: | - set -euo pipefail cd auditbeat mage build unitTest retry: automatic: - limit: 3 agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" + provider: "aws" + imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" + instanceType: "${AWS_ARM_INSTANCE_TYPE}" artifact_paths: - "auditbeat/build/*.xml" - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Ubuntu Unit Tests" + context: "auditbeat: Linux arm64 Unit Tests" - - label: ":rhel: Auditbeat Unit Tests" + - label: ":ubuntu: Auditbeat Unit Tests" command: | - set -euo pipefail cd auditbeat mage build unitTest retry: @@ -60,14 +59,14 @@ steps: - limit: 3 agents: provider: "gcp" - image: "${IMAGE_RHEL9}" + image: "${IMAGE_UBUNTU_X86_64}" machineType: "${GCP_DEFAULT_MACHINE_TYPE}" artifact_paths: - "auditbeat/build/*.xml" - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: RHEL Unit Tests" + context: "auditbeat: Ubuntu Unit Tests" - label: "Module compat tests: next major" # Run module integration tests under next major of Elastic stack. @@ -94,36 +93,32 @@ steps: - github_commit_status: context: "auditbeat: Module compat tests / next major" - - label: ":windows: Auditbeat Win-2016 Unit Tests" + - label: ":linux: Auditbeat Crosscompile" command: | - Set-Location -Path auditbeat - mage build unitTest + make -C auditbeat crosscompile + env: + GOX_FLAGS: "-arch amd64" retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" + image: "${IMAGE_UBUNTU_X86_64}" + machineType: "${GCP_HI_PERF_MACHINE_TYPE}" notify: - github_commit_status: - context: "auditbeat: Windows 2016 Unit Tests" + context: "auditbeat: Crosscompile" - - label: ":windows: Auditbeat Win-2022 Unit Tests" + - label: ":windows: Auditbeat Win-2019 Unit Tests" command: | Set-Location -Path auditbeat mage build unitTest retry: automatic: - - limit: 3 + - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_2022}" + image: "${IMAGE_WIN_2019}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 200 disk_type: "pd-ssd" @@ -132,92 +127,13 @@ steps: - "auditbeat/build/*.json" notify: - github_commit_status: - context: "auditbeat: Windows 2022 Unit Tests" - - - label: ":linux: Auditbeat Crosscompile" - command: | - make -C auditbeat crosscompile - env: - GOX_FLAGS: "-arch amd64" - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" - notify: - - github_commit_status: - context: "auditbeat: Crosscompile" + context: "auditbeat: Windows 2019 Unit Tests" - group: "Auditbeat Extended Tests" key: "auditbeat-extended-tests" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*(macOS|arm|integrations).*/ steps: - - label: ":linux: Auditbeat Ubuntu Integration Tests" - key: "auditbeat-extended-integ-tests" - if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ - command: | - set -euo pipefail - cd auditbeat - mage build integTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - notify: - - github_commit_status: - context: "auditbeat: Linux Integration Tests" - - - label: ":linux: Auditbeat Ubuntu arm64 Integration Tests" - key: "auditbeat-extended-arm64-integ-tests" - if: build.env("GITHUB_PR_LABELS") =~ /.*integrations.*/ - command: | - set -euo pipefail - cd auditbeat - mage build integTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - notify: - - github_commit_status: - context: "auditbeat: Linux arm64 Integration Tests" - - - label: ":linux: Auditbeat Ubuntu arm64 Unit Tests" - key: "auditbeat-extended-arm64-unit-tests" - if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*arm.*/ - command: | - set -euo pipefail - cd auditbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - notify: - - github_commit_status: - context: "auditbeat: Linux arm64 Unit Tests" - - label: ":mac: Auditbeat macOS x86_64 Unit Tests" if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ command: | @@ -238,33 +154,13 @@ steps: - github_commit_status: context: "auditbeat: macOS x86_64 Unit Tests" - - label: ":mac: Auditbeat macOS arm64 ARM Unit Tests" - if: build.env("GITHUB_PR_LABELS") =~ /.*macOS.*/ - command: | - set -euo pipefail - source .buildkite/scripts/install_macos_tools.sh - cd auditbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - notify: - - github_commit_status: - context: "auditbeat: macOS arm64 Unit Tests" - - group: "Auditbeat Windows Extended Testing" key: "auditbeat-extended-tests-win" if: build.env("BUILDKITE_PULL_REQUEST") == "false" || build.env("GITHUB_PR_LABELS") =~ /.*[Ww]indows.*/ steps: - - label: ":windows: Auditbeat Win-2019 Unit Tests" - key: "auditbeat-extended-win-2019-unit-tests" + - label: ":windows: Auditbeat Win-2016 Unit Tests" + key: "auditbeat-extended-win-2016-unit-tests" command: | Set-Location -Path auditbeat mage build unitTest @@ -273,7 +169,7 @@ steps: - limit: 3 agents: provider: "gcp" - image: "${IMAGE_WIN_2019}" + image: "${IMAGE_WIN_2016}" machine_type: "${GCP_WIN_MACHINE_TYPE}" disk_size: 100 disk_type: "pd-ssd" @@ -305,27 +201,6 @@ steps: - github_commit_status: context: "auditbeat: Extended Windows 10 Unit Tests" - - label: ":windows: Auditbeat Win-11 Unit Tests" - key: "auditbeat-extended-win-11-unit-tests" - command: | - Set-Location -Path auditbeat - mage build unitTest - retry: - automatic: - - limit: 3 - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 100 - disk_type: "pd-ssd" - artifact_paths: - - "auditbeat/build/*.xml" - - "auditbeat/build/*.json" - notify: - - github_commit_status: - context: "auditbeat: Extended Windows 11 Unit Tests" - - wait: ~ # with PRs, we want to run packaging only if mandatory tests succeed # for other cases, e.g. merge commits, we want to run packaging (and publish) independently of other tests @@ -342,7 +217,6 @@ steps: PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64" SNAPSHOT: true command: | - set -euo pipefail cd auditbeat mage package retry: @@ -364,7 +238,6 @@ steps: PACKAGES: "docker" SNAPSHOT: true command: | - set -euo pipefail cd auditbeat mage package retry: diff --git a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh b/.buildkite/auditbeat/generate_auditbeat_pipeline.sh deleted file mode 100644 index 56befb51d25..00000000000 --- a/.buildkite/auditbeat/generate_auditbeat_pipeline.sh +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env bash - -source .buildkite/scripts/common.sh - -set -euo pipefail - -pipelineName="pipeline.auditbeat-dynamic.yml" - -# TODO: steps: must be always included -echo "Add the mandatory and extended tests without additional conditions into the pipeline" -if are_conditions_met_mandatory_tests; then - cat > $pipelineName <<- YAML - -steps: - - group: "Auditbeat Mandatory Testing" - key: "mandatory-tests" - - steps: - - label: ":ubuntu: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: linux/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":rhel: Unit Tests" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: rhel/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_RHEL9}" - machineType: "${GCP_DEFAULT_MACHINE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2016 Unit Tests" - key: "windows-2016" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2016/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2016}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-2022 Unit Tests" - key: "windows-2022" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: windows-2022/Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2022}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":linux: Crosscompile" - command: - - "make -C auditbeat crosscompile" - env: - GOX_FLAGS: "-arch amd64" - notify: - - github_commit_status: - context: "Auditbeat: Crosscompile" - agents: - provider: "gcp" - image: "${IMAGE_UBUNTU_X86_64}" - machineType: "${GCP_HI_PERF_MACHINE_TYPE}" -YAML -else - echo "The conditions don't match to requirements for generating pipeline steps." - exit 0 -fi - -echo "Check and add the Extended Tests into the pipeline" - -if are_conditions_met_arm_tests || are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - group: "Extended Tests" - key: "extended-tests" - steps: - -YAML -fi - -if are_conditions_met_macos_tests; then - cat >> $pipelineName <<- YAML - - - label: ":mac: MacOS Unit Tests" - key: "macos-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_X86_64}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":mac: MacOS ARM Unit Tests" - key: "macos-arm64-unit-tests-extended" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: MacOS ARM Unit Tests" - agents: - provider: "orka" - imagePrefix: "${IMAGE_MACOS_ARM}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_arm_tests; then - cat >> $pipelineName <<- YAML - - label: ":linux: ARM Ubuntu Unit Tests" - key: "extended-arm64-unit-test" - command: "cd ${BEATS_PROJECT_NAME} && mage unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Unit Tests ARM" - agents: - provider: "aws" - imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}" - instanceType: "${AWS_ARM_INSTANCE_TYPE}" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - -YAML -fi - -if are_conditions_met_win_tests; then - cat >> $pipelineName <<- YAML - - group: "Windows Extended Testing" - key: "extended-tests-win" - - steps: - - label: ":windows:-2019 Unit Tests" - key: "windows-2019-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-2019 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_2019}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-11 Unit Tests" - key: "windows-11-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-11 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_11}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" - - - label: ":windows:-10 Unit Tests" - key: "windows-10-extended" - command: "mage -d ${BEATS_PROJECT_NAME} unitTest" - notify: - - github_commit_status: - context: "Auditbeat: Win-10 Unit Tests" - agents: - provider: "gcp" - image: "${IMAGE_WIN_10}" - machine_type: "${GCP_WIN_MACHINE_TYPE}" - disk_size: 200 - disk_type: "pd-ssd" - artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*" -YAML -fi - -echo "Check and add the Packaging into the pipeline" -if are_conditions_met_packaging; then -cat >> $pipelineName <<- YAML - - group: "Packaging" - key: "packaging" - depends_on: - - "mandatory-tests" - - steps: - - label: Package pipeline - commands: ".buildkite/scripts/packaging/package-step.sh" - notify: - - github_commit_status: - context: "Auditbeat: Packaging" - -YAML -fi - -echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public -cat $pipelineName - -echo "--- Loading dynamic steps" -buildkite-agent pipeline upload $pipelineName