From 19c2807c2de476c9d231886e2726d06040e9d20a Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Wed, 16 Oct 2024 09:26:22 +0200 Subject: [PATCH] feat(vault): Migrate datadog api/app keys (#29735) --- .gitlab-ci.yml | 14 ++++---- .gitlab/e2e_install_packages/common.yml | 4 +-- .../functional_test/regression_detector.yml | 2 +- .gitlab/kernel_matrix_testing/common.yml | 12 +++---- .../kernel_matrix_testing/security_agent.yml | 2 +- .../kernel_matrix_testing/system_probe.yml | 6 ++-- .gitlab/notify/notify.yml | 10 +++--- .gitlab/pkg_metrics/pkg_metrics.yml | 2 +- .gitlab/setup/setup.yml | 4 +-- .gitlab/source_test/golang_deps_diff.yml | 4 +-- tasks/libs/common/omnibus.py | 32 +++++++++++-------- tasks/unit_tests/linter_tests.py | 2 +- tasks/unit_tests/omnibus_tests.py | 2 ++ tools/ci/junit_upload.sh | 2 +- 14 files changed, 54 insertions(+), 44 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7594334956486..02d0c606ebddf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -258,14 +258,16 @@ variables: WINGET_PAT: ci.datadog-agent.winget_pat # windows-agent # End aws ssm variables # Start vault variables + AGENT_API_KEY_ORG2: agent-api-key-org-2 # agent-devx-infra + AGENT_APP_KEY_ORG2: agent-ci-app-key-org-2 # agent-devx-infra + AGENT_GITHUB_APP: agent-github-app # agent-devx-infra + ATLASSIAN_WRITE: atlassian-write # agent-devx-infra + DOCKER_REGISTRY_RO: dockerhub-readonly # agent-delivery + INSTALL_SCRIPT_API_KEY_ORG2: install-script-api-key-org-2 # agent-devx-infra + MACOS_GITHUB_APP_1: macos-github-app-one # agent-devx-infra + MACOS_GITHUB_APP_2: macos-github-app-two # agent-devx-infra # End vault variables - ATLASSIAN_WRITE: atlassian-write - AGENT_GITHUB_APP: agent-github-app - MACOS_GITHUB_APP_1: macos-github-app-one - MACOS_GITHUB_APP_2: macos-github-app-two - DOCKER_REGISTRY_RO: dockerhub-readonly - DD_PKG_VERSION: "latest" # Job stage attempts (see https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts) diff --git a/.gitlab/e2e_install_packages/common.yml b/.gitlab/e2e_install_packages/common.yml index 1bb6c813d84cc..6216c5dec8069 100644 --- a/.gitlab/e2e_install_packages/common.yml +++ b/.gitlab/e2e_install_packages/common.yml @@ -28,7 +28,7 @@ - START_MAJOR_VERSION: [5, 6, 7] END_MAJOR_VERSION: [7] script: - - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY ) || exit $?; export DATADOG_AGENT_API_KEY + - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token ) || exit $?; export DATADOG_AGENT_API_KEY - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --src-agent-version $START_MAJOR_VERSION --dest-agent-version $END_MAJOR_VERSION --test-washer .new-e2e_script_upgrade_persisting_integrations: @@ -48,5 +48,5 @@ TEAM: agent-delivery EXTRA_PARAMS: --osversion $E2E_OSVERS --platform $E2E_PLATFORM --arch $E2E_ARCH script: - - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY) || exit $?; export DATADOG_AGENT_API_KEY + - DATADOG_AGENT_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $INSTALL_SCRIPT_API_KEY_ORG2 token) || exit $?; export DATADOG_AGENT_API_KEY - inv -e new-e2e-tests.run --targets $TARGETS --junit-tar "junit-${CI_JOB_ID}.tgz" ${EXTRA_PARAMS} --test-washer diff --git a/.gitlab/functional_test/regression_detector.yml b/.gitlab/functional_test/regression_detector.yml index c288fe0824f42..ec44f3d9fc9ed 100644 --- a/.gitlab/functional_test/regression_detector.yml +++ b/.gitlab/functional_test/regression_detector.yml @@ -128,7 +128,7 @@ single-machine-performance-regression_detector: # invoke task has additional logic that does not seem to apply well to SMP's # JUnit XML. Agent CI seems to use `datadog-agent` as the service name when # uploading JUnit XML, so the upload command below respects that convention. - - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$API_KEY_ORG2")" || exit $?; export DATADOG_API_KEY + - DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2" token)" || exit $?; export DATADOG_API_KEY - datadog-ci junit upload --service datadog-agent outputs/junit.xml # Finally, exit 1 if the job signals a regression else 0. - RUST_LOG="${RUST_LOG}" ./smp --team-id ${SMP_AGENT_TEAM_ID} --api-base ${SMP_API} --aws-named-profile ${AWS_NAMED_PROFILE} diff --git a/.gitlab/kernel_matrix_testing/common.yml b/.gitlab/kernel_matrix_testing/common.yml index b2acf5a35d422..81e4cb5dc359a 100644 --- a/.gitlab/kernel_matrix_testing/common.yml +++ b/.gitlab/kernel_matrix_testing/common.yml @@ -60,7 +60,7 @@ - echo "COLLECT_COMPLEXITY=${COLLECT_COMPLEXITY}" .collect_outcomes_kmt: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - export MICRO_VM_IP=$(jq --exit-status --arg TAG $TAG --arg ARCH $ARCH --arg TEST_SET $TEST_SET -r '.[$ARCH].microvms | map(select(."vmset-tags"| index($TEST_SET))) | map(select(.tag==$TAG)) | .[].ip' $CI_PROJECT_DIR/stack.output) # Collect setup-ddvm systemd service logs - mkdir -p $CI_PROJECT_DIR/logs @@ -114,7 +114,7 @@ scp $DD_AGENT_TESTING_DIR/kmt-dockers-$ARCH.tar.gz metal_instance:/opt/kernel-version-testing fi after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key @@ -144,7 +144,7 @@ VMCONFIG_FILE: "${CI_PROJECT_DIR}/vmconfig-${CI_PIPELINE_ID}-${ARCH}.json" EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json before_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.retrieve_linux_go_deps] - !reference [.kmt_new_profile] - !reference [.write_ssh_key_file] @@ -159,7 +159,7 @@ - jq "." $CI_PROJECT_DIR/stack.output - pulumi logout after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - export AWS_PROFILE=agent-qa-ci - !reference [.shared_filters_and_queries] - mkdir -p $CI_PROJECT_DIR/libvirt/log/$ARCH $CI_PROJECT_DIR/libvirt/xml $CI_PROJECT_DIR/libvirt/qemu $CI_PROJECT_DIR/libvirt/dnsmasq @@ -217,7 +217,7 @@ aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}" fi after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] # Manual cleanup jobs, these will be used to cleanup the instances after the tests @@ -247,7 +247,7 @@ RETRY: 2 EXTERNAL_LINKS_PATH: external_links_$CI_JOB_ID.json before_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.kmt_new_profile] - !reference [.write_ssh_key_file] - echo "CI_JOB_URL=${CI_JOB_URL}" >> $DD_AGENT_TESTING_DIR/job_env.txt diff --git a/.gitlab/kernel_matrix_testing/security_agent.yml b/.gitlab/kernel_matrix_testing/security_agent.yml index c75f78c5449d8..1abe30df7c72b 100644 --- a/.gitlab/kernel_matrix_testing/security_agent.yml +++ b/.gitlab/kernel_matrix_testing/security_agent.yml @@ -72,7 +72,7 @@ kmt_setup_env_secagent_x64: # upload connector to metal instance - scp $CI_PROJECT_DIR/connector-${ARCH} metal_instance:/home/ubuntu/connector after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key diff --git a/.gitlab/kernel_matrix_testing/system_probe.yml b/.gitlab/kernel_matrix_testing/system_probe.yml index da9c47e167702..c3b6410bca4dc 100644 --- a/.gitlab/kernel_matrix_testing/system_probe.yml +++ b/.gitlab/kernel_matrix_testing/system_probe.yml @@ -35,7 +35,7 @@ upload_dependencies_sysprobe_arm64: - mkdir $KMT_DOCKERS - inv -e system-probe.save-test-dockers --use-crane --output-dir $KMT_DOCKERS --arch $ARCH after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] artifacts: expire_in: 1 day @@ -82,7 +82,7 @@ pull_test_dockers_arm64: - !reference [.setup_ssh_config] - scp $CI_PROJECT_DIR/kmt-deps/ci/$ARCH/$ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/ after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: DEPENDENCIES: $CI_PROJECT_DIR/kmt-deps/ci/$ARCH/btfs @@ -161,7 +161,7 @@ kmt_setup_env_sysprobe_x64: # upload connector to metal instance - scp $CI_PROJECT_DIR/connector-${ARCH} metal_instance:/home/ubuntu/connector after_script: - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - !reference [.tag_kmt_ci_job] variables: AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key diff --git a/.gitlab/notify/notify.yml b/.gitlab/notify/notify.yml index d148fb9729438..fec4b7daf371c 100644 --- a/.gitlab/notify/notify.yml +++ b/.gitlab/notify/notify.yml @@ -26,7 +26,7 @@ notify: timeout: 15 minutes # Added to prevent a stuck job blocking the resource_group defined above script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_READ_API_TOKEN) || exit $?; export GITLAB_TOKEN - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt - | # Do not send notifications if this is a child pipeline of another repo @@ -54,7 +54,7 @@ send_pipeline_stats: dependencies: [] script: - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_READ_API_TOKEN) || exit $?; export GITLAB_TOKEN - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - invoke -e notify.send-stats notify_github: @@ -116,7 +116,7 @@ notify_gitlab_ci_changes: .failure_summary_setup: - SLACK_API_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $SLACK_AGENT_CI_TOKEN) || exit $?; export SLACK_API_TOKEN - GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $GITLAB_READ_API_TOKEN) || exit $?; export GITLAB_TOKEN - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - python3 -m pip install -r requirements.txt -r tasks/libs/requirements-notifications.txt # Upload failure summary data to S3 at the end of each main pipeline @@ -172,8 +172,8 @@ close_failing_tests_stale_issues: echo "This script is run weekly on Fridays" exit fi - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY - - DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $APP_KEY_ORG2) || exit $?; export DD_APP_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY + - DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_APP_KEY_ORG2 token) || exit $?; export DD_APP_KEY - ATLASSIAN_PASSWORD=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE token) || exit $?; export ATLASSIAN_PASSWORD - ATLASSIAN_USERNAME=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $ATLASSIAN_WRITE user) || exit $?; export ATLASSIAN_USERNAME - python3 -m pip install -r requirements.txt -r tasks/requirements_release_tasks.txt # For Atlassian / Jira dependencies diff --git a/.gitlab/pkg_metrics/pkg_metrics.yml b/.gitlab/pkg_metrics/pkg_metrics.yml index 7001321d669c4..c2927f8e4ac27 100644 --- a/.gitlab/pkg_metrics/pkg_metrics.yml +++ b/.gitlab/pkg_metrics/pkg_metrics.yml @@ -45,7 +45,7 @@ send_pkg_size: optional: true script: # Get API key to send metrics - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY # Allow failures: some packages are not always built, and therefore stats cannot be sent for them - set +e diff --git a/.gitlab/setup/setup.yml b/.gitlab/setup/setup.yml index 28779481845c8..8321782a850ea 100644 --- a/.gitlab/setup/setup.yml +++ b/.gitlab/setup/setup.yml @@ -21,12 +21,12 @@ github_rate_limit_info: - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 key_b64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 app_id) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_1 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 1 # Send stats for app 2 - GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 key_b64) || exit $?; export GITHUB_KEY_B64 - GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 app_id) || exit $?; export GITHUB_APP_ID - GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $MACOS_GITHUB_APP_2 installation_id) || exit $?; export GITHUB_INSTALLATION_ID - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv github.send-rate-limit-info-datadog --pipeline-id $CI_PIPELINE_ID --app-instance 2 allow_failure: true diff --git a/.gitlab/source_test/golang_deps_diff.yml b/.gitlab/source_test/golang_deps_diff.yml index 6d156dd6f4963..b12d9b3ee1e2d 100644 --- a/.gitlab/source_test/golang_deps_diff.yml +++ b/.gitlab/source_test/golang_deps_diff.yml @@ -15,7 +15,7 @@ golang_deps_diff: - !reference [.retrieve_linux_go_deps] script: # Get API key to send metrics - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv -e diff.go-deps --report-file=deps-report.md --report-metrics --git-ref "${CI_COMMIT_REF_NAME}" artifacts: paths: @@ -64,7 +64,7 @@ golang_deps_send_count_metrics: - !reference [.retrieve_linux_go_deps] script: # Get API key to send metrics - - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $API_KEY_ORG2) || exit $?; export DD_API_KEY + - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY - inv -e go-deps.send-count-metrics --git-sha "${CI_COMMIT_SHA}" --git-ref "${CI_COMMIT_REF_NAME}" golang_deps_test: diff --git a/tasks/libs/common/omnibus.py b/tasks/libs/common/omnibus.py index b2e227638c81e..6809892feac7e 100644 --- a/tasks/libs/common/omnibus.py +++ b/tasks/libs/common/omnibus.py @@ -121,7 +121,7 @@ def env_filter(item): "HOSTNAME", "HOST_IP", "INFOPATH", - "INSTALL_SCRIPT_API_KEY", + "INSTALL_SCRIPT_API_KEY_ORG2", "INTEGRATION_WHEELS_CACHE_BUCKET", "IRBRC", "KITCHEN_INFRASTRUCTURE_FLAKES_RETRY", @@ -233,9 +233,7 @@ def should_retry_bundle_install(res): def send_build_metrics(ctx, overall_duration): # We only want to generate those metrics from the CI src_dir = os.environ.get('CI_PROJECT_DIR') - aws_cmd = "aws" if sys.platform == 'win32': - aws_cmd = "aws.cmd" if src_dir is None: src_dir = os.environ.get("REPO_ROOT", os.getcwd()) @@ -316,10 +314,16 @@ def send_build_metrics(ctx, overall_duration): 'type': 0, } ) - dd_api_key = ctx.run( - f'{aws_cmd} ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', - hide=True, - ).stdout.strip() + if sys.platform == 'win32': + dd_api_key = ctx.run( + f'aws.cmd ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', + hide=True, + ).stdout.strip() + else: + dd_api_key = ctx.run( + f'vault kv get -field=token kv/k8s/gitlab-runner/datadog-agent/{os.environ["AGENT_API_KEY_ORG2"]}', + hide=True, + ).stdout.strip() headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'DD-API-KEY': dd_api_key} r = requests.post("https://api.datadoghq.com/api/v2/series", json={'series': series}, headers=headers) if r.ok: @@ -331,13 +335,15 @@ def send_build_metrics(ctx, overall_duration): def send_cache_miss_event(ctx, pipeline_id, job_name, job_id): if sys.platform == 'win32': - aws_cmd = "aws.cmd" + dd_api_key = ctx.run( + f'aws.cmd ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', + hide=True, + ).stdout.strip() else: - aws_cmd = "aws" - dd_api_key = ctx.run( - f'{aws_cmd} ssm get-parameter --region us-east-1 --name {os.environ["API_KEY_ORG2"]} --with-decryption --query "Parameter.Value" --out text', - hide=True, - ).stdout.strip() + dd_api_key = ctx.run( + f'vault kv get -field=token kv/k8s/gitlab-runner/datadog-agent/{os.environ["AGENT_API_KEY_ORG2"]}', + hide=True, + ).stdout.strip() headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'DD-API-KEY': dd_api_key} payload = { 'title': 'omnibus cache miss', diff --git a/tasks/unit_tests/linter_tests.py b/tasks/unit_tests/linter_tests.py index 715b93e802db8..7178e9010f891 100644 --- a/tasks/unit_tests/linter_tests.py +++ b/tasks/unit_tests/linter_tests.py @@ -51,7 +51,7 @@ def test_with_wrapper_no_env(self): def test_with_wrapper_with_env(self): with open(self.test_file, "w") as f: f.write( - "DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $APP_KEY_ORG2) || exit $?; export DD_APP_KEY" + "DD_APP_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_APP_KEY_ORG2 token) || exit $?; export DD_APP_KEY" ) matched = linter.list_get_parameter_calls(self.test_file) self.assertListEqual([], matched) diff --git a/tasks/unit_tests/omnibus_tests.py b/tasks/unit_tests/omnibus_tests.py index 1aeaf35e411f7..dba84635d9503 100644 --- a/tasks/unit_tests/omnibus_tests.py +++ b/tasks/unit_tests/omnibus_tests.py @@ -42,6 +42,7 @@ def _run_calls_to_string(mock_calls): 'RELEASE_VERSION_7': 'nightly', 'S3_OMNIBUS_CACHE_BUCKET': 'omnibus-cache', 'API_KEY_ORG2': 'api-key', + 'AGENT_API_KEY_ORG2': 'agent-api-key', }, clear=True, ) @@ -60,6 +61,7 @@ def _set_up_default_command_mocks(self): (r'go mod .*', Result()), (r'grep .*', Result()), (r'aws ssm .*', Result()), + (r'vault kv get .*', Result()), ] for pattern, result in patterns: self.mock_ctx.set_result_for('run', re.compile(pattern), result) diff --git a/tools/ci/junit_upload.sh b/tools/ci/junit_upload.sh index 27690cced9747..97312066045e1 100755 --- a/tools/ci/junit_upload.sh +++ b/tools/ci/junit_upload.sh @@ -6,7 +6,7 @@ if [[ -n "$1" ]]; then junit_files="$1" fi -DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$API_KEY_ORG2")" +DATADOG_API_KEY="$("$CI_PROJECT_DIR"/tools/ci/fetch_secret.sh "$AGENT_API_KEY_ORG2" token)" export DATADOG_API_KEY error=0 for file in $junit_files; do