-
Notifications
You must be signed in to change notification settings - Fork 2
/
.gitlab-ci.yml
145 lines (139 loc) · 8.02 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
variables:
BUILD_STABLE_REGISTRY: registry.ddbuild.io
CI_IMAGE_REPO: "ci/${CI_PROJECT_NAME}"
CI_REGISTRY_IMAGE: "$BUILD_STABLE_REGISTRY/$CI_IMAGE_REPO/runner"
CI_REGISTRY_IMAGE_TEST: "$BUILD_STABLE_REGISTRY/$CI_IMAGE_REPO/runner-dev"
stages:
- build
- test
- release
- post-release
build-runner-image:
stage: build
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci-containers-project:v2.0.0
tags: ["arch:amd64"]
variables:
RELEASE_IMAGE: "false"
rules:
- when: on_success
before_script:
- set +x
- export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.github_app_id --with-decryption --query "Parameter.Value" --out text)
- export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.github_installation_id --with-decryption --query "Parameter.Value" --out text)
- export GITHUB_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.github_private_key --with-decryption --query "Parameter.Value" --out text)
- export JWT=$(.github/jwt.sh "$GITHUB_APP_ID" <(echo "$GITHUB_PRIVATE_KEY"))
- |
export GITHUB_TOKEN=`curl -s --fail --retry 10 -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $JWT" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/app/installations/$GITHUB_INSTALLATION_ID/access_tokens | jq -r '.token'`
script:
- docker buildx build --no-cache --pull --push --label target=build --tag ${CI_REGISTRY_IMAGE_TEST}:${CI_COMMIT_SHORT_SHA} --tag ${CI_REGISTRY_IMAGE_TEST}:${CI_COMMIT_SHA:0:12} --secret id=github_token,env=GITHUB_TOKEN .
retry: 2
build-pulumi-go-main:
stage: build
image: registry.ddbuild.io/ci/datadog-agent-buildimages/deb_x64:v48815877-9bfad02c
tags: ["arch:amd64"]
rules:
- when: on_success
script:
- go build -o dist/main -gcflags="all=-c=6" main.go
artifacts:
paths:
- dist/main
expire_in: "1 day"
variables:
KUBERNETES_MEMORY_REQUEST: 12Gi
KUBERNETES_MEMORY_LIMIT: 16Gi
KUBERNETES_CPU_REQUEST: 6
integration-testing:
stage: test
image: ${CI_REGISTRY_IMAGE_TEST}:${CI_COMMIT_SHORT_SHA}
tags: ["arch:amd64"]
needs:
- build-runner-image
- build-pulumi-go-main
rules:
- when: on_success
before_script:
# Setup GCP credentials https://cloud.google.com/docs/authentication/application-default-credentials#GAC
- aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.gcp_credentials --with-decryption --query "Parameter.Value" --out text > ~/gcp-credentials.json || exit $?
- export GOOGLE_APPLICATION_CREDENTIALS=~/gcp-credentials.json
# Setup Azure credentials. https://www.pulumi.com/registry/packages/azure-native/installation-configuration/#set-configuration-using-pulumi-config
# The app is called `agent-e2e-tests`
- export ARM_CLIENT_ID=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.azure_client_id --with-decryption --query "Parameter.Value" --out text)
- export ARM_CLIENT_SECRET=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.azure_client_secret --with-decryption --query "Parameter.Value" --out text)
- export ARM_TENANT_ID=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.azure_tenant_id --with-decryption --query "Parameter.Value" --out text)
- export ARM_SUBSCRIPTION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.azure_subscription_id --with-decryption --query "Parameter.Value" --out text)
# Setup AWS Credentials
- mkdir -p ~/.aws
- aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
- export AWS_PROFILE=agent-qa-ci
- aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.ssh_public_key_integration_test --with-decryption --query "Parameter.Value" --out text > $E2E_PUBLIC_KEY_PATH
- aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.ssh_key_integration_test --with-decryption --query "Parameter.Value" --out text > $E2E_PRIVATE_KEY_PATH
- export PULUMI_CONFIG_PASSPHRASE=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.pulumi_config_phrase_integration_test --with-decryption --query "Parameter.Value" --out text)
- eval $(ssh-agent -s)
- chmod 400 $E2E_PRIVATE_KEY_PATH
- ssh-add $E2E_PRIVATE_KEY_PATH
- pip install -r requirements.txt
script:
- |
if [ ! -f ./dist/main ]; then
echo "no main binary found, please run 'build-pulumi-go-main' job"
exit 1
fi
# execute test from dist directory to use the generated binary
- go test ./integration-tests -v -timeout 0s -workingDir=dist
variables:
E2E_PUBLIC_KEY_PATH: /tmp/agent-integration-test-ssh-key.pub
E2E_PRIVATE_KEY_PATH: /tmp/agent-integration-test-ssh-key
E2E_KEY_PAIR_NAME: e2e-integration-test-ssh-key
release-runner-image:
stage: release
image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10.13
tags: ["arch:amd64"]
script:
- crane copy ${CI_REGISTRY_IMAGE_TEST}:${CI_COMMIT_SHORT_SHA} ${CI_REGISTRY_IMAGE}:${CI_COMMIT_SHA:0:12}
rules:
- if: $CI_COMMIT_BRANCH == "main"
when: on_success
bump-version-on-datadog-agent:
stage: post-release
image: ${CI_REGISTRY_IMAGE_TEST}:${CI_COMMIT_SHORT_SHA}
tags: ["arch:amd64"]
rules:
- if: $CI_COMMIT_BRANCH == "main"
needs: ["release-runner-image"]
when: on_success
- if: $CI_COMMIT_BRANCH =~ /^mq-working-branch-/
when: never
- when: manual
needs: ["build-runner-image"]
allow_failure: true
variables:
EXTRA_UPDATE_ARGS: "--is-dev-image"
variables:
EXTRA_UPDATE_ARGS: ""
before_script:
- set +x
- export GITHUB_APP_USER_ID=153269286 # Can be found on https://api.github.com/users/agent-platform-auto-pr[bot]
- export GITHUB_APP_ID=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.platform-github-app-id --with-decryption --query "Parameter.Value" --out text)
- export GITHUB_INSTALLATION_ID=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.platform-github-app-installation-id --with-decryption --query "Parameter.Value" --out text)
- export GITHUB_PRIVATE_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.test-infra-definitions.platform-github-app-key --with-decryption --query "Parameter.Value" --out text)
- export JWT=$(.github/jwt.sh "$GITHUB_APP_ID" <(echo "$GITHUB_PRIVATE_KEY"))
- |
export GITHUB_TOKEN=`curl -s --fail --retry 10 -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer $JWT" -H "X-GitHub-Api-Version: 2022-11-28" https://api.github.com/app/installations/$GITHUB_INSTALLATION_ID/access_tokens | jq -r '.token'`
script:
- git config --global user.email "$GITHUB_APP_USER_ID+agent-platform-auto-pr[bot]@users.noreply.github.com"
- git config --global user.name "agent-platform-auto-pr[bot]"
# Set up the Git credential helper with your GitHub token
- git config --global credential.helper '!f() { echo "username=x-access-token"; echo "password='${GITHUB_TOKEN}'"; }; f'
- git clone https://github.com/DataDog/datadog-agent.git datadog-agent
- pushd datadog-agent
- git checkout -b auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA
- export PREVIOUS_SHA=$(cat .gitlab/common/test_infra_version.yml | grep 'TEST_INFRA_DEFINITIONS_BUILDIMAGES:' | awk -F " " '{print $NF}')
- inv -e buildimages.update-test-infra-definitions --commit-sha $CI_COMMIT_SHA $EXTRA_UPDATE_ARGS
- inv -e tidy
- git add -u
- git commit -m "[test-infra-definitions][automated] Bump test-infra-definitions to $CI_COMMIT_SHORT_SHA"
- git push -f origin auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA
- popd
- pip install -r requirements.txt
- inv ci.create-bump-pr-and-close-stale-ones-on-datadog-agent --branch auto-bump/bump-test-infra-$CI_COMMIT_SHORT_SHA --new-commit-sha $CI_COMMIT_SHA --old-commit-sha $PREVIOUS_SHA