From 073b29f2cc14e4fc89b8509e3bb7ea5097cc694b Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Fri, 15 Nov 2024 14:55:30 +0100 Subject: [PATCH 1/4] ci(non-gitlab): Misc updates --- .circleci/config.yml | 5 --- .github/workflows/windows-linters.yml | 64 --------------------------- 2 files changed, 69 deletions(-) delete mode 100644 .github/workflows/windows-linters.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 469b1f99d9747..bc7a82f92d969 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -310,8 +310,3 @@ workflows: - unit_tests - integration_tests - docker_tests - - build_iot_agent: - requires: - - unit_tests - - integration_tests - - docker_tests diff --git a/.github/workflows/windows-linters.yml b/.github/workflows/windows-linters.yml deleted file mode 100644 index e3f507d58dc70..0000000000000 --- a/.github/workflows/windows-linters.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Windows linters" - -on: - push: - branches: - - main - - "[0-9]+.[0-9]+.x" - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - windows-lint: - runs-on: windows-2019 # https://github.com/actions/virtual-environments/blob/main/images/win/Windows2019-Readme.md - steps: - - name: Checkout datadog-agent repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 #needed for 'git describe' to work - - - name: Install python - uses: actions/setup-python@v5 - with: - python-version: "3.9.5" - cache: "pip" - - run: | - python -m pip install -r requirements.txt - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - - name: Install go - uses: actions/setup-go@v5 - with: - go-version-file: ".go-version" - - - name: Install Dotnet - uses: actions/setup-dotnet@v4 - with: - dotnet-version: '7.0.x' - - - name: Set up runner - run: | - $ErrorActionPreference = 'Stop'; - # install dependencies - inv -e install-tools - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - inv -e deps - If ($lastExitCode -ne "0") { throw "Previous command returned $lastExitCode" } - - # Run this first as it's quite quick compared to Go - - name: Run Dotnet linter - shell: bash # using bash so we don't have to check $lastExitCode all the time - run: | - dotnet format --verify-no-changes .\\tools\\windows\\DatadogAgentInstaller - - - name: Run Go linters - shell: bash # using bash so we don't have to check $lastExitCode all the time - run: | - export PATH="/c/msys64/mingw64/bin:/c/msys64/usr/bin/:$PATH" # prepend msys, otherwise make from mingw gets used - echo $PATH - powershell.exe -Command ./tasks/winbuildscripts/pre-go-build.ps1 - inv -e rtloader.format --raise-if-changed - inv -e lint-go --timeout 30 From d3a0cff4d36bd22074eef839fc96dad5e1092ba7 Mon Sep 17 00:00:00 2001 From: Nicolas Schweitzer Date: Fri, 15 Nov 2024 15:53:56 +0100 Subject: [PATCH 2/4] Fix license linting --- LICENSE-3rdparty.csv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 94c3e7a04c4ca..db9f46da2b50f 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -2049,7 +2049,7 @@ core,golang.org/x/text/unicode/bidi,BSD-3-Clause,Copyright (c) 2009 The Go Autho core,golang.org/x/text/unicode/norm,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved core,golang.org/x/text/width,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved core,golang.org/x/time/rate,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved -core,golang/go,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved. +core,golang/go,BSD-Source-Code,Copyright (c) 2009 The Go Authors. All rights reserved. core,gomodules.xyz/jsonpatch/v2,Apache-2.0,Copyright (c) 2015 The Authors core,gonum.org/v1/gonum/blas,BSD-3-Clause,Alexander Egurnov | Andrei Blinnikov | Andrew Brampton | Bailey Lissington | Bill Gray | Bill Noon | Brendan Tracey | Brent Pedersen | Bulat Khasanov | Chad Kunde | Chan Kwan Yin | Chih-Wei Chang | Chong-Yeol Nah | Chris Tessum | Christophe Meessen | Christopher Waldon | Clayton Northey | Copyright ©2013 The Gonum Authors. All rights reserved | Dan Kortschak | Dan Lorenc | Daniel Fireman | Dario Heinisch | David Kleiven | David Samborski | Davor Kapsa | DeepMind Technologies | Delaney Gillilan | Dezmond Goff | Dong-hee Na | Dustin Spicuzza | Egon Elbre | Ekaterina Efimova | Ethan Burns | Evert Lammerts | Evgeny Savinov | Fabian Wickborn | Facundo Gaich | Fazlul Shahriar | Francesc Campoy | Google Inc | Gustaf Johansson | Hossein Zolfi | Iakov Davydov | Igor Mikushkin | Iskander Sharipov | Jalem Raj Rohit | James Bell | James Bowman | James Holmes <32bitkid@gmail.com> | Janne Snabb | Jeremy Atkinson | Jes Cok | Jinesi Yelizati | Jon Richards | Jonas Kahler | Jonas Schulze | Jonathan J Lawlor | Jonathan Reiter | Jonathan Schroeder | Joost van Amersfoort | Joseph Watson | Josh Wilson | Julien Roland | Kai Trukenmüller | Kent English | Kevin C. Zimmerman | Kirill Motkov | Konstantin Shaposhnikov | Leonid Kneller | Lyron Winderbaum | Marco Leogrande | Mark Canning | Mark Skilbeck | Martin Diz | Matthew Connelly | Matthieu Di Mercurio | Max Halford | Maxim Sergeev | Microsoft Corporation | MinJae Kwon | Nathan Edwards | Nick Potts | Nils Wogatzky | Olivier Wulveryck | Or Rikon | Patricio Whittingslow | Patrick DeVivo | Pontus Melke | Renee French | Rishi Desai | Robin Eklind | Roger Welin | Roman Werpachowski | Rondall Jones | Sam Zaydel | Samuel Kelemen | Saran Ahluwalia | Scott Holden | Scott Kiesel | Sebastien Binet | Shawn Smith | Sintela Ltd | Spencer Lyon | Steve McCoy | Taesu Pyo | Takeshi Yoneda | Tamir Hyman | The University of Adelaide | The University of Minnesota | The University of Washington | Thomas Berg | Tobin Harding | Valentin Deleplace | Vincent Thiery | Vladimír Chalupecký | Will Tekulve | Yasuhiro Matsumoto | Yevgeniy Vahlis | Yucheng Zhu | Yunomi | Zoe Juozapaitis | antichris | source{d} core,gonum.org/v1/gonum/blas/blas64,BSD-3-Clause,Alexander Egurnov | Andrei Blinnikov | Andrew Brampton | Bailey Lissington | Bill Gray | Bill Noon | Brendan Tracey | Brent Pedersen | Bulat Khasanov | Chad Kunde | Chan Kwan Yin | Chih-Wei Chang | Chong-Yeol Nah | Chris Tessum | Christophe Meessen | Christopher Waldon | Clayton Northey | Copyright ©2013 The Gonum Authors. All rights reserved | Dan Kortschak | Dan Lorenc | Daniel Fireman | Dario Heinisch | David Kleiven | David Samborski | Davor Kapsa | DeepMind Technologies | Delaney Gillilan | Dezmond Goff | Dong-hee Na | Dustin Spicuzza | Egon Elbre | Ekaterina Efimova | Ethan Burns | Evert Lammerts | Evgeny Savinov | Fabian Wickborn | Facundo Gaich | Fazlul Shahriar | Francesc Campoy | Google Inc | Gustaf Johansson | Hossein Zolfi | Iakov Davydov | Igor Mikushkin | Iskander Sharipov | Jalem Raj Rohit | James Bell | James Bowman | James Holmes <32bitkid@gmail.com> | Janne Snabb | Jeremy Atkinson | Jes Cok | Jinesi Yelizati | Jon Richards | Jonas Kahler | Jonas Schulze | Jonathan J Lawlor | Jonathan Reiter | Jonathan Schroeder | Joost van Amersfoort | Joseph Watson | Josh Wilson | Julien Roland | Kai Trukenmüller | Kent English | Kevin C. Zimmerman | Kirill Motkov | Konstantin Shaposhnikov | Leonid Kneller | Lyron Winderbaum | Marco Leogrande | Mark Canning | Mark Skilbeck | Martin Diz | Matthew Connelly | Matthieu Di Mercurio | Max Halford | Maxim Sergeev | Microsoft Corporation | MinJae Kwon | Nathan Edwards | Nick Potts | Nils Wogatzky | Olivier Wulveryck | Or Rikon | Patricio Whittingslow | Patrick DeVivo | Pontus Melke | Renee French | Rishi Desai | Robin Eklind | Roger Welin | Roman Werpachowski | Rondall Jones | Sam Zaydel | Samuel Kelemen | Saran Ahluwalia | Scott Holden | Scott Kiesel | Sebastien Binet | Shawn Smith | Sintela Ltd | Spencer Lyon | Steve McCoy | Taesu Pyo | Takeshi Yoneda | Tamir Hyman | The University of Adelaide | The University of Minnesota | The University of Washington | Thomas Berg | Tobin Harding | Valentin Deleplace | Vincent Thiery | Vladimír Chalupecký | Will Tekulve | Yasuhiro Matsumoto | Yevgeniy Vahlis | Yucheng Zhu | Yunomi | Zoe Juozapaitis | antichris | source{d} From 54222135d29e4102fd8dc31149604055f9e689de Mon Sep 17 00:00:00 2001 From: Paul Cacheux Date: Thu, 9 May 2024 23:00:59 +0200 Subject: [PATCH 3/4] migrate to docker compose v2 in integration tests (#25465) Co-authored-by: sblumenthal Co-authored-by: spencergilbert --- tasks/docker_tasks.py | 5 ++++- .../corechecks/docker/basemetrics_test.go | 2 +- .../integration/corechecks/docker/events_test.go | 6 +++--- .../corechecks/docker/exitcode_test.go | 6 +++--- test/integration/utils/compose.go | 16 +++++++++++----- 5 files changed, 22 insertions(+), 13 deletions(-) diff --git a/tasks/docker_tasks.py b/tasks/docker_tasks.py index 5017695169040..3a71bbf908cf4 100644 --- a/tasks/docker_tasks.py +++ b/tasks/docker_tasks.py @@ -54,8 +54,11 @@ def dockerize_test(ctx, binary, skip_cleanup=False): with open(f"{temp_folder}/Dockerfile", 'w') as stream: stream.write( - """FROM docker/compose:debian-1.29.2 + """FROM ubuntu:20.04 +COPY --from=docker/compose-bin:v2.26.1 /docker-compose /usr/bin/compose +COPY --from=docker:26.1-cli /usr/local/bin/docker /usr/bin/docker ENV DOCKER_DD_AGENT=yes +RUN apt-get update && apt-get install -y ca-certificates WORKDIR / CMD /test.bin COPY test.bin /test.bin diff --git a/test/integration/corechecks/docker/basemetrics_test.go b/test/integration/corechecks/docker/basemetrics_test.go index c3838bbec068d..60de49c4928c0 100644 --- a/test/integration/corechecks/docker/basemetrics_test.go +++ b/test/integration/corechecks/docker/basemetrics_test.go @@ -19,7 +19,7 @@ func init() { func TestContainerMetricsTagging(t *testing.T) { expectedTags := []string{ - "container_name:basemetrics_redis_1", // Container name + "container_name:basemetrics-redis-1", // Container name "docker_image:datadog/docker-library:redis_3_2_11-alpine", "image_name:datadog/docker-library", "short_image:docker-library", diff --git a/test/integration/corechecks/docker/events_test.go b/test/integration/corechecks/docker/events_test.go index afec2fa69a4b7..72957e5e74ba5 100644 --- a/test/integration/corechecks/docker/events_test.go +++ b/test/integration/corechecks/docker/events_test.go @@ -42,8 +42,8 @@ func TestEvents(t *testing.T) { "image_name:datadog/docker-library", "short_image:docker-library", "image_tag:busybox_1_28_0", - "container_name:events_recordingevent0_1", - "container_name:events_recordingevent1_1", + "container_name:events-recordingevent0-1", + "container_name:events-recordingevent1-1", }...), AggregationKey: "docker:datadog/docker-library:busybox_1_28_0", SourceTypeName: "docker", @@ -60,7 +60,7 @@ func TestEvents(t *testing.T) { "image_name:datadog/docker-library", "short_image:docker-library", "image_tag:redis_3_2_11-alpine", - "container_name:events_recordingevent2_1", + "container_name:events-recordingevent2-1", }...), AggregationKey: "docker:datadog/docker-library:redis_3_2_11-alpine", SourceTypeName: "docker", diff --git a/test/integration/corechecks/docker/exitcode_test.go b/test/integration/corechecks/docker/exitcode_test.go index 6d4d9df1a512f..aa04d540bc2d1 100644 --- a/test/integration/corechecks/docker/exitcode_test.go +++ b/test/integration/corechecks/docker/exitcode_test.go @@ -26,7 +26,7 @@ func TestContainerExit(t *testing.T) { "highcardenvtag:exithighenv", "lowcardenvtag:exitlowenv", } - sender.AssertServiceCheck(t, "docker.exit", servicecheck.ServiceCheckOK, "", append(expectedTags, "container_name:exitcode_exit0_1"), "Container exitcode_exit0_1 exited with 0") - sender.AssertServiceCheck(t, "docker.exit", servicecheck.ServiceCheckCritical, "", append(expectedTags, "container_name:exitcode_exit1_1"), "Container exitcode_exit1_1 exited with 1") - sender.AssertServiceCheck(t, "docker.exit", servicecheck.ServiceCheckCritical, "", append(expectedTags, "container_name:exitcode_exit54_1"), "Container exitcode_exit54_1 exited with 54") + sender.AssertServiceCheck(t, "docker.exit", servicecheck.ServiceCheckOK, "", append(expectedTags, "container_name:exitcode-exit0-1"), "Container exitcode-exit0-1 exited with 0") + sender.AssertServiceCheck(t, "docker.exit", servicecheck.ServiceCheckCritical, "", append(expectedTags, "container_name:exitcode-exit1-1"), "Container exitcode-exit1-1 exited with 1") + sender.AssertServiceCheck(t, "docker.exit", servicecheck.ServiceCheckCritical, "", append(expectedTags, "container_name:exitcode-exit54-1"), "Container exitcode-exit54-1 exited with 54") } diff --git a/test/integration/utils/compose.go b/test/integration/utils/compose.go index c6ad85c3debef..c3b631aa31f3c 100644 --- a/test/integration/utils/compose.go +++ b/test/integration/utils/compose.go @@ -58,7 +58,7 @@ func (c *ComposeConf) Start() ([]byte, error) { "--project-name", c.ProjectName, "--file", c.FilePath, } - pullCmd := exec.Command("docker-compose", append(args, "pull", "--parallel")...) + pullCmd := exec.Command("compose", append(args, "pull", "--parallel")...) pullCmd.Env = customEnv output, err := pullCmd.CombinedOutput() if err != nil { @@ -70,7 +70,7 @@ func (c *ComposeConf) Start() ([]byte, error) { */ log.Infof("retrying pull...") // We need to rebuild a new command because the file-descriptors of stdout/err are already set - retryPull := exec.Command("docker-compose", append(args, "pull", "--parallel")...) + retryPull := exec.Command("compose", append(args, "pull", "--parallel")...) retryPull.Env = customEnv output, err = retryPull.CombinedOutput() if err != nil { @@ -82,7 +82,7 @@ func (c *ComposeConf) Start() ([]byte, error) { if c.RemoveRebuildImages { args = append(args, "--build") } - runCmd := exec.Command("docker-compose", args...) + runCmd := exec.Command("compose", args...) runCmd.Env = customEnv return runCmd.CombinedOutput() @@ -98,17 +98,23 @@ func (c *ComposeConf) Stop() ([]byte, error) { if c.RemoveRebuildImages { args = append(args, "--rmi", "all") } - runCmd := exec.Command("docker-compose", args...) + runCmd := exec.Command("compose", args...) return runCmd.CombinedOutput() } // ListContainers lists the running container IDs func (c *ComposeConf) ListContainers() ([]string, error) { + customEnv := os.Environ() + for k, v := range c.Variables { + customEnv = append(customEnv, fmt.Sprintf("%s=%s", k, v)) + } + runCmd := exec.Command( - "docker-compose", + "compose", "--project-name", c.ProjectName, "--file", c.FilePath, "ps", "-q") + runCmd.Env = customEnv out, err := runCmd.Output() if err != nil { From 7992e50b35ddf9ad62034783ec44362b8ca6bc6e Mon Sep 17 00:00:00 2001 From: Alexandre Menasria <47357713+amenasria@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:38:46 +0100 Subject: [PATCH 4/4] [CI] Fix the integrations tests Dockerhub rate limit (#30699) --- tasks/docker_tasks.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tasks/docker_tasks.py b/tasks/docker_tasks.py index 3a71bbf908cf4..b3f6ba44b0b09 100644 --- a/tasks/docker_tasks.py +++ b/tasks/docker_tasks.py @@ -54,11 +54,20 @@ def dockerize_test(ctx, binary, skip_cleanup=False): with open(f"{temp_folder}/Dockerfile", 'w') as stream: stream.write( - """FROM ubuntu:20.04 -COPY --from=docker/compose-bin:v2.26.1 /docker-compose /usr/bin/compose -COPY --from=docker:26.1-cli /usr/local/bin/docker /usr/bin/docker + """FROM public.ecr.aws/docker/library/ubuntu:20.04 +# Install Docker +COPY --from=public.ecr.aws/docker/library/docker:26.1-cli /usr/local/bin/docker /usr/bin/docker + +# Install Docker Compose +ARG COMPOSE_VERSION=2.26.1 +ARG COMPOSE_SHA256=2f61856d1b8c9de29ffdaedaa1c6d0a5fc5c79da45068f1f4310feed8d3a3f61 +RUN apt-get update && apt-get install -y ca-certificates curl +RUN curl -SL "https://github.com/docker/compose/releases/download/v${COMPOSE_VERSION}/docker-compose-linux-x86_64" -o /usr/bin/compose +RUN echo "${COMPOSE_SHA256} /usr/bin/compose" | sha256sum --check +RUN chmod +x /usr/bin/compose + +# Final settings ENV DOCKER_DD_AGENT=yes -RUN apt-get update && apt-get install -y ca-certificates WORKDIR / CMD /test.bin COPY test.bin /test.bin