feat(ci): Add a new task to compare a branch to itself (#26101)

.gitlab-ci.yml

@@ -219,6 +219,9 @@ variables:
   GITLAB_FULL_API_TOKEN_SSM_NAME: ci.datadog-agent.gitlab_full_api_token  # ci-cd
   INSTALL_SCRIPT_API_KEY_SSM_NAME: ci.agent-linux-install-script.datadog_api_key_2  # agent-delivery
   JIRA_READ_API_TOKEN_SSM_NAME: ci.datadog-agent.jira_read_api_token  # agent-devx-infra
+  AGENT_GITHUB_APP_ID_SSM_NAME: ci.datadog-agent.platform-github-app-id  # agent-devx-infra
+  AGENT_GITHUB_INSTALLATION_ID_SSM_NAME: ci.datadog-agent.platform-github-app-installation-id  # agent-devx-infra
+  AGENT_GITHUB_KEY_SSM_NAME: ci.datadog-agent.platform-github-app-key  # agent-devx-infra
   MACOS_GITHUB_APP_ID_SSM_NAME: ci.datadog-agent.macos_github_app_id  # agent-devx-infra
   MACOS_GITHUB_INSTALLATION_ID_SSM_NAME: ci.datadog-agent.macos_github_installation_id  # agent-devx-infra
   MACOS_GITHUB_KEY_SSM_NAME: ci.datadog-agent.macos_github_key_b64  # agent-devx-infra

.gitlab/.pre/test_gitlab_configuration.yml

@@ -10,3 +10,16 @@ test_gitlab_configuration:
     - inv -e linter.gitlab-ci
     - inv -e linter.job-change-path
     - inv -e linter.gitlab-change-paths
+
+test_gitlab_compare_to:
+  stage: .pre
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  rules:
+    - !reference [.on_gitlab_changes]
+  script:
+    - source /root/.bashrc
+    - export GITLAB_TOKEN=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $GITLAB_FULL_API_TOKEN_SSM_NAME)
+    - !reference [.setup_agent_github_app]
+    - pip install -r tasks/requirements.txt
+    - inv pipeline.compare-to-itself

.gitlab/common/shared.yml

@@ -41,6 +41,12 @@
       echo "Using GitHub App instance 2"
     fi
 
+.setup_agent_github_app:
+  - export GITHUB_KEY_B64=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_KEY_SSM_NAME)
+  - export GITHUB_APP_ID=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_APP_ID_SSM_NAME)
+  - export GITHUB_INSTALLATION_ID=$($CI_PROJECT_DIR/tools/ci/aws_ssm_get_wrapper.sh $AGENT_GITHUB_INSTALLATION_ID_SSM_NAME)
+  - echo "Using agent GitHub App"
+
 # Install `dd-pkg` and lint packages produced by Omnibus, supports only deb and rpm packages
 .lint_linux_packages:
   - curl -sSL "https://dd-package-tools.s3.amazonaws.com/dd-pkg/${DD_PKG_VERSION}/dd-pkg_Linux_${DD_PKG_ARCH}.tar.gz" | tar -xz -C /usr/local/bin dd-pkg

tasks/libs/ciproviders/gitlab_api.py

@@ -18,6 +18,7 @@
 from invoke.exceptions import Exit
 
 from tasks.libs.common.color import Color, color_message
+from tasks.libs.common.git import get_common_ancestor, get_current_branch
 from tasks.libs.common.utils import retry_function
 
 BASE_URL = "https://gitlab.ddbuild.io"
@@ -868,3 +869,58 @@ def retrieve_all_paths(yaml):
     elif isinstance(yaml, list):
         for item in yaml:
             yield from retrieve_all_paths(item)
+
+
+def gitlab_configuration_is_modified(ctx):
+    branch = get_current_branch(ctx)
+    if branch == "main":
+        # We usually squash merge on main so comparing only to the last commit
+        diff = ctx.run("git diff HEAD^1..HEAD", hide=True).stdout.strip().splitlines()
+    else:
+        # On dev branch we compare all the new commits
+        ctx.run("git fetch origin main:main")
+        ancestor = get_common_ancestor(ctx, branch)
+        diff = ctx.run(f"git diff {ancestor}..HEAD", hide=True).stdout.strip().splitlines()
+    modified_files = re.compile(r"^diff --git a/(.*) b/(.*)")
+    changed_lines = re.compile(r"^@@ -\d+,\d+ \+(\d+),(\d+) @@")
+    leading_space = re.compile(r"^(\s*).*$")
+    in_config = False
+    for line in diff:
+        if line.startswith("diff --git"):
+            files = modified_files.match(line)
+            new_file = files.group(1)
+            # Third condition is only for testing purposes
+            if (
+                new_file.startswith(".gitlab") and new_file.endswith(".yml")
+            ) or "testdata/yaml_configurations" in new_file:
+                in_config = True
+                print(f"Found a gitlab configuration file: {new_file}")
+            else:
+                in_config = False
+        if in_config and line.startswith("@@"):
+            lines = changed_lines.match(line)
+            start = int(lines.group(1))
+            with open(new_file) as f:
+                content = f.readlines()
+                item = leading_space.match(content[start])
+                if item:
+                    for above_line in reversed(content[:start]):
+                        current = leading_space.match(above_line)
+                        if current[1] < item[1]:
+                            if any(keyword in above_line for keyword in ["needs:", "dependencies:"]):
+                                print(f"> Found a gitlab configuration change on line: {content[start]}")
+                                return True
+                            else:
+                                break
+        if (
+            in_config
+            and line.startswith("+")
+            and (
+                (len(line) > 1 and line[1].isalpha())
+                or any(keyword in line for keyword in ["needs:", "dependencies:", "!reference"])
+            )
+        ):
+            print(f"> Found a gitlab configuration change on line: {line}")
+            return True
+
+    return False

tasks/libs/common/git.py

@@ -54,6 +54,10 @@ def get_current_branch(ctx) -> str:
     return ctx.run("git rev-parse --abbrev-ref HEAD", hide=True).stdout.strip()
 
 
+def get_common_ancestor(ctx, branch) -> str:
+    return ctx.run(f"git merge-base {branch} main", hide=True).stdout.strip()
+
+
 def check_uncommitted_changes(ctx):
     """
     Checks if there are uncommitted changes in the local git repository.

tasks/pipeline.py

@@ -12,8 +12,13 @@
 from invoke.exceptions import Exit
 
 from tasks.libs.ciproviders.github_api import GithubAPI
-from tasks.libs.ciproviders.gitlab_api import get_gitlab_bot_token, get_gitlab_repo, refresh_pipeline
-from tasks.libs.common.color import color_message
+from tasks.libs.ciproviders.gitlab_api import (
+    get_gitlab_bot_token,
+    get_gitlab_repo,
+    gitlab_configuration_is_modified,
+    refresh_pipeline,
+)
+from tasks.libs.common.color import Color, color_message
 from tasks.libs.common.constants import DEFAULT_BRANCH, GITHUB_REPO_NAME
 from tasks.libs.common.git import check_clean_branch_state, get_commit_sha, get_current_branch
 from tasks.libs.common.utils import (
@@ -32,6 +37,8 @@
 )
 from tasks.libs.releasing.documentation import nightly_entry_for, release_entry_for
 
+BOT_NAME = "github-actions[bot]"
+
 
 class GitlabReference(yaml.YAMLObject):
     def __init__(self, refs):
@@ -991,3 +998,74 @@ def test_merge_queue(ctx):
     ctx.run(f"git push origin :{test_main}", hide=True)
     if not success:
         raise Exit(message="Merge queue test failed", code=1)
+
+
+@task
+def compare_to_itself(ctx):
+    """
+    Create a new branch with 'compare_to_itself' in gitlab-ci.yml and trigger a pipeline
+    """
+    if not gitlab_configuration_is_modified(ctx):
+        print("No modification in the gitlab configuration, ignoring this test.")
+        return
+    agent = get_gitlab_repo()
+    gh = GithubAPI()
+    current_branch = os.environ["CI_COMMIT_REF_NAME"]
+    if current_branch.startswith("compare/"):
+        print("Branch already in compare_to_itself mode, ignoring this test to prevent infinite loop")
+        return
+    new_branch = f"compare/{current_branch}/{int(datetime.now(timezone.utc).timestamp())}"
+    ctx.run(f"git checkout -b {new_branch}", hide=True)
+    ctx.run(
+        f"git remote set-url origin https://x-access-token:{gh._auth.token}@github.com/DataDog/datadog-agent.git",
+        hide=True,
+    )
+    ctx.run(f"git config --global user.name '{BOT_NAME}'")
+    ctx.run("git config --global user.email 'github-app[bot]@users.noreply.github.com'")
+    # The branch must exist in gitlab to be able to "compare_to"
+    ctx.run(f"git push origin {new_branch}", hide=True)
+    for file in ['.gitlab-ci.yml', '.gitlab/notify/notify.yml']:
+        with open(file) as f:
+            content = f.read()
+        with open(file, 'w') as f:
+            f.write(content.replace('compare_to: main', f'compare_to: {new_branch}'))
+    ctx.run("git commit -am 'Compare to itself'", hide=True)
+    ctx.run(f"git push origin {new_branch}", hide=True)
+    max_attempts = 6
+    compare_to_pipeline = None
+    for attempt in range(max_attempts):
+        print(f"[{datetime.now()}] Waiting 30s for the pipelines to be created")
+        time.sleep(30)
+        pipelines = agent.pipelines.list(ref=new_branch, get_all=True)
+        for pipeline in pipelines:
+            commit = agent.commits.get(pipeline.sha)
+            if commit.author_name == BOT_NAME:
+                compare_to_pipeline = pipeline
+                print(f"Test pipeline found: {pipeline.web_url}")
+        if compare_to_pipeline:
+            break
+        if attempt == max_attempts - 1:
+            # Clean up the branch and possible pipelines
+            for pipeline in pipelines:
+                pipeline.cancel()
+            ctx.run(f"git checkout {current_branch}", hide=True)
+            ctx.run(f"git branch -D {new_branch}", hide=True)
+            ctx.run(f"git push origin :{new_branch}", hide=True)
+            raise RuntimeError(f"No pipeline found for {new_branch}")
+    try:
+        if len(compare_to_pipeline.jobs.list(get_all=False)) == 0:
+            print(
+                f"[{color_message('ERROR', Color.RED)}] Failed to generate a pipeline for {new_branch}, please check {compare_to_pipeline.web_url}"
+            )
+            raise Exit(message="compare_to itself failed", code=1)
+        else:
+            print(f"Pipeline correctly created, {color_message('congrats', Color.GREEN)}")
+    finally:
+        # Clean up
+        print("Cleaning up the pipelines")
+        for pipeline in pipelines:
+            pipeline.cancel()
+        print("Cleaning up git")
+        ctx.run(f"git checkout {current_branch}", hide=True)
+        ctx.run(f"git branch -D {new_branch}", hide=True)
+        ctx.run(f"git push origin :{new_branch}", hide=True)

tasks/unit_tests/gitlab_api_tests.py

@@ -1,11 +1,13 @@
 import unittest
+from unittest.mock import MagicMock, patch
 
-from invoke.context import MockContext
+from invoke import MockContext, Result
 
 from tasks.libs.ciproviders.gitlab_api import (
     GitlabCIDiff,
     clean_gitlab_ci_configuration,
     filter_gitlab_ci_configuration,
+    gitlab_configuration_is_modified,
     read_includes,
     retrieve_all_paths,
 )
@@ -195,3 +197,90 @@ def test_all_configs(self):
             'hand_of_the_king',
         ]
         self.assertListEqual(paths, expected_paths)
+
+
+class TestGitlabConfigurationIsModified(unittest.TestCase):
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_needs_one_line(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..e9a74219ba 100644\n--- a/{file}\n+++ b/{file}\n@@ -4,7 +4,7 @@\n \n .linux_tests:\n   stage: source_test\n-  needs: ["go_deps", "go_tools_deps"]\n+  needs: ["go_deps", "go_tools_deps", "new"]\n   rules:\n     - !reference [.except_disable_unit_tests]\n     - !reference [.fast_on_dev_branch_only]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertTrue(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_reference_one_line(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..e9a74219ba 100644\n--- a/{file}\n+++ b/{file}\n@@ -4,7 +4,7 @@\n \n .linux_tests:\n   stage: source_test\n  needs: ["go_deps", "go_tools_deps"]\n   rules:\n+     - !reference [.adding_new_reference]\n     - !reference [.except_disable_unit_tests]\n     - !reference [.fast_on_dev_branch_only]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertTrue(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_needs_removed(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..e9a74219ba 100644\n--- a/{file}\n+++ b/{file}\n@@ -4,6 +4,6 @@\n \n .linux_tests:\n   stage: source_test\n-  needs: ["go_deps", "go_tools_deps"]\n   rules:\n     - !reference [.except_disable_unit_tests]\n     - !reference [.fast_on_dev_branch_only]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertFalse(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_artifacts_modified_and_needs_above(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..e9a74219ba 100644\n--- a/{file}\n+++ b/{file}\n@@ -12,6 +12,6 @@\n \n  artifacts:\n    expire_in: 2 years\n-    when: always\n+    when: never\n    paths:\n      - none-shall-pass.txt'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertFalse(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_needs_multiple_lines(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_several_lines.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..e9a74219ba 100644\n--- a/{file}\n+++ b/{file}\n@@ -6,7 +6,7 @@\n \n    - go_tools_deps\n    - go_go_dancer\n    - go_go_ackman\n+    - go_nagai\n   rules:\n     - !reference [.except_disable_unit_tests]\n     - !reference [.fast_on_dev_branch_only]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertTrue(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_not_a_needs_multiple_lines(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/no_needs_several_lines.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..e9a74219ba 100644\n--- a/{file}\n+++ b/{file}\n@@ -6,7 +6,7 @@\n \n    - go_tools_deps\n    - go_go_dancer\n    - go_go_ackman\n+    - go_nagai\n   rules:\n     - !reference [.except_disable_unit_tests]\n     - !reference [.fast_on_dev_branch_only]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertFalse(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_new_reference(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..5e43218090 100644\n--- a/{file}\n+++ b/{file}\n@@ -1,4 +1,11 @@\n ---\n+.rtloader_tests:\n+  stage: source_test\n+  noods: ["go_deps"]\n+  before_script:\n+    - source /root/.bashrc && conda activate $CONDA_ENV\n+  script: ["# Skipping go tests"]\n+\n nerd_tests\n   stage: source_test\n   needs: ["go_deps"]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertFalse(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_new_reference_with_needs(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..5e43218090 100644\n--- a/{file}\n+++ b/{file}\n@@ -1,4 +1,11 @@\n ---\n+.rtloader_tests:\n+  stage: source_test\n+  needs: ["go_deps"]\n+  before_script:\n+    - source /root/.bashrc && conda activate $CONDA_ENV\n+  script: ["# Skipping go tests"]\n+\n nerd_tests\n   stage: source_test\n   needs: ["go_deps"]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertTrue(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_new_reference_with_dependencies(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..5e43218090 100644\n--- a/{file}\n+++ b/{file}\n@@ -1,4 +1,11 @@\n ---\n+.rtloader_tests:\n+  stage: source_test\n+  dependencies: ["go_deps"]\n+  before_script:\n+    - source /root/.bashrc && conda activate $CONDA_ENV\n+  script: ["# Skipping go tests"]\n+\n nerd_tests\n   stage: source_test\n   needs: ["go_deps"]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertTrue(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_new_job(self):
+        file = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..5e43218090 100644\n--- a/{file}\n+++ b/{file}\n@@ -1,4 +1,11 @@\n ---\n+rtloader_tests:\n+  stage: source_test\n+  noods: ["go_deps"]\n+  before_script:\n+    - source /root/.bashrc && conda activate $CONDA_ENV\n+  script: ["# Skipping go tests"]\n+\n nerd_tests\n   stage: source_test\n   needs: ["go_deps"]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertTrue(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_ignored_file(self):
+        file = "tasks/unit_tests/testdata/d.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..5e43218090 100644\n--- a/{file}\n+++ b/{file}\n@@ -1,4 +1,11 @@\n ---\n+rtloader_tests:\n+  stage: source_test\n+  needs: ["go_deps"]\n+  before_script:\n+    - source /root/.bashrc && conda activate $CONDA_ENV\n+  script: ["# Skipping go tests"]\n+\n nerd_tests\n   stage: source_test\n   needs: ["go_deps"]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertFalse(gitlab_configuration_is_modified(c))
+
+    @patch("tasks.libs.ciproviders.gitlab_api.get_current_branch", new=MagicMock(return_value="main"))
+    def test_two_modified_files(self):
+        file = "tasks/unit_tests/testdata/d.yml"
+        yaml = "tasks/unit_tests/testdata/yaml_configurations/needs_one_line.yml"
+        diff = f'diff --git a/{file} b/{file}\nindex 561eb1a201..5e43218090 100644\n--- a/{file}\n+++ b/{file}\n@@ -1,4 +1,11 @@\n ---\n+rtloader_tests:\n+  stage: source_test\n+  needs: ["go_deps"]\n+  before_script:\n+    - source /root/.bashrc && conda activate $CONDA_ENV\n+  script: ["# Skipping go tests"]\n+\n nerd_tests\n   stage: source_test\n   needs: ["go_deps"]\ndiff --git a/{yaml} b/{yaml}\nindex 561eb1a201..5e43218090 100644\n--- a/{yaml}\n+++ b/{yaml}\n@@ -1,4 +1,11 @@\n ---\n+rtloader_tests:\n+  stage: source_test\n+  noods: ["go_deps"]\n+  before_script:\n+    - source /root/.bashrc && conda activate $CONDA_ENV\n+  script: ["# Skipping go tests"]\n+\n nerd_tests\n   stage: source_test\n   needs: ["go_deps"]'
+        c = MockContext(run={"git diff HEAD^1..HEAD": Result(diff)})
+        self.assertTrue(gitlab_configuration_is_modified(c))