Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into pducolin/e2e-migrate-…
Browse files Browse the repository at this point in the history
…snmp-testsuite
  • Loading branch information
pducolin committed Dec 18, 2023
2 parents 36f10a2 + 6bb4dd6 commit 404236e
Show file tree
Hide file tree
Showing 140 changed files with 2,747 additions and 396 deletions.
20 changes: 18 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,15 @@ jobs:
command: inv -e lint-teamassignment
name: run PR check for team assignment labels

skip_qa:
<<: *job_template
steps:
- restore_cache: *restore_source
- restore_cache: *restore_deps
- run:
command: inv -e lint-skip-qa
name: run PR check for skip-qa labels

milestone:
<<: *job_template
steps:
Expand Down Expand Up @@ -208,8 +217,8 @@ jobs:
- run:
name: setting env vars for click
command: |
echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV
echo 'export LANG="C.UTF-8"' >> $BASH_ENV
echo 'export LC_ALL="C.UTF-8"' >> $BASH_ENV
echo 'export LANG="C.UTF-8"' >> $BASH_ENV
- run:
name: lint python files
command: inv -e lint-python
Expand Down Expand Up @@ -307,6 +316,13 @@ workflows:
- main
requires:
- dependencies
- skip_qa:
filters:
branches:
ignore:
- main
requires:
- dependencies
- milestone:
filters:
branches:
Expand Down
2 changes: 2 additions & 0 deletions .ddqa/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ qa_statuses = [
]
ignored_labels = [
"qa/skip-qa",
"qa/done",
"qa/no-code-change",
]

[teams."Agent Metrics Logs"]
Expand Down
2 changes: 2 additions & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
/.github/workflows/windows-*.yml @DataDog/windows-agent
/.github/workflows/cws-btfhub-sync.yml @DataDog/agent-security
/.github/workflows/gohai.yml @DataDog/agent-shared-components
/.github/workflows/go-update-commenter.yml @DataDog/agent-shared-components

# Gitlab files
# Files containing job contents are owned by teams in charge of the jobs + agent-platform
Expand Down Expand Up @@ -507,6 +508,7 @@
/tools/ @DataDog/agent-platform
/tools/ebpf/ @DataDog/ebpf-platform
/tools/gdb/ @DataDog/agent-shared-components
/tools/go-update/ @DataDog/agent-shared-components
/tools/retry_file_dump/ @DataDog/agent-metrics-logs
/tools/windows/ @DataDog/windows-agent
/tools/windows/DatadogAgentInstaller/WixSetup/localization-en-us.wxl @DataDog/windows-agent @DataDog/documentation
Expand Down
2 changes: 1 addition & 1 deletion .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ Note: Adding GitHub labels is only possible for contributors with write access.
- [ ] Use the `major_change` label if your change either has a major impact on the code base, is impacting multiple teams or is changing important well-established internals of the Agent. This label will be use during QA to make sure each team pay extra attention to the changed behavior. For any customer facing change use a releasenote.
- [ ] A [release note](https://github.com/DataDog/datadog-agent/blob/main/docs/dev/contributing.md#reno) has been added or the `changelog/no-changelog` label has been applied.
- [ ] Changed code has automated tests for its functionality.
- [ ] Adequate QA/testing plan information is provided if the `qa/skip-qa` label is not applied.
- [ ] Adequate QA/testing plan information is provided. Except if the `qa/skip-qa` label, with required either `qa/done` or `qa/no-code-change` labels, are applied.
- [ ] At least one `team/..` label has been applied, indicating the team(s) that should QA this change.
- [ ] If applicable, docs team has been notified or [an issue has been opened on the documentation repo](https://github.com/DataDog/documentation/issues/new).
- [ ] If applicable, the `need-change/operator` and `need-change/helm` labels have been applied.
Expand Down
7 changes: 7 additions & 0 deletions .github/dependabot.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ updates:
- team/agent-platform
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
milestone: 22
schedule:
interval: monthly
Expand All @@ -139,6 +140,7 @@ updates:
- team/agent-e2e-test
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/testing
milestone: 22
ignore:
Expand All @@ -157,6 +159,7 @@ updates:
- team/agent-e2e-test
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/testing
milestone: 22
schedule:
Expand All @@ -170,6 +173,7 @@ updates:
- team/agent-platform
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/tooling
milestone: 22
schedule:
Expand All @@ -183,6 +187,7 @@ updates:
- team/agent-security
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/tooling
milestone: 22
schedule:
Expand All @@ -195,6 +200,7 @@ updates:
- team/agent-e2e-test
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/testing
milestone: 22
schedule:
Expand All @@ -208,6 +214,7 @@ updates:
- team/agent-platform
- changelog/no-changelog
- qa/skip-qa
- qa/no-code-change
- dev/tooling
milestone: 22
schedule:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cws-btfhub-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,5 +85,5 @@ jobs:
owner,
repo,
issue_number: result.data.number,
labels: ['changelog/no-changelog', 'qa/skip-qa', 'team/agent-security']
labels: ['changelog/no-changelog', 'qa/skip-qa', 'qa/no-code-change', 'team/agent-security']
});
65 changes: 65 additions & 0 deletions .github/workflows/go-update-commenter.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: "Go update commenter"

on:
pull_request:
# Only run on PR label events (in particular not on every commit)
types: [ labeled ]

jobs:
old-versions-match:
# Only run if the PR is labeled with 'go-update'
if: ${{ github.event.label.name == 'go-update' }}
runs-on: ubuntu-latest
steps:
# get the Go version of the target branch
- uses: actions/checkout@v3
with:
ref: ${{ github.base_ref }}
- name: Get former Go version
id: former_go_version
run: |
echo version="$(cat .go-version)" >> $GITHUB_OUTPUT
# get the Go version of the PR branch
- uses: actions/checkout@v3
- name: Get current Go version
id: new_go_version
run: |
echo version="$(cat .go-version)" >> $GITHUB_OUTPUT
# build the comment
- name: Build full comment
id: old_versions
run: |
set -euxo pipefail
# build the base of the Github URL to the current commit
GITHUB_HEAD_URL='${{ github.server_url }}/${{ github.repository }}/blob/${{ github.sha }}'
{
echo "matches<<EOF"
echo "Here are potential matches of the former version:"
echo ""
# this step builds a Markdown list of potential matches
# the script `detect-old-version.sh` displays each matching line as "file:line_number:line_content"
# the sed command transforms this format into a markdown list with Github permalink URL for each file/line
# note that the sed command only works properly with GNU sed (MacOS sed doesn't seem to understand \S properly)
bash -x ./tools/go-update/detect-old-version.sh "${{ steps.former_go_version.outputs.version }}" "${{ steps.new_go_version.outputs.version }}" | \
sed -E 's|^([^:]+):([^:]+):\s*(\S.*)$|- [\1:\2]('"$GITHUB_HEAD_URL"'/\1#L\2): `\3`|'
echo "EOF"
} >> $GITHUB_OUTPUT
# and display it
- uses: actions/github-script@v7
env:
# We need to store the output in an environment variable and not use it directly in the createComment,
# as it will likely not be a valid JS string (eg. if it contains a quote character)
CONTENT: ${{ steps.old_versions.outputs.matches }}
with:
script: |
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body: process.env.CONTENT
})
2 changes: 1 addition & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ variables:
# To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
# and check the job creating the image to make sure you have the right SHA prefix
TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
TEST_INFRA_DEFINITIONS_BUILDIMAGES: 281b2a324002
TEST_INFRA_DEFINITIONS_BUILDIMAGES: 67d2009bcd81
DATADOG_AGENT_BUILDERS: v22276738-b36b132

DATADOG_AGENT_EMBEDDED_PATH: /opt/datadog-agent/embedded
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/kernel_version_testing/system_probe.yml
Original file line number Diff line number Diff line change
Expand Up @@ -437,7 +437,7 @@ kernel_matrix_testing_run_tests_arm64:
- INSTANCE_ID="$(jq -r '.[0][0]' < instance.json)"
- echo ${INSTANCE_ID}
- |
if [[ "${INSTANCE_ID}" != "" ]]; then
if [[ "${INSTANCE_ID}" != "" ]] && [[ "${INSTANCE_ID}" != "null" ]]; then
aws ec2 terminate-instances --instance-ids "${INSTANCE_ID}"
fi
Expand Down
13 changes: 10 additions & 3 deletions cmd/agent/subcommands/run/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
"github.com/DataDog/datadog-agent/cmd/agent/subcommands/run/internal/clcrunnerapi"
"github.com/DataDog/datadog-agent/cmd/manager"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl"
netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server"

// checks implemented as components
Expand All @@ -52,6 +53,7 @@ import (
"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
"github.com/DataDog/datadog-agent/comp/core/workloadmeta/collectors"
"github.com/DataDog/datadog-agent/comp/dogstatsd"
"github.com/DataDog/datadog-agent/comp/dogstatsd/replay"
dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server"
dogstatsddebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug"
"github.com/DataDog/datadog-agent/comp/forwarder"
Expand All @@ -65,8 +67,10 @@ import (
"github.com/DataDog/datadog-agent/comp/logs"
logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent"
"github.com/DataDog/datadog-agent/comp/metadata"
"github.com/DataDog/datadog-agent/comp/metadata/host"
"github.com/DataDog/datadog-agent/comp/metadata/inventoryagent"
"github.com/DataDog/datadog-agent/comp/metadata/inventorychecks"
"github.com/DataDog/datadog-agent/comp/metadata/inventoryhost"
"github.com/DataDog/datadog-agent/comp/metadata/runner"
"github.com/DataDog/datadog-agent/comp/ndmtmp"
"github.com/DataDog/datadog-agent/comp/netflow"
Expand Down Expand Up @@ -197,6 +201,7 @@ func run(log log.Component,
telemetry telemetry.Component,
sysprobeconfig sysprobeconfig.Component,
server dogstatsdServer.Component,
_ replay.Component,
serverDebug dogstatsddebug.Component,
forwarder defaultforwarder.Component,
wmeta workloadmeta.Component,
Expand All @@ -208,7 +213,9 @@ func run(log log.Component,
cliParams *cliParams,
logsAgent optional.Option[logsAgent.Component],
otelcollector otelcollector.Component,
_ host.Component,
invAgent inventoryagent.Component,
_ inventoryhost.Component,
_ secrets.Component,
invChecks inventorychecks.Component,
_ netflowServer.Component,
Expand Down Expand Up @@ -346,12 +353,12 @@ func getSharedFxOption() fx.Option {
metadata.Bundle(),
// injecting the aggregator demultiplexer to FX until we migrate it to a proper component. This allows
// other already migrated components to request it.
fx.Provide(func(config config.Component) demultiplexer.Params {
fx.Provide(func(config config.Component) demultiplexerimpl.Params {
opts := aggregator.DefaultAgentDemultiplexerOptions()
opts.EnableNoAggregationPipeline = config.GetBool("dogstatsd_no_aggregation_pipeline")
return demultiplexer.Params{Options: opts}
return demultiplexerimpl.Params{Options: opts}
}),
demultiplexer.Module(),
demultiplexerimpl.Module(),
orchestratorForwarderImpl.Module(),
fx.Supply(orchestratorForwarderImpl.NewDefaultParams()),
// injecting the shared Serializer to FX until we migrate it to a prpoper component. This allows other
Expand Down
6 changes: 6 additions & 0 deletions cmd/agent/subcommands/run/command_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,12 +44,15 @@ import (
"github.com/DataDog/datadog-agent/comp/core/sysprobeconfig/sysprobeconfigimpl"
"github.com/DataDog/datadog-agent/comp/core/telemetry"
"github.com/DataDog/datadog-agent/comp/core/workloadmeta"
"github.com/DataDog/datadog-agent/comp/dogstatsd/replay"
dogstatsdServer "github.com/DataDog/datadog-agent/comp/dogstatsd/server"
dogstatsddebug "github.com/DataDog/datadog-agent/comp/dogstatsd/serverDebug"
"github.com/DataDog/datadog-agent/comp/forwarder/defaultforwarder"
logsAgent "github.com/DataDog/datadog-agent/comp/logs/agent"
"github.com/DataDog/datadog-agent/comp/metadata/host"
"github.com/DataDog/datadog-agent/comp/metadata/inventoryagent"
"github.com/DataDog/datadog-agent/comp/metadata/inventorychecks"
"github.com/DataDog/datadog-agent/comp/metadata/inventoryhost"
"github.com/DataDog/datadog-agent/comp/metadata/runner"
netflowServer "github.com/DataDog/datadog-agent/comp/netflow/server"
otelcollector "github.com/DataDog/datadog-agent/comp/otelcol/collector"
Expand Down Expand Up @@ -80,6 +83,7 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error
telemetry telemetry.Component,
sysprobeconfig sysprobeconfig.Component,
server dogstatsdServer.Component,
_ replay.Component,
serverDebug dogstatsddebug.Component,
wmeta workloadmeta.Component,
rcclient rcclient.Component,
Expand All @@ -89,7 +93,9 @@ func StartAgentWithDefaults(ctxChan <-chan context.Context) (<-chan error, error
sharedSerializer serializer.MetricSerializer,
otelcollector otelcollector.Component,
demultiplexer demultiplexer.Component,
_ host.Component,
invAgent inventoryagent.Component,
_ inventoryhost.Component,
_ secrets.Component,
invChecks inventorychecks.Component,
_ netflowServer.Component,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

global "github.com/DataDog/datadog-agent/cmd/agent/dogstatsd"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl"
"github.com/DataDog/datadog-agent/comp/core"
"github.com/DataDog/datadog-agent/comp/dogstatsd"
"github.com/DataDog/datadog-agent/comp/dogstatsd/server"
Expand Down Expand Up @@ -45,7 +46,7 @@ func TestDogstatsdMetricsStats(t *testing.T) {
}),
dogstatsd.Bundle(),
defaultforwarder.MockModule(),
demultiplexer.MockModule(),
demultiplexerimpl.MockModule(),
))

demux := deps.Demultiplexer
Expand Down
7 changes: 4 additions & 3 deletions cmd/cluster-agent-cloudfoundry/subcommands/run/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/DataDog/datadog-agent/cmd/cluster-agent/api"
dcav1 "github.com/DataDog/datadog-agent/cmd/cluster-agent/api/v1"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl"
"github.com/DataDog/datadog-agent/comp/core"
"github.com/DataDog/datadog-agent/comp/core/config"
"github.com/DataDog/datadog-agent/comp/core/log"
Expand Down Expand Up @@ -69,13 +70,13 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
core.Bundle(),
forwarder.Bundle(),
fx.Provide(defaultforwarder.NewParamsWithResolvers),
demultiplexer.Module(),
demultiplexerimpl.Module(),
orchestratorForwarderImpl.Module(),
fx.Supply(orchestratorForwarderImpl.NewDisabledParams()),
fx.Provide(func() demultiplexer.Params {
fx.Provide(func() demultiplexerimpl.Params {
opts := aggregator.DefaultAgentDemultiplexerOptions()
opts.UseEventPlatformForwarder = false
return demultiplexer.Params{Options: opts}
return demultiplexerimpl.Params{Options: opts}
}),
// setup workloadmeta
collectors.GetCatalog(),
Expand Down
7 changes: 4 additions & 3 deletions cmd/cluster-agent/subcommands/start/command.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/DataDog/datadog-agent/cmd/cluster-agent/command"
"github.com/DataDog/datadog-agent/cmd/cluster-agent/custommetrics"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer"
"github.com/DataDog/datadog-agent/comp/aggregator/demultiplexer/demultiplexerimpl"
"github.com/DataDog/datadog-agent/comp/core"
"github.com/DataDog/datadog-agent/comp/core/config"
"github.com/DataDog/datadog-agent/comp/core/log"
Expand Down Expand Up @@ -94,13 +95,13 @@ func Commands(globalParams *command.GlobalParams) []*cobra.Command {
params.Options.DisableAPIKeyChecking = true
return params
}),
demultiplexer.Module(),
demultiplexerimpl.Module(),
orchestratorForwarderImpl.Module(),
fx.Supply(orchestratorForwarderImpl.NewDefaultParams()),
fx.Provide(func() demultiplexer.Params {
fx.Provide(func() demultiplexerimpl.Params {
opts := aggregator.DefaultAgentDemultiplexerOptions()
opts.UseEventPlatformForwarder = false
return demultiplexer.Params{Options: opts}
return demultiplexerimpl.Params{Options: opts}
}),
// setup workloadmeta
collectors.GetCatalog(),
Expand Down
Loading

0 comments on commit 404236e

Please sign in to comment.