[enhancement] add ocb custom collector support for ddflareextension (#…

…30345)

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

.gitlab/JOBOWNERS

@@ -132,6 +132,7 @@ trigger_auto_staging_release           @DataDog/agent-delivery
 integration_tests_windows*    @DataDog/windows-agent
 integration_tests_otel        @DataDog/opentelemetry
 docker_image_build_otel       @DataDog/opentelemetry
+ddflare_extension_ocb_build   @DataDog/opentelemetry
 agent_integration_tests       @DataDog/container-integrations
 docker_integration_tests      @DataDog/container-integrations
 

.gitlab/integration_test/otel.yml

@@ -54,3 +54,39 @@ docker_image_build_otel:
     - if: $CI_COMMIT_REF_NAME =~ /.*-skip-cancel$/
       when: never
     - when: always
+
+
+ddflare_extension_ocb_build:
+  stage: integration_test
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/datadog-agent-buildimages/deb_x64$DATADOG_AGENT_BUILDIMAGES_SUFFIX:$DATADOG_AGENT_BUILDIMAGES
+  tags: ["arch:amd64"]
+  needs: ["go_deps"]
+  before_script:
+    - !reference [.retrieve_linux_go_deps]
+    - mkdir -p /tmp/otel-ci
+    - cp test/otel/testdata/* /tmp/otel-ci/
+    - wget -O /tmp/otel-ci/ocb https://github.com/open-telemetry/opentelemetry-collector-releases/releases/download/cmd%2Fbuilder%2Fv0.111.0/ocb_0.111.0_linux_amd64
+    - chmod +x /tmp/otel-ci/ocb
+  script:
+    - echo 'Building collector with OCB and test ddflare extension'
+    - /tmp/otel-ci/ocb --config=/tmp/otel-ci/builder-config.yaml > ocb-output.log 2>&1
+    - grep -q 'Compiled' ocb-output.log || (echo "OCB failed to compile" && exit 1)
+    - "grep -q '{\"binary\": \"/tmp/otel-ci/otelcol-custom/otelcol-custom\"}' ocb-output.log || (echo \"OCB failed to compile\" && exit 1)"
+    - /tmp/otel-ci/otelcol-custom/otelcol-custom --config /tmp/otel-ci/collector-config.yaml > otelcol-custom.log 2>&1 &
+    - OTELCOL_PID=$!  # Capture the process ID
+    - sleep 10  # Wait for the process to start
+    - grep -q 'Everything is ready. Begin running and processing data.' otelcol-custom.log || (echo "custom collector failed to start" && kill $OTELCOL_PID && exit 1)
+    - curl -k https://localhost:7777 > flare-info.log 2>&1 # get the dd flare info
+    - "grep -q '\"provided_configuration\": \"\"' flare-info.log || (echo \"provided config should not be supported with ocb\" && kill $OTELCOL_PID && exit 1)"
+    - grep -q 'extensions:\\n  - ddflare\\n' flare-info.log || (echo "ddflare extension should be enabled" && kill $OTELCOL_PID && exit 1)
+    - kill $OTELCOL_PID  # Kill the process
+  rules:
+    - if: $CI_PIPELINE_SOURCE =~ /^schedule.*$/
+      when: never
+    - if: $CI_COMMIT_TAG
+      when: never
+    - if: $CI_COMMIT_MESSAGE =~ /.*\[skip cancel\].*/
+      when: never
+    - if: $CI_COMMIT_REF_NAME =~ /.*-skip-cancel$/
+      when: never
+    - when: always

comp/core/tagger/types/go.mod

@@ -31,7 +31,7 @@ replace (
 )
 
 require (
-	github.com/DataDog/datadog-agent/comp/core/tagger/utils v0.56.2
+	github.com/DataDog/datadog-agent/comp/core/tagger/utils v0.59.0
 	github.com/stretchr/testify v1.9.0
 )
 

comp/otelcol/collector/impl/collector.go

@@ -135,7 +135,7 @@ func addFactories(reqs Requires, factories otelcol.Factories) {
 	}
 	factories.Processors[infraattributesprocessor.Type] = infraattributesprocessor.NewFactory(reqs.Tagger, generateID)
 	factories.Connectors[component.MustNewType("datadog")] = datadogconnector.NewFactory()
-	factories.Extensions[ddextension.Type] = ddextension.NewFactory(&factories, newConfigProviderSettings(reqs.URIs, reqs.Converter, false))
+	factories.Extensions[ddextension.Type] = ddextension.NewFactoryForAgent(&factories, newConfigProviderSettings(reqs.URIs, reqs.Converter, false))
 }
 
 var buildInfo = component.BuildInfo{
@@ -194,7 +194,7 @@ func NewComponentNoAgent(reqs RequiresNoAgent) (Provides, error) {
 		return Provides{}, err
 	}
 	factories.Connectors[component.MustNewType("datadog")] = datadogconnector.NewFactory()
-	factories.Extensions[ddextension.Type] = ddextension.NewFactory(&factories, newConfigProviderSettings(reqs.URIs, reqs.Converter, false))
+	factories.Extensions[ddextension.Type] = ddextension.NewFactoryForAgent(&factories, newConfigProviderSettings(reqs.URIs, reqs.Converter, false))
 
 	converterEnabled := reqs.Config.GetBool("otelcollector.converter.enabled")
 	set := otelcol.CollectorSettings{

comp/otelcol/converter/impl/go.mod

@@ -13,7 +13,6 @@ replace (
 	github.com/DataDog/datadog-agent/comp/def => ../../../../comp/def
 	github.com/DataDog/datadog-agent/comp/otelcol/converter/def => ../def
 	github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/def => ../../ddflareextension/def/
-	github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/impl => ../../ddflareextension/impl/
 	github.com/DataDog/datadog-agent/pkg/api => ../../../../pkg/api
 	github.com/DataDog/datadog-agent/pkg/collector/check/defaults => ../../../../pkg/collector/check/defaults
 	github.com/DataDog/datadog-agent/pkg/config/env => ../../../../pkg/config/env

comp/otelcol/ddflareextension/impl/configstore_test.go

@@ -12,11 +12,12 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector"
+	"go.opentelemetry.io/collector/component/componenttest"
+
 	converterimpl "github.com/DataDog/datadog-agent/comp/otelcol/converter/impl"
 	"github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/exporter/datadogexporter"
 	"github.com/DataDog/datadog-agent/comp/otelcol/otlp/components/processor/infraattributesprocessor"
-	"github.com/open-telemetry/opentelemetry-collector-contrib/connector/datadogconnector"
-	"go.opentelemetry.io/collector/component/componenttest"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -39,7 +40,7 @@ func addFactories(factories otelcol.Factories) {
 	factories.Exporters[datadogexporter.Type] = datadogexporter.NewFactory(nil, nil, nil, nil, nil)
 	factories.Processors[infraattributesprocessor.Type] = infraattributesprocessor.NewFactory(nil, nil)
 	factories.Connectors[component.MustNewType("datadog")] = datadogconnector.NewFactory()
-	factories.Extensions[Type] = NewFactory(nil, otelcol.ConfigProviderSettings{})
+	factories.Extensions[Type] = NewFactoryForAgent(nil, otelcol.ConfigProviderSettings{})
 }
 
 func TestGetConfDump(t *testing.T) {
@@ -56,7 +57,7 @@ func TestGetConfDump(t *testing.T) {
 		factories:              &factories,
 		configProviderSettings: newConfigProviderSettings(uriFromFile("simple-dd/config.yaml"), false),
 	}
-	extension, err := NewExtension(context.TODO(), &config, componenttest.NewNopTelemetrySettings(), component.BuildInfo{})
+	extension, err := NewExtension(context.TODO(), &config, componenttest.NewNopTelemetrySettings(), component.BuildInfo{}, true)
 	assert.NoError(t, err)
 
 	ext, ok := extension.(*ddExtension)
@@ -148,6 +149,7 @@ func TestGetConfDump(t *testing.T) {
 
 		assertEqual(t, expectedStringMap, actualStringMap)
 	})
+
 }
 
 func confmapFromResolverSettings(t *testing.T, resolverSettings confmap.ResolverSettings) *confmap.Conf {
@@ -189,11 +191,11 @@ func newResolverSettings(uris []string, enhanced bool) confmap.ResolverSettings
 			httpprovider.NewFactory(),
 			httpsprovider.NewFactory(),
 		},
-		ConverterFactories: newConverterFactorie(enhanced),
+		ConverterFactories: newConverterFactory(enhanced),
 	}
 }
 
-func newConverterFactorie(enhanced bool) []confmap.ConverterFactory {
+func newConverterFactory(enhanced bool) []confmap.ConverterFactory {
 	converterFactories := []confmap.ConverterFactory{}
 
 	converter, err := converterimpl.NewConverter(converterimpl.Requires{})

comp/otelcol/ddflareextension/impl/extension.go

@@ -17,6 +17,7 @@ import (
 	"go.opentelemetry.io/collector/component/componentstatus"
 	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/extension"
+	"go.opentelemetry.io/collector/extension/extensioncapabilities"
 	"go.opentelemetry.io/collector/otelcol"
 	"go.uber.org/zap"
 
@@ -41,7 +42,7 @@ type ddExtension struct {
 	configStore *configStore
 }
 
-var _ extension.Extension = (*ddExtension)(nil)
+var _ extensioncapabilities.ConfigWatcher = (*ddExtension)(nil)
 
 func extensionType(s string) string {
 	index := strings.Index(s, "/")
@@ -116,17 +117,7 @@ func (ext *ddExtension) NotifyConfig(_ context.Context, conf *confmap.Conf) erro
 }
 
 // NewExtension creates a new instance of the extension.
-func NewExtension(_ context.Context, cfg *Config, telemetry component.TelemetrySettings, info component.BuildInfo) (extensionDef.Component, error) {
-	ocpProvided, err := otelcol.NewConfigProvider(cfg.configProviderSettings)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create configprovider: %w", err)
-	}
-
-	providedConf, err := ocpProvided.Get(context.Background(), *cfg.factories)
-	if err != nil {
-		return nil, err
-	}
-
+func NewExtension(_ context.Context, cfg *Config, telemetry component.TelemetrySettings, info component.BuildInfo, providedConfigSupported bool) (extensionDef.Component, error) {
 	ext := &ddExtension{
 		cfg:         cfg,
 		telemetry:   telemetry,
@@ -136,16 +127,28 @@ func NewExtension(_ context.Context, cfg *Config, telemetry component.TelemetryS
 			Sources: map[string]extensionDef.OTelFlareSource{},
 		},
 	}
+	// only initiate the configprovider and set provided config if factories are provided
+	if providedConfigSupported {
+		ocpProvided, err := otelcol.NewConfigProvider(cfg.configProviderSettings)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create configprovider: %w", err)
+		}
+		providedConf, err := ocpProvided.Get(context.Background(), *cfg.factories)
+		if err != nil {
+			return nil, err
+		}
+		conf := confmap.New()
+		err = conf.Marshal(providedConf)
+		if err != nil {
+			return nil, err
+		}
 
-	conf := confmap.New()
-	err = conf.Marshal(providedConf)
-	if err != nil {
-		return nil, err
+		ext.configStore.setProvidedConf(conf)
 	}
-
-	ext.configStore.setProvidedConf(conf)
-
-	ext.server, err = newServer(cfg.HTTPConfig.Endpoint, ext)
+	var err error
+	// auth = providedConfigSupported; if value true, component was likely built by Agent and has
+	// bearer auth token, if false, component was likely built by OCB and has no auth token
+	ext.server, err = newServer(cfg.HTTPConfig.Endpoint, ext, providedConfigSupported)
 	if err != nil {
 		return nil, err
 	}
@@ -177,20 +180,31 @@ func (ext *ddExtension) Shutdown(ctx context.Context) error {
 
 // ServeHTTP the request handler for the extension.
 func (ext *ddExtension) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
-	customer, err := ext.configStore.getProvidedConfAsString()
+	var (
+		customer  string
+		err       error
+		envconfig string
+	)
+	providedConfig, err := ext.configStore.getProvidedConf()
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		fmt.Fprintf(w, "Unable to get provided config\n")
 		return
 	}
+	if providedConfig != nil {
+		customer, err = ext.configStore.getProvidedConfAsString()
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprintf(w, "Unable to get provided config\n")
+			return
+		}
+	}
 	enhanced, err := ext.configStore.getEnhancedConfAsString()
 	if err != nil {
 		w.WriteHeader(http.StatusInternalServerError)
 		fmt.Fprintf(w, "Unable to get enhanced config\n")
 		return
 	}
-
-	envconfig := ""
 	envvars := getEnvironmentAsMap()
 	if envbytes, err := json.Marshal(envvars); err == nil {
 		envconfig = string(envbytes)

comp/otelcol/ddflareextension/impl/extension_test.go

@@ -40,6 +40,7 @@ import (
 	"go.opentelemetry.io/collector/receiver"
 	"go.opentelemetry.io/collector/receiver/nopreceiver"
 	"go.opentelemetry.io/collector/receiver/otlpreceiver"
+
 	"go.uber.org/zap"
 )
 
@@ -61,7 +62,7 @@ func getTestExtension(t *testing.T) (ddflareextension.Component, error) {
 	info := component.NewDefaultBuildInfo()
 	cfg := getExtensionTestConfig(t)
 
-	return NewExtension(c, cfg, telemetry, info)
+	return NewExtension(c, cfg, telemetry, info, true)
 }
 
 func getResponseToHandlerRequest(t *testing.T, tokenOverride string) *httptest.ResponseRecorder {

comp/otelcol/ddflareextension/impl/factory.go

@@ -10,11 +10,12 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/impl/internal/metadata"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/config/confighttp"
 	"go.opentelemetry.io/collector/extension"
 	"go.opentelemetry.io/collector/otelcol"
+
+	"github.com/DataDog/datadog-agent/comp/otelcol/ddflareextension/impl/internal/metadata"
 )
 
 const (
@@ -28,21 +29,42 @@ type ddExtensionFactory struct {
 	configProviderSettings otelcol.ConfigProviderSettings
 }
 
-// NewFactory creates a factory for HealthCheck extension.
-func NewFactory(factories *otelcol.Factories, configProviderSettings otelcol.ConfigProviderSettings) extension.Factory {
+// isOCB returns true if extension was built with OCB
+func (f *ddExtensionFactory) isOCB() bool {
+	return f.factories == nil
+}
+
+// NewFactory creates a factory for Datadog Flare Extension for use with OCB and OSS Collector
+func NewFactory() extension.Factory {
+	return &ddExtensionFactory{}
+}
+
+// NewFactoryForAgent creates a factory for Datadog Flare Extension for use with Agent
+func NewFactoryForAgent(factories *otelcol.Factories, configProviderSettings otelcol.ConfigProviderSettings) extension.Factory {
 	return &ddExtensionFactory{
 		factories:              factories,
 		configProviderSettings: configProviderSettings,
 	}
 }
 
+// CreateExtension is deprecated as of v0.112.0
 func (f *ddExtensionFactory) CreateExtension(ctx context.Context, set extension.Settings, cfg component.Config) (extension.Extension, error) {
 	config := &Config{
 		factories:              f.factories,
 		configProviderSettings: f.configProviderSettings,
 	}
 	config.HTTPConfig = cfg.(*Config).HTTPConfig
-	return NewExtension(ctx, config, set.TelemetrySettings, set.BuildInfo)
+	return NewExtension(ctx, config, set.TelemetrySettings, set.BuildInfo, !f.isOCB())
+}
+
+// Create creates a new instance of the Datadog Flare Extension, as of v0.112.0 or later
+func (f *ddExtensionFactory) Create(ctx context.Context, set extension.Settings, cfg component.Config) (extension.Extension, error) {
+	config := &Config{
+		factories:              f.factories,
+		configProviderSettings: f.configProviderSettings,
+	}
+	config.HTTPConfig = cfg.(*Config).HTTPConfig
+	return NewExtension(ctx, config, set.TelemetrySettings, set.BuildInfo, !f.isOCB())
 }
 
 func (f *ddExtensionFactory) CreateDefaultConfig() component.Config {
@@ -57,6 +79,12 @@ func (f *ddExtensionFactory) Type() component.Type {
 	return metadata.Type
 }
 
+// ExtensionStability is deprecated as of v0.112.0
 func (f *ddExtensionFactory) ExtensionStability() component.StabilityLevel {
 	return metadata.ExtensionStability
 }
+
+// Stability returns the stability level of the component as of v0.112.0 or later
+func (f *ddExtensionFactory) Stability() component.StabilityLevel {
+	return metadata.ExtensionStability
+}

comp/otelcol/ddflareextension/impl/factory_test.go

@@ -19,10 +19,10 @@ func getTestFactory(t *testing.T) extension.Factory {
 	factories, err := components()
 	assert.NoError(t, err)
 
-	return NewFactory(&factories, newConfigProviderSettings(uriFromFile("config.yaml"), false))
+	return NewFactoryForAgent(&factories, newConfigProviderSettings(uriFromFile("config.yaml"), false))
 }
 
-func TestNewFactory(t *testing.T) {
+func TestNewFactoryForAgent(t *testing.T) {
 	factory := getTestFactory(t)
 	assert.NotNil(t, factory)