[CONTP-388] switch to composite store and use composite entity id in …

…tagger component (#30019)
DataDog · Oct 23, 2024 · b69a9ed · b69a9ed
1 parent 58a515b
commit b69a9ed
Show file tree

Hide file tree

Showing 59 changed files with 323 additions and 2,243 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -445,7 +445,6 @@
 /pkg/util/kubernetes/                   @DataDog/container-integrations @DataDog/container-platform @DataDog/container-app
 /pkg/util/podman/                       @DataDog/container-integrations
 /pkg/util/prometheus                    @DataDog/container-integrations
-/pkg/util/tagger                        @DataDog/container-platform
 /pkg/util/trivy/                        @DataDog/container-integrations @DataDog/agent-security
 /pkg/util/uuid/                         @DataDog/agent-shared-components
 /pkg/util/cgroups/                      @DataDog/container-integrations

diff --git a/cmd/serverless/dependencies_linux_amd64.txt b/cmd/serverless/dependencies_linux_amd64.txt
@@ -297,7 +297,6 @@ github.com/DataDog/datadog-agent/pkg/util/statstracker
 github.com/DataDog/datadog-agent/pkg/util/sync
 github.com/DataDog/datadog-agent/pkg/util/system
 github.com/DataDog/datadog-agent/pkg/util/system/socket
-github.com/DataDog/datadog-agent/pkg/util/tagger
 github.com/DataDog/datadog-agent/pkg/util/tmplvar
 github.com/DataDog/datadog-agent/pkg/version
 github.com/DataDog/datadog-api-client-go/v2

diff --git a/cmd/serverless/dependencies_linux_arm64.txt b/cmd/serverless/dependencies_linux_arm64.txt
@@ -297,7 +297,6 @@ github.com/DataDog/datadog-agent/pkg/util/statstracker
 github.com/DataDog/datadog-agent/pkg/util/sync
 github.com/DataDog/datadog-agent/pkg/util/system
 github.com/DataDog/datadog-agent/pkg/util/system/socket
-github.com/DataDog/datadog-agent/pkg/util/tagger
 github.com/DataDog/datadog-agent/pkg/util/tmplvar
 github.com/DataDog/datadog-agent/pkg/version
 github.com/DataDog/datadog-api-client-go/v2

diff --git a/comp/api/api/apiimpl/grpc.go b/comp/api/api/apiimpl/grpc.go
@@ -120,7 +120,7 @@ func (s *serverSecure) DogstatsdSetTaggerState(_ context.Context, req *pb.Tagger
 		}
 
 		state = append(state, taggerTypes.Entity{
-			ID:                          entityID,
+			ID:                          *entityID,
 			HighCardinalityTags:         entity.HighCardinalityTags,
 			OrchestratorCardinalityTags: entity.OrchestratorCardinalityTags,
 			LowCardinalityTags:          entity.LowCardinalityTags,

diff --git a/comp/core/autodiscovery/listeners/container.go b/comp/core/autodiscovery/listeners/container.go
@@ -114,7 +114,7 @@ func (l *ContainerListener) createContainerService(entity workloadmeta.Entity) {
 
 	svc := &service{
 		entity:   container,
-		tagsHash: tagger.GetEntityHash(types.NewEntityID(types.ContainerID, container.ID).String(), tagger.ChecksCardinality()),
+		tagsHash: tagger.GetEntityHash(types.NewEntityID(types.ContainerID, container.ID), tagger.ChecksCardinality()),
 		adIdentifiers: computeContainerServiceIDs(
 			containers.BuildEntityName(string(container.Runtime), container.ID),
 			containerImg.RawName,

diff --git a/comp/core/autodiscovery/listeners/kubelet.go b/comp/core/autodiscovery/listeners/kubelet.go
@@ -93,7 +93,7 @@ func (l *KubeletListener) createPodService(
 	})
 
 	entity := kubelet.PodUIDToEntityName(pod.ID)
-	taggerEntityID := common.BuildTaggerEntityID(pod.GetID()).String()
+	taggerEntityID := common.BuildTaggerEntityID(pod.GetID())
 	svc := &service{
 		entity:        pod,
 		tagsHash:      tagger.GetEntityHash(taggerEntityID, tagger.ChecksCardinality()),
@@ -158,7 +158,7 @@ func (l *KubeletListener) createContainerService(
 	entity := containers.BuildEntityName(string(container.Runtime), container.ID)
 	svc := &service{
 		entity:   container,
-		tagsHash: tagger.GetEntityHash(types.NewEntityID(types.ContainerID, container.ID).String(), tagger.ChecksCardinality()),
+		tagsHash: tagger.GetEntityHash(types.NewEntityID(types.ContainerID, container.ID), tagger.ChecksCardinality()),
 		ready:    pod.Ready,
 		ports:    ports,
 		extraConfig: map[string]string{

diff --git a/comp/core/tagger/component.go b/comp/core/tagger/component.go
@@ -39,13 +39,13 @@ type Component interface {
 	ReplayTagger() ReplayTagger
 	GetTaggerTelemetryStore() *telemetry.Store
 	Tag(entityID string, cardinality types.TagCardinality) ([]string, error)
-	AccumulateTagsFor(entityID string, cardinality types.TagCardinality, tb tagset.TagsAccumulator) error
-	Standard(entityID string) ([]string, error)
+	AccumulateTagsFor(entityID types.EntityID, cardinality types.TagCardinality, tb tagset.TagsAccumulator) error
+	Standard(entityID types.EntityID) ([]string, error)
 	List() types.TaggerListResponse
-	GetEntity(entityID string) (*types.Entity, error)
+	GetEntity(entityID types.EntityID) (*types.Entity, error)
 	// subscriptionID is used for logging and debugging purposes
 	Subscribe(subscriptionID string, filter *types.Filter) (types.Subscription, error)
-	GetEntityHash(entityID string, cardinality types.TagCardinality) string
+	GetEntityHash(entityID types.EntityID, cardinality types.TagCardinality) string
 	AgentTags(cardinality types.TagCardinality) ([]string, error)
 	GlobalTags(cardinality types.TagCardinality) ([]string, error)
 	SetNewCaptureTagger(newCaptureTagger Component)

diff --git a/comp/core/tagger/component_mock.go b/comp/core/tagger/component_mock.go
@@ -8,12 +8,14 @@
 
 package tagger
 
+import "github.com/DataDog/datadog-agent/comp/core/tagger/types"
+
 // Mock implements mock-specific methods for the tagger component.
 type Mock interface {
 	Component
 
 	// SetTags allows to set tags in the mock fake tagger
-	SetTags(entityID string, source string, low, orch, high, std []string)
+	SetTags(entityID types.EntityID, source string, low, orch, high, std []string)
 
 	// SetGlobalTags allows to set tags in store for the global entity
 	SetGlobalTags(low, orch, high, std []string)

diff --git a/comp/core/tagger/global.go b/comp/core/tagger/global.go
@@ -37,7 +37,7 @@ func UnlockGlobalTaggerClient() {
 }
 
 // GetEntity returns the hash for the provided entity id.
-func GetEntity(entityID string) (*types.Entity, error) {
+func GetEntity(entityID types.EntityID) (*types.Entity, error) {
 	if globalTagger == nil {
 		return nil, fmt.Errorf("a global tagger must be set before calling GetEntity")
 	}
@@ -53,7 +53,7 @@ func Tag(entity string, cardinality types.TagCardinality) ([]string, error) {
 }
 
 // GetEntityHash is an interface function that queries taggerclient singleton
-func GetEntityHash(entityID string, cardinality types.TagCardinality) string {
+func GetEntityHash(entityID types.EntityID, cardinality types.TagCardinality) string {
 	if globalTagger != nil {
 		return globalTagger.GetEntityHash(entityID, cardinality)
 	}

diff --git a/comp/core/tagger/noopimpl/tagger.go b/comp/core/tagger/noopimpl/tagger.go
@@ -58,27 +58,27 @@ func (n *noopTagger) Tag(string, types.TagCardinality) ([]string, error) {
 	return nil, nil
 }
 
-func (n *noopTagger) AccumulateTagsFor(string, types.TagCardinality, tagset.TagsAccumulator) error {
+func (n *noopTagger) AccumulateTagsFor(types.EntityID, types.TagCardinality, tagset.TagsAccumulator) error {
 	return nil
 }
 
-func (n *noopTagger) Standard(string) ([]string, error) {
+func (n *noopTagger) Standard(types.EntityID) ([]string, error) {
 	return nil, nil
 }
 
 func (n *noopTagger) List() types.TaggerListResponse {
 	return types.TaggerListResponse{}
 }
 
-func (n *noopTagger) GetEntity(string) (*types.Entity, error) {
+func (n *noopTagger) GetEntity(types.EntityID) (*types.Entity, error) {
 	return nil, nil
 }
 
 func (n *noopTagger) Subscribe(string, *types.Filter) (types.Subscription, error) {
 	return nil, fmt.Errorf("not implemented")
 }
 
-func (n *noopTagger) GetEntityHash(string, types.TagCardinality) string {
+func (n *noopTagger) GetEntityHash(types.EntityID, types.TagCardinality) string {
 	return ""
 }
 

diff --git a/comp/core/tagger/proto/proto.go b/comp/core/tagger/proto/proto.go
@@ -59,12 +59,13 @@ func Tagger2PbEntityEvent(event types.EntityEvent) (*pb.StreamTagsEvent, error)
 }
 
 // Pb2TaggerEntityID helper to convert a protobuf Entity ID to its expected format.
-func Pb2TaggerEntityID(entityID *pb.EntityId) (types.EntityID, error) {
+func Pb2TaggerEntityID(entityID *pb.EntityId) (*types.EntityID, error) {
 	if entityID == nil {
 		return nil, errors.New("Invalid entityID argument")
 	}
 
-	return types.NewEntityID(types.EntityIDPrefix(entityID.Prefix), entityID.Uid), nil
+	id := types.NewEntityID(types.EntityIDPrefix(entityID.Prefix), entityID.Uid)
+	return &id, nil
 }
 
 // Pb2TaggerCardinality helper to convert protobuf cardinality to native tag cardinality.

diff --git a/comp/core/tagger/taggerimpl/empty/empty.go b/comp/core/tagger/taggerimpl/empty/empty.go
@@ -18,7 +18,7 @@ type Tagger struct{}
 
 // GetEntityHash returns the hash for the tags associated with the given entity
 // Returns an empty string if the tags lookup fails
-func (t *Tagger) GetEntityHash(string, types.TagCardinality) string {
+func (t *Tagger) GetEntityHash(types.EntityID, types.TagCardinality) string {
 	return ""
 }
 

diff --git a/comp/core/tagger/taggerimpl/generic_store/default_store.go b/comp/core/tagger/taggerimpl/generic_store/default_store.go
diff --git a/comp/core/tagger/taggerimpl/generic_store/doc.go b/comp/core/tagger/taggerimpl/generic_store/doc.go
@@ -6,10 +6,4 @@
 // Package genericstore defines a generic object store that satisfies a redundant use-case in the tagger component implementation.
 // The implementation of the tagger component requires storing objects indexed by keys.
 // Keys are in the form of `{prefix}://{id}`.
-//
-// The package provides a generic interface ObjectStore which can store objects of a given type and index by tagger EntityID (i.e. a prefix + an id).
-// It also provides 2 implementations of this interface:
-// - defaultObjectStore: implements the object store as a plain from entity id to entity object. It is intended to be used when EntityID is stored as a string.
-// - compositeObjectStore: implements the object store as a 2-layered map. The first map is indexed by prefix, and the second map is indexed by id. It is intended to be used when EntityID is stored
-// as a struct separating prefix and id into 2 fields. This implementation is optimised for quick lookups, listing and filtering by prefix.
 package genericstore
diff --git a/comp/core/tagger/taggerimpl/generic_store/store.go b/comp/core/tagger/taggerimpl/generic_store/store.go
@@ -6,15 +6,10 @@
 package genericstore
 
 import (
-	"github.com/DataDog/datadog-agent/comp/core/config"
 	"github.com/DataDog/datadog-agent/comp/core/tagger/types"
 )
 
 // NewObjectStore constructs and returns a an ObjectStore
-func NewObjectStore[T any](cfg config.Component) types.ObjectStore[T] {
-	// TODO: use composite object store always or use component framework for config component
-	if cfg.GetBool("tagger.tagstore_use_composite_entity_id") {
-		return newCompositeObjectStore[T]()
-	}
-	return newDefaultObjectStore[T]()
+func NewObjectStore[T any]() types.ObjectStore[T] {
+	return newCompositeObjectStore[T]()
 }