From cf1fe25664057e52c793c87ec67f56aa745b634f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Farias?= Date: Tue, 19 Mar 2024 14:10:54 +0100 Subject: [PATCH] Enrich cloud field --- internal/inventory/asset.go | 37 +++++++++-- .../inventory/aws/fetcher_ec2_instance.go | 62 ++++++++++++++----- .../aws/fetcher_ec2_instance_test.go | 42 +++++++++++-- internal/inventory/aws/fetcher_s3_bucket.go | 21 +++++-- .../inventory/aws/fetcher_s3_bucket_test.go | 20 +++++- internal/inventory/inventory.go | 4 +- 6 files changed, 152 insertions(+), 34 deletions(-) diff --git a/internal/inventory/asset.go b/internal/inventory/asset.go index f6f3a19c27..ec467ebd86 100644 --- a/internal/inventory/asset.go +++ b/internal/inventory/asset.go @@ -54,10 +54,8 @@ const ( SubTypeS3 assetSubType = "s3" ) -type assetCloudProvider string - const ( - AwsCloudProvider assetCloudProvider = "aws" + AwsCloudProvider = "aws" ) // AssetEvent holds the whole asset @@ -100,8 +98,37 @@ type AssetNetwork struct { // AssetCloud contains information about the cloud provider type AssetCloud struct { - Provider assetCloudProvider `json:"provider"` - Region string `json:"region"` + AvailabilityZone *string `json:"availability_zone,omitempty"` + Provider string `json:"provider,omitempty"` + Region string `json:"region,omitempty"` + Account AssetCloudAccount `json:"account"` + Instance *AssetCloudInstance `json:"instance,omitempty"` + Machine *AssetCloudMachine `json:"machine,omitempty"` + Project *AssetCloudProject `json:"project,omitempty"` + Service *AssetCloudService `json:"service,omitempty"` +} + +type AssetCloudAccount struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type AssetCloudInstance struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type AssetCloudMachine struct { + MachineType string `json:"machineType,omitempty"` +} + +type AssetCloudProject struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type AssetCloudService struct { + Name string `json:"name,omitempty"` } // AssetHost contains information of the asset in case it is a host diff --git a/internal/inventory/aws/fetcher_ec2_instance.go b/internal/inventory/aws/fetcher_ec2_instance.go index 5f13480886..1b72c8e785 100644 --- a/internal/inventory/aws/fetcher_ec2_instance.go +++ b/internal/inventory/aws/fetcher_ec2_instance.go @@ -31,8 +31,10 @@ import ( ) type Ec2InstanceFetcher struct { - logger *logp.Logger - provider ec2InstancesProvider + logger *logp.Logger + provider ec2InstancesProvider + AccountId string + AccountName string } type ec2InstancesProvider interface { @@ -49,8 +51,10 @@ var ec2InstanceClassification = inventory.AssetClassification{ func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher { provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{}) return &Ec2InstanceFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: identity.Account, + AccountName: identity.AccountAlias, } } @@ -74,25 +78,31 @@ func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve }) } - tags := make(map[string]string, len(instance.Tags)) - for _, t := range instance.Tags { - if t.Key == nil { - continue - } - - tags[*t.Key] = pointers.Deref(t.Value) - } - assetChannel <- inventory.NewAssetEvent( ec2InstanceClassification, instance.GetResourceArn(), instance.GetResourceName(), inventory.WithRawAsset(instance), - inventory.WithTags(tags), + inventory.WithTags(e.getTags(instance)), inventory.WithCloud(inventory.AssetCloud{ - Provider: inventory.AwsCloudProvider, - Region: instance.Region, + Provider: inventory.AwsCloudProvider, + Region: instance.Region, + AvailabilityZone: e.getAvailabilityZone(instance), + Account: inventory.AssetCloudAccount{ + Id: e.AccountId, + Name: e.AccountName, + }, + Instance: &inventory.AssetCloudInstance{ + Id: pointers.Deref(instance.InstanceId), + Name: instance.GetResourceName(), + }, + Machine: &inventory.AssetCloudMachine{ + MachineType: string(instance.InstanceType), + }, + Service: &inventory.AssetCloudService{ + Name: "AWS EC2", + }, }), inventory.WithHost(inventory.AssetHost{ Architecture: string(instance.Architecture), @@ -114,3 +124,23 @@ func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve ) } } + +func (e *Ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]string { + tags := make(map[string]string, len(instance.Tags)) + for _, t := range instance.Tags { + if t.Key == nil { + continue + } + + tags[*t.Key] = pointers.Deref(t.Value) + } + return tags +} + +func (e *Ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string { + if instance.Placement == nil { + return nil + } + + return instance.Placement.AvailabilityZone +} diff --git a/internal/inventory/aws/fetcher_ec2_instance_test.go b/internal/inventory/aws/fetcher_ec2_instance_test.go index 7211508f1d..4d5f0c3a63 100644 --- a/internal/inventory/aws/fetcher_ec2_instance_test.go +++ b/internal/inventory/aws/fetcher_ec2_instance_test.go @@ -62,6 +62,9 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { PrivateIpAddress: pointers.Ref("private-ip-addre"), PublicDnsName: pointers.Ref("public-dns"), PrivateDnsName: pointers.Ref("private-dns"), + Placement: &types.Placement{ + AvailabilityZone: pointers.Ref("1a"), + }, }, Region: "us-east", } @@ -81,8 +84,23 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { inventory.WithRawAsset(instance1), inventory.WithTags(map[string]string{"Name": "test-server", "key": "value"}), inventory.WithCloud(inventory.AssetCloud{ - Provider: inventory.AwsCloudProvider, - Region: "us-east", + Provider: inventory.AwsCloudProvider, + Region: "us-east", + AvailabilityZone: pointers.Ref("1a"), + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Instance: &inventory.AssetCloudInstance{ + Id: "234567890", + Name: "test-server", + }, + Machine: &inventory.AssetCloudMachine{ + MachineType: "instance-type", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS EC2", + }, }), inventory.WithHost(inventory.AssetHost{ Architecture: string(types.ArchitectureValuesX8664), @@ -115,6 +133,20 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, Region: "us-east", + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Instance: &inventory.AssetCloudInstance{ + Id: "", + Name: "", + }, + Machine: &inventory.AssetCloudMachine{ + MachineType: "", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS EC2", + }, }), inventory.WithHost(inventory.AssetHost{}), inventory.WithNetwork(inventory.AssetNetwork{}), @@ -126,8 +158,10 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil) fetcher := Ec2InstanceFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: "123", + AccountName: "alias", } ch := make(chan inventory.AssetEvent) diff --git a/internal/inventory/aws/fetcher_s3_bucket.go b/internal/inventory/aws/fetcher_s3_bucket.go index 3be6990f57..1cf272534b 100644 --- a/internal/inventory/aws/fetcher_s3_bucket.go +++ b/internal/inventory/aws/fetcher_s3_bucket.go @@ -31,8 +31,10 @@ import ( ) type S3BucketFetcher struct { - logger *logp.Logger - provider s3BucketProvider + logger *logp.Logger + provider s3BucketProvider + AccountId string + AccountName string } var s3BucketClassification = inventory.AssetClassification{ @@ -49,8 +51,10 @@ type s3BucketProvider interface { func NewS3BucketFetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher { provider := s3.NewProvider(logger, cfg, &awslib.MultiRegionClientFactory[s3.Client]{}, identity.Account) return &S3BucketFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: identity.Account, + AccountName: identity.AccountAlias, } } @@ -76,7 +80,14 @@ func (s S3BucketFetcher) Fetch(ctx context.Context, assetChannel chan<- inventor inventory.WithRawAsset(bucket), inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, - Region: bucket.GetRegion(), + Region: bucket.Region, + Account: inventory.AssetCloudAccount{ + Id: s.AccountId, + Name: s.AccountName, + }, + Service: &inventory.AssetCloudService{ + Name: "AWS S3", + }, }), ) } diff --git a/internal/inventory/aws/fetcher_s3_bucket_test.go b/internal/inventory/aws/fetcher_s3_bucket_test.go index 0220ff8732..bdfd7ebaad 100644 --- a/internal/inventory/aws/fetcher_s3_bucket_test.go +++ b/internal/inventory/aws/fetcher_s3_bucket_test.go @@ -79,6 +79,13 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, Region: "europe-west-1", + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS S3", + }, }), ), inventory.NewAssetEvent( @@ -89,6 +96,13 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, Region: "europe-west-1", + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS S3", + }, }), ), } @@ -98,8 +112,10 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { provider.EXPECT().DescribeBuckets(mock.Anything).Return(in, nil) fetcher := S3BucketFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: "123", + AccountName: "alias", } ch := make(chan inventory.AssetEvent) diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 6ed8742a0d..574bd06cb6 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -54,8 +54,8 @@ func NewAssetInventory(logger *logp.Logger, fetchers []AssetFetcher, publisher A fetchers: fetchers, publisher: publisher, // move to a configuration parameter - bufferFlushInterval: 15 * time.Second, - bufferMaxSize: 100, + bufferFlushInterval: 10 * time.Second, + bufferMaxSize: 1600, assetCh: make(chan AssetEvent), now: now, }