Skip to content

Commit

Permalink
Enrich cloud field
Browse files Browse the repository at this point in the history
  • Loading branch information
@romulets
romulets committed Mar 19, 2024
1 parent 1911946 commit 4e76a0e
Show file tree
Hide file tree
Showing 6 changed files with 152 additions and 35 deletions.
37 changes: 32 additions & 5 deletions internal/inventory/asset.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,8 @@ const (
SubTypeS3 assetSubType = "s3"
)

type assetCloudProvider string

const (
AwsCloudProvider assetCloudProvider = "aws"
AwsCloudProvider = "aws"
)

// AssetEvent holds the whole asset
Expand Down Expand Up @@ -100,8 +98,37 @@ type AssetNetwork struct {

// AssetCloud contains information about the cloud provider
type AssetCloud struct {
Provider assetCloudProvider `json:"provider"`
Region string `json:"region"`
AvailabilityZone *string `json:"availability_zone,omitempty"`
Provider string `json:"provider,omitempty"`
Region string `json:"region,omitempty"`
Account AssetCloudAccount `json:"account"`
Instance *AssetCloudInstance `json:"instance,omitempty"`
Machine *AssetCloudMachine `json:"machine,omitempty"`
Project *AssetCloudProject `json:"project,omitempty"`
Service *AssetCloudService `json:"service,omitempty"`
}

type AssetCloudAccount struct {
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
}

type AssetCloudInstance struct {
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
}

type AssetCloudMachine struct {
MachineType string `json:"machineType,omitempty"`
}

type AssetCloudProject struct {
Id string `json:"id,omitempty"`
Name string `json:"name,omitempty"`
}

type AssetCloudService struct {
Name string `json:"name,omitempty"`
}

// AssetHost contains information of the asset in case it is a host
Expand Down
62 changes: 46 additions & 16 deletions internal/inventory/aws/fetcher_ec2_instance.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@ import (
)

type Ec2InstanceFetcher struct {
logger *logp.Logger
provider ec2InstancesProvider
logger *logp.Logger
provider ec2InstancesProvider
AccountId string
AccountName string
}

type ec2InstancesProvider interface {
Expand All @@ -49,8 +51,10 @@ var ec2InstanceClassification = inventory.AssetClassification{
func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher {
provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{})
return &Ec2InstanceFetcher{
logger: logger,
provider: provider,
logger: logger,
provider: provider,
AccountId: identity.Account,
AccountName: identity.AccountAlias,
}
}

Expand All @@ -74,25 +78,31 @@ func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve
})
}

tags := make(map[string]string, len(instance.Tags))
for _, t := range instance.Tags {
if t.Key == nil {
continue
}

tags[*t.Key] = pointers.Deref(t.Value)
}

assetChannel <- inventory.NewAssetEvent(
ec2InstanceClassification,
instance.GetResourceArn(),
instance.GetResourceName(),

inventory.WithRawAsset(instance),
inventory.WithTags(tags),
inventory.WithTags(e.getTags(instance)),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: instance.Region,
Provider: inventory.AwsCloudProvider,
Region: instance.Region,
AvailabilityZone: e.getAvailabilityZone(instance),
Account: inventory.AssetCloudAccount{
Id: e.AccountId,
Name: e.AccountName,
},
Instance: &inventory.AssetCloudInstance{
Id: pointers.Deref(instance.InstanceId),
Name: instance.GetResourceName(),
},
Machine: &inventory.AssetCloudMachine{
MachineType: string(instance.InstanceType),
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
}),
inventory.WithHost(inventory.AssetHost{
Architecture: string(instance.Architecture),
Expand All @@ -114,3 +124,23 @@ func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve
)
}
}

func (e *Ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]string {
tags := make(map[string]string, len(instance.Tags))
for _, t := range instance.Tags {
if t.Key == nil {
continue
}

tags[*t.Key] = pointers.Deref(t.Value)
}
return tags
}

func (e *Ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string {
if instance.Placement == nil {
return nil
}

return instance.Placement.AvailabilityZone
}
42 changes: 38 additions & 4 deletions internal/inventory/aws/fetcher_ec2_instance_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) {
PrivateIpAddress: pointers.Ref("private-ip-addre"),
PublicDnsName: pointers.Ref("public-dns"),
PrivateDnsName: pointers.Ref("private-dns"),
Placement: &types.Placement{
AvailabilityZone: pointers.Ref("1a"),
},
},
Region: "us-east",
}
Expand All @@ -81,8 +84,23 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) {
inventory.WithRawAsset(instance1),
inventory.WithTags(map[string]string{"Name": "test-server", "key": "value"}),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: "us-east",
Provider: inventory.AwsCloudProvider,
Region: "us-east",
AvailabilityZone: pointers.Ref("1a"),
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Instance: &inventory.AssetCloudInstance{
Id: "234567890",
Name: "test-server",
},
Machine: &inventory.AssetCloudMachine{
MachineType: "instance-type",
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
}),
inventory.WithHost(inventory.AssetHost{
Architecture: string(types.ArchitectureValuesX8664),
Expand Down Expand Up @@ -115,6 +133,20 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) {
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: "us-east",
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Instance: &inventory.AssetCloudInstance{
Id: "",
Name: "",
},
Machine: &inventory.AssetCloudMachine{
MachineType: "",
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
}),
inventory.WithHost(inventory.AssetHost{}),
inventory.WithNetwork(inventory.AssetNetwork{}),
Expand All @@ -126,8 +158,10 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) {
provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil)

fetcher := Ec2InstanceFetcher{
logger: logger,
provider: provider,
logger: logger,
provider: provider,
AccountId: "123",
AccountName: "alias",
}

ch := make(chan inventory.AssetEvent)
Expand Down
22 changes: 16 additions & 6 deletions internal/inventory/aws/fetcher_s3_bucket.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ package aws

import (
"context"

Check failure on line 21 in internal/inventory/aws/fetcher_s3_bucket.go

View workflow job for this annotation

GitHub Actions / golangci-lint 

File is not `gci`-ed with --skip-generated -s standard -s default -s prefix(github.com/elastic/cloudbeat) (gci)

"github.com/aws/aws-sdk-go-v2/aws"
"github.com/elastic/elastic-agent-libs/logp"
"github.com/samber/lo"
Expand All @@ -31,8 +30,10 @@ import (
)

type S3BucketFetcher struct {
logger *logp.Logger
provider s3BucketProvider
logger *logp.Logger
provider s3BucketProvider
AccountId string
AccountName string
}

var s3BucketClassification = inventory.AssetClassification{
Expand All @@ -49,8 +50,10 @@ type s3BucketProvider interface {
func NewS3BucketFetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher {
provider := s3.NewProvider(logger, cfg, &awslib.MultiRegionClientFactory[s3.Client]{}, identity.Account)
return &S3BucketFetcher{
logger: logger,
provider: provider,
logger: logger,
provider: provider,
AccountId: identity.Account,
AccountName: identity.AccountAlias,
}
}

Expand All @@ -76,7 +79,14 @@ func (s S3BucketFetcher) Fetch(ctx context.Context, assetChannel chan<- inventor
inventory.WithRawAsset(bucket),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: bucket.GetRegion(),
Region: bucket.Region,
Account: inventory.AssetCloudAccount{
Id: s.AccountId,
Name: s.AccountName,
},
Service: &inventory.AssetCloudService{
Name: "AWS S3",
},
}),
)
}
Expand Down
20 changes: 18 additions & 2 deletions internal/inventory/aws/fetcher_s3_bucket_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,13 @@ func TestS3BucketFetcher_Fetch(t *testing.T) {
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: "europe-west-1",
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Service: &inventory.AssetCloudService{
Name: "AWS S3",
},
}),
),
inventory.NewAssetEvent(
Expand All @@ -89,6 +96,13 @@ func TestS3BucketFetcher_Fetch(t *testing.T) {
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: "europe-west-1",
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Service: &inventory.AssetCloudService{
Name: "AWS S3",
},
}),
),
}
Expand All @@ -98,8 +112,10 @@ func TestS3BucketFetcher_Fetch(t *testing.T) {
provider.EXPECT().DescribeBuckets(mock.Anything).Return(in, nil)

fetcher := S3BucketFetcher{
logger: logger,
provider: provider,
logger: logger,
provider: provider,
AccountId: "123",
AccountName: "alias",
}

ch := make(chan inventory.AssetEvent)
Expand Down
4 changes: 2 additions & 2 deletions internal/inventory/inventory.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ func NewAssetInventory(logger *logp.Logger, fetchers []AssetFetcher, publisher A
fetchers: fetchers,
publisher: publisher,
// move to a configuration parameter
bufferFlushInterval: 15 * time.Second,
bufferMaxSize: 100,
bufferFlushInterval: 10 * time.Second,
bufferMaxSize: 1600,
assetCh: make(chan AssetEvent),
now: now,
}
Expand Down

0 comments on commit 4e76a0e

Please sign in to comment.