[Asset Inventory POC] Fetch S3 buckets (#2045)

* Fetch S3 buckets

* Enrich cloud field

* Convert bucket policy to an indexable format

internal/inventory/asset.go

@@ -35,43 +35,45 @@ type assetSubCategory string
 
 const (
  SubCategoryCompute assetSubCategory = "compute"
+ SubCategoryStorage assetSubCategory = "storage"
 )
 
 // assetType is used to build the document index. Use only numbers, letters and dashes (-)
 type assetType string
 
 const (
  TypeVirtualMachine assetType = "virtual-machine"
+ TypeObjectStorage assetType = "object-storage"
 )
 
 // assetSubType is used to build the document index. Use only numbers, letters and dashes (-)
 type assetSubType string
 
 const (
  SubTypeEC2 assetSubType = "ec2"
+ SubTypeS3 assetSubType = "s3"
 )
 
-type assetCloudProvider string
-
 const (
- AwsCloudProvider assetCloudProvider = "aws"
+ AwsCloudProvider = "aws"
 )
 
 // AssetEvent holds the whole asset
 type AssetEvent struct {
- Asset Asset
- Network *AssetNetwork
- Cloud *AssetCloud
- Host *AssetHost
- IAM *AssetIAM
+ Asset Asset
+ Network *AssetNetwork
+ Cloud *AssetCloud
+ Host *AssetHost
+ IAM *AssetIAM
+ ResourcePolicies []AssetResourcePolicy
 }
 
 // AssetClassification holds the taxonomy of an asset
 type AssetClassification struct {
  Category assetCategory `json:"category"`
- SubCategory assetSubCategory `json:"subCategory"`
+ SubCategory assetSubCategory `json:"sub_category"`
  Type assetType `json:"type"`
- SubStype assetSubType `json:"subStype"`
+ SubType  assetSubType `json:"sub_type"`
 }
 
 // Asset contains the identifiers of the asset
@@ -86,35 +88,77 @@ type Asset struct {
 
 // AssetNetwork contains network information
 type AssetNetwork struct {
- NetworkId *string `json:"networkId"`
- SubnetId *string `json:"subnetId"`
- Ipv6Address *string `json:"ipv6Address"`
- PublicIpAddress *string `json:"publicIpAddress"`
- PrivateIpAddress *string `json:"privateIpAddress"`
- PublicDnsName *string `json:"publicDnsName"`
- PrivateDnsName *string `json:"privateDnsName"`
+ NetworkId *string `json:"network_id"`
+ SubnetId *string `json:"subnet_id"`
+ Ipv6Address *string `json:"ipv6_address"`
+ PublicIpAddress *string `json:"public_ip_address"`
+ PrivateIpAddress *string `json:"private_ip_address"`
+ PublicDnsName *string `json:"public_dns_name"`
+ PrivateDnsName *string `json:"private_dns_name"`
 }
 
 // AssetCloud contains information about the cloud provider
 type AssetCloud struct {
- Provider assetCloudProvider `json:"provider"`
- Region string `json:"region"`
+ AvailabilityZone *string `json:"availability_zone,omitempty"`
+ Provider string `json:"provider,omitempty"`
+ Region string `json:"region,omitempty"`
+ Account AssetCloudAccount `json:"account"`
+ Instance *AssetCloudInstance `json:"instance,omitempty"`
+ Machine *AssetCloudMachine `json:"machine,omitempty"`
+ Project *AssetCloudProject `json:"project,omitempty"`
+ Service *AssetCloudService `json:"service,omitempty"`
+}
+
+type AssetCloudAccount struct {
+ Id string `json:"id,omitempty"`
+ Name string `json:"name,omitempty"`
+}
+
+type AssetCloudInstance struct {
+ Id string `json:"id,omitempty"`
+ Name string `json:"name,omitempty"`
+}
+
+type AssetCloudMachine struct {
+ MachineType string `json:"machine_type,omitempty"`
+}
+
+type AssetCloudProject struct {
+ Id string `json:"id,omitempty"`
+ Name string `json:"name,omitempty"`
+}
+
+type AssetCloudService struct {
+ Name string `json:"name,omitempty"`
 }
 
 // AssetHost contains information of the asset in case it is a host
 type AssetHost struct {
  Architecture string `json:"architecture"`
  ImageId *string `json:"imageId"`
- InstanceType string `json:"instanceType"`
+ InstanceType string `json:"instance_type"`
  Platform string `json:"platform"`
- PlatformDetails *string `json:"platformDetails"`
+ PlatformDetails *string `json:"platform_details"`
 }
 
 type AssetIAM struct {
  Id *string `json:"id"`
  Arn *string `json:"arn"`
 }
 
+// AssetResourcePolicy maps security policies applied directly on resources
+type AssetResourcePolicy struct {
+ Version *string `json:"version,omitempty"`
+ Id *string `json:"id,omitempty"`
+ Effect string `json:"effect,omitempty"`
+ Principal map[string]any `json:"principal,omitempty"`
+ Action []string `json:"action,omitempty"`
+ NotAction []string `json:"notAction,omitempty"`
+ Resource []string `json:"resource,omitempty"`
+ NoResource []string `json:"noResource,omitempty"`
+ Condition map[string]any `json:"condition,omitempty"`
+}
+
 // AssetEnricher functional builder function
 type AssetEnricher func(asset *AssetEvent)
 
@@ -171,13 +215,19 @@ func WithIAM(iam AssetIAM) AssetEnricher {
  }
 }
 
+func WithResourcePolicies(policies ...AssetResourcePolicy) AssetEnricher {
+ return func(a *AssetEvent) {
+ a.ResourcePolicies = policies
+ }
+}
+
 func EmptyEnricher() AssetEnricher {
  return func(_ *AssetEvent) {}
 }
 
 func generateUniqueId(c AssetClassification, resourceId string) string {
  hasher := sha256.New()
- toBeHashed := fmt.Sprintf("%s-%s-%s-%s-%s", resourceId, c.Category, c.SubCategory, c.Type, c.SubStype)
+ toBeHashed := fmt.Sprintf("%s-%s-%s-%s-%s", resourceId, c.Category, c.SubCategory, c.Type, c.SubType)
  hasher.Write([]byte(toBeHashed)) //nolint:revive
  hash := hasher.Sum(nil)
  encoded := base64.StdEncoding.EncodeToString(hash)

internal/inventory/aws/fetcher_ec2.go → internal/inventory/aws/fetcher_ec2_instance.go

@@ -30,31 +30,35 @@ import (
  "github.com/elastic/cloudbeat/internal/resources/utils/pointers"
 )
 
-type Ec2Fetcher struct {
- logger *logp.Logger
- provider instancesProvider
+type Ec2InstanceFetcher struct {
+ logger *logp.Logger
+ provider ec2InstancesProvider
+ AccountId string
+ AccountName string
 }
 
-type instancesProvider interface {
+type ec2InstancesProvider interface {
  DescribeInstances(ctx context.Context) ([]*ec2.Ec2Instance, error)
 }
 
-var ec2Classification = inventory.AssetClassification{
+var ec2InstanceClassification = inventory.AssetClassification{
  Category: inventory.CategoryInfrastructure,
  SubCategory: inventory.SubCategoryCompute,
  Type: inventory.TypeVirtualMachine,
- SubStype: inventory.SubTypeEC2,
+ SubType:  inventory.SubTypeEC2,
 }
 
 func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher {
  provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{})
- return &Ec2Fetcher{
- logger: logger,
- provider: provider,
+ return &Ec2InstanceFetcher{
+ logger: logger,
+ provider: provider,
+ AccountId: identity.Account,
+ AccountName: identity.AccountAlias,
  }
 }
 
-func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
+func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
  instances, err := e.provider.DescribeInstances(ctx)
  if err != nil {
  e.logger.Errorf("Could not list ec2 instances: %v", err)
@@ -74,25 +78,31 @@ func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As
  })
  }
 
- tags := make(map[string]string, len(instance.Tags))
- for _, t := range instance.Tags {
- if t.Key == nil {
- continue
- }
-
- tags[*t.Key] = pointers.Deref(t.Value)
- }
-
  assetChannel <- inventory.NewAssetEvent(
- ec2Classification,
+ ec2InstanceClassification,
  instance.GetResourceArn(),
  instance.GetResourceName(),
 
  inventory.WithRawAsset(instance),
- inventory.WithTags(tags),
+ inventory.WithTags(e.getTags(instance)),
  inventory.WithCloud(inventory.AssetCloud{
- Provider: inventory.AwsCloudProvider,
- Region: instance.Region,
+ Provider: inventory.AwsCloudProvider,
+ Region: instance.Region,
+ AvailabilityZone: e.getAvailabilityZone(instance),
+ Account: inventory.AssetCloudAccount{
+ Id: e.AccountId,
+ Name: e.AccountName,
+ },
+ Instance: &inventory.AssetCloudInstance{
+ Id: pointers.Deref(instance.InstanceId),
+ Name: instance.GetResourceName(),
+ },
+ Machine: &inventory.AssetCloudMachine{
+ MachineType: string(instance.InstanceType),
+ },
+ Service: &inventory.AssetCloudService{
+ Name: "AWS EC2",
+ },
  }),
  inventory.WithHost(inventory.AssetHost{
  Architecture: string(instance.Architecture),
@@ -114,3 +124,23 @@ func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As
  )
  }
 }
+
+func (e *Ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]string {
+ tags := make(map[string]string, len(instance.Tags))
+ for _, t := range instance.Tags {
+ if t.Key == nil {
+ continue
+ }
+
+ tags[*t.Key] = pointers.Deref(t.Value)
+ }
+ return tags
+}
+
+func (e *Ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string {
+ if instance.Placement == nil {
+ return nil
+ }
+
+ return instance.Placement.AvailabilityZone
+}

internal/inventory/aws/fetcher_ec2_test.go → internal/inventory/aws/fetcher_ec2_instance_test.go

@@ -32,7 +32,7 @@ import (
  "github.com/elastic/cloudbeat/internal/resources/utils/pointers"
 )
 
-func TestFetch(t *testing.T) {
+func TestEC2InstanceFetcher_Fetch(t *testing.T) {
  instance1 := &ec2beat.Ec2Instance{
  Instance: types.Instance{
  IamInstanceProfile: &types.IamInstanceProfile{
@@ -62,6 +62,9 @@ func TestFetch(t *testing.T) {
  PrivateIpAddress: pointers.Ref("private-ip-addre"),
  PublicDnsName: pointers.Ref("public-dns"),
  PrivateDnsName: pointers.Ref("private-dns"),
+ Placement: &types.Placement{
+ AvailabilityZone: pointers.Ref("1a"),
+ },
  },
  Region: "us-east",
  }
@@ -75,14 +78,29 @@ func TestFetch(t *testing.T) {
 
  expected := []inventory.AssetEvent{
  inventory.NewAssetEvent(
- ec2Classification,
+ ec2InstanceClassification,
  "arn:aws:ec2:us-east::ec2/234567890",
  "test-server",
  inventory.WithRawAsset(instance1),
  inventory.WithTags(map[string]string{"Name": "test-server", "key": "value"}),
  inventory.WithCloud(inventory.AssetCloud{
- Provider: inventory.AwsCloudProvider,
- Region: "us-east",
+ Provider: inventory.AwsCloudProvider,
+ Region: "us-east",
+ AvailabilityZone: pointers.Ref("1a"),
+ Account: inventory.AssetCloudAccount{
+ Id: "123",
+ Name: "alias",
+ },
+ Instance: &inventory.AssetCloudInstance{
+ Id: "234567890",
+ Name: "test-server",
+ },
+ Machine: &inventory.AssetCloudMachine{
+ MachineType: "instance-type",
+ },
+ Service: &inventory.AssetCloudService{
+ Name: "AWS EC2",
+ },
  }),
  inventory.WithHost(inventory.AssetHost{
  Architecture: string(types.ArchitectureValuesX8664),
@@ -107,27 +125,43 @@ func TestFetch(t *testing.T) {
  ),
 
  inventory.NewAssetEvent(
- ec2Classification,
+ ec2InstanceClassification,
  "",
  "",
  inventory.WithRawAsset(instance2),
  inventory.WithTags(map[string]string{}),
  inventory.WithCloud(inventory.AssetCloud{
  Provider: inventory.AwsCloudProvider,
  Region: "us-east",
+ Account: inventory.AssetCloudAccount{
+ Id: "123",
+ Name: "alias",
+ },
+ Instance: &inventory.AssetCloudInstance{
+ Id: "",
+ Name: "",
+ },
+ Machine: &inventory.AssetCloudMachine{
+ MachineType: "",
+ },
+ Service: &inventory.AssetCloudService{
+ Name: "AWS EC2",
+ },
  }),
  inventory.WithHost(inventory.AssetHost{}),
  inventory.WithNetwork(inventory.AssetNetwork{}),
  ),
  }
 
  logger := logp.NewLogger("test_fetcher_ec2")
- provider := newMockInstancesProvider(t)
+ provider := newMockEc2InstancesProvider(t)
  provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil)
 
- fetcher := Ec2Fetcher{
- logger: logger,
- provider: provider,
+ fetcher := Ec2InstanceFetcher{
+ logger: logger,
+ provider: provider,
+ AccountId: "123",
+ AccountName: "alias",
  }
 
  ch := make(chan inventory.AssetEvent)