[Asset Inventory Poc] Fech IAM Users, Roles and Policies (#2052)

* Fetch IAM Users

* Fetch IAM Roles

* Rename package to awsfetchers

* Fetch IAM Policies

* Refactor provider injection

* Unexport unnecessary structs

* Fix mocks

internal/flavors/asset_inventory.go

@@ -29,7 +29,7 @@ import (
 
  "github.com/elastic/cloudbeat/internal/config"
  "github.com/elastic/cloudbeat/internal/inventory"
- awsinventory "github.com/elastic/cloudbeat/internal/inventory/aws"
+ "github.com/elastic/cloudbeat/internal/inventory/awsfetcher"
  "github.com/elastic/cloudbeat/internal/resources/providers/awslib"
 )
 
@@ -92,7 +92,7 @@ func initAwsFetchers(ctx context.Context, cfg *config.Config, logger *logp.Logge
  return nil, err
  }
 
- return awsinventory.Fetchers(logger, awsIdentity, awsConfig), nil
+ return awsfetcher.New(logger, awsIdentity, awsConfig), nil
 }
 
 func (bt *assetInventory) Run(*beat.Beat) error {

internal/inventory/asset.go

@@ -28,6 +28,7 @@ type assetCategory string
 
 const (
  CategoryInfrastructure assetCategory = "infrastructure"
+ CategoryIdentity assetCategory = "identity"
 )
 
 // assetSubCategory is used to build the document index. Use only numbers, letters and dashes (-)
@@ -36,6 +37,8 @@ type assetSubCategory string
 const (
  SubCategoryCompute assetSubCategory = "compute"
  SubCategoryStorage assetSubCategory = "storage"
+
+ SubCategoryCloudProviderAccount assetSubCategory = "cloud-provider-account"
 )
 
 // assetType is used to build the document index. Use only numbers, letters and dashes (-)
@@ -44,6 +47,10 @@ type assetType string
 const (
  TypeVirtualMachine assetType = "virtual-machine"
  TypeObjectStorage assetType = "object-storage"
+
+ TypeUser assetType = "user"
+ TypeServiceAccount assetType = "service-account"
+ TypePermissions assetType = "permissions"
 )
 
 // assetSubType is used to build the document index. Use only numbers, letters and dashes (-)
@@ -52,6 +59,7 @@ type assetSubType string
 const (
  SubTypeEC2 assetSubType = "ec2"
  SubTypeS3 assetSubType = "s3"
+ SubTypeIAM assetSubType = "iam"
 )
 
 const (
@@ -187,6 +195,10 @@ func WithRawAsset(raw any) AssetEnricher {
 
 func WithTags(tags map[string]string) AssetEnricher {
  return func(a *AssetEvent) {
+ if len(tags) == 0 {
+ return
+ }
+
  a.Asset.Tags = tags
  }
 }
@@ -217,6 +229,10 @@ func WithIAM(iam AssetIAM) AssetEnricher {
 
 func WithResourcePolicies(policies ...AssetResourcePolicy) AssetEnricher {
  return func(a *AssetEvent) {
+ if len(policies) == 0 {
+ return
+ }
+
  a.ResourcePolicies = policies
  }
 }

internal/inventory/awsfetcher/awsfetchers.go

@@ -0,0 +1,44 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package awsfetcher
+
+import (
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/elastic/elastic-agent-libs/logp"
+
+ "github.com/elastic/cloudbeat/internal/dataprovider/providers/cloud"
+ "github.com/elastic/cloudbeat/internal/inventory"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib/ec2"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib/iam"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib/s3"
+)
+
+func New(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) []inventory.AssetFetcher {
+ iamProvider := iam.NewIAMProvider(logger, cfg, &awslib.MultiRegionClientFactory[iam.AccessAnalyzerClient]{})
+ ec2Provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{})
+ s3Provider := s3.NewProvider(logger, cfg, &awslib.MultiRegionClientFactory[s3.Client]{}, identity.Account)
+
+ return []inventory.AssetFetcher{
+ newEc2InstancesFetcher(logger, identity, ec2Provider),
+ newS3BucketFetcher(logger, identity, s3Provider),
+ newIamUserFetcher(logger, identity, iamProvider),
+ newIamRoleFetcher(logger, identity, iamProvider),
+ newIamPolicyFetcher(logger, identity, iamProvider),
+ }
+}

internal/inventory/aws/fetchers.go → internal/inventory/awsfetcher/awsfetchers_test.go

@@ -15,19 +15,38 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package aws
+package awsfetcher
 
 import (
- "github.com/aws/aws-sdk-go-v2/aws"
- "github.com/elastic/elastic-agent-libs/logp"
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
 
- "github.com/elastic/cloudbeat/internal/dataprovider/providers/cloud"
  "github.com/elastic/cloudbeat/internal/inventory"
 )
 
-func Fetchers(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) []inventory.AssetFetcher {
- return []inventory.AssetFetcher{
- newEc2Fetcher(logger, identity, cfg),
- NewS3BucketFetcher(logger, identity, cfg),
+func collectResourcesAndMatch(t *testing.T, fetcher inventory.AssetFetcher, expected []inventory.AssetEvent) {
+ t.Helper()
+
+ ch := make(chan inventory.AssetEvent)
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+ go func() {
+ fetcher.Fetch(ctx, ch)
+ }()
+
+ received := make([]inventory.AssetEvent, 0, len(expected))
+ for len(expected) != len(received) {
+ select {
+ case <-ctx.Done():
+ assert.ElementsMatch(t, expected, received)
+ return
+ case event := <-ch:
+ received = append(received, event)
+ }
  }
+
+ assert.ElementsMatch(t, expected, received)
 }

internal/inventory/aws/fetcher_ec2_instance.go → internal/inventory/awsfetcher/fetcher_ec2_instance.go

@@ -15,22 +15,20 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package aws
+package awsfetcher
 
 import (
  "context"
 
- "github.com/aws/aws-sdk-go-v2/aws"
  "github.com/elastic/elastic-agent-libs/logp"
 
  "github.com/elastic/cloudbeat/internal/dataprovider/providers/cloud"
  "github.com/elastic/cloudbeat/internal/inventory"
- "github.com/elastic/cloudbeat/internal/resources/providers/awslib"
  "github.com/elastic/cloudbeat/internal/resources/providers/awslib/ec2"
  "github.com/elastic/cloudbeat/internal/resources/utils/pointers"
 )
 
-type Ec2InstanceFetcher struct {
+type ec2InstanceFetcher struct {
  logger *logp.Logger
  provider ec2InstancesProvider
  AccountId string
@@ -48,17 +46,19 @@ var ec2InstanceClassification = inventory.AssetClassification{
  SubType: inventory.SubTypeEC2,
 }
 
-func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher {
- provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{})
- return &Ec2InstanceFetcher{
+func newEc2InstancesFetcher(logger *logp.Logger, identity *cloud.Identity, provider ec2InstancesProvider) inventory.AssetFetcher {
+ return &ec2InstanceFetcher{
  logger: logger,
  provider: provider,
  AccountId: identity.Account,
  AccountName: identity.AccountAlias,
  }
 }
 
-func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
+func (e *ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
+ e.logger.Info("Fetching EC2 Instances")
+ defer e.logger.Info("Fetching EC2 Instances - Finished")
+
  instances, err := e.provider.DescribeInstances(ctx)
  if err != nil {
  e.logger.Errorf("Could not list ec2 instances: %v", err)
@@ -125,7 +125,7 @@ func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve
  }
 }
 
-func (e *Ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]string {
+func (e *ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]string {
  tags := make(map[string]string, len(instance.Tags))
  for _, t := range instance.Tags {
  if t.Key == nil {
@@ -137,7 +137,7 @@ func (e *Ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]strin
  return tags
 }
 
-func (e *Ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string {
+func (e *ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string {
  if instance.Placement == nil {
  return nil
  }

internal/inventory/aws/fetcher_ec2_instance_test.go → internal/inventory/awsfetcher/fetcher_ec2_instance_test.go

@@ -15,18 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
-package aws
+package awsfetcher
 
 import (
- "context"
  "testing"
- "time"
 
  "github.com/aws/aws-sdk-go-v2/service/ec2/types"
  "github.com/elastic/elastic-agent-libs/logp"
- "github.com/stretchr/testify/assert"
  "github.com/stretchr/testify/mock"
 
+ "github.com/elastic/cloudbeat/internal/dataprovider/providers/cloud"
  "github.com/elastic/cloudbeat/internal/inventory"
  ec2beat "github.com/elastic/cloudbeat/internal/resources/providers/awslib/ec2"
  "github.com/elastic/cloudbeat/internal/resources/utils/pointers"
@@ -157,30 +155,8 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) {
  provider := newMockEc2InstancesProvider(t)
  provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil)
 
- fetcher := Ec2InstanceFetcher{
- logger: logger,
- provider: provider,
- AccountId: "123",
- AccountName: "alias",
- }
-
- ch := make(chan inventory.AssetEvent)
- ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
- defer cancel()
- go func() {
- fetcher.Fetch(ctx, ch)
- }()
-
- received := make([]inventory.AssetEvent, 0, len(expected))
- for len(expected) != len(received) {
- select {
- case <-ctx.Done():
- assert.ElementsMatch(t, expected, received)
- return
- case event := <-ch:
- received = append(received, event)
- }
- }
+ identity := &cloud.Identity{Account: "123", AccountAlias: "alias"}
+ fetcher := newEc2InstancesFetcher(logger, identity, provider)
 
- assert.ElementsMatch(t, expected, received)
+ collectResourcesAndMatch(t, fetcher, expected)
 }