From 00087672a4415819d45689c75e4de7369599ad74 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 18 Mar 2024 12:10:22 +0100 Subject: [PATCH 1/8] add regional fallback for AWS Gov --- internal/flavors/benchmark/aws.go | 36 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/internal/flavors/benchmark/aws.go b/internal/flavors/benchmark/aws.go index 11b0942584..c3b7fdd958 100644 --- a/internal/flavors/benchmark/aws.go +++ b/internal/flavors/benchmark/aws.go @@ -22,6 +22,8 @@ import ( "errors" "fmt" + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" @@ -59,21 +61,43 @@ func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Conf return nil, nil, nil, err } + var ( + awsConfig *awssdk.Config + awsIdentity *cloud.Identity + err error + ) + + awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) + if err != nil && cfg.CloudConfig.Aws.Cred.DefaultRegion == "" { + log.Warn("failed to initialize; checking if running in AWSGov") + cfg.CloudConfig.Aws.Cred.DefaultRegion = endpoints.UsGovEast1RegionID + awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) + } + + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get AWS Identity: %w", err) + } + log.Info("successfully retrieved AWS Identity") + + return registry.NewRegistry( + log, + registry.WithFetchersMap(preset.NewCisAwsFetchers(log, *awsConfig, ch, awsIdentity)), + ), cloud.NewDataProvider(cloud.WithAccount(*awsIdentity)), nil, nil +} + +func (a *AWS) getIdentity(ctx context.Context, cfg *config.Config) (*awssdk.Config, *cloud.Identity, error) { // TODO: make this mock-able awsConfig, err := aws.InitializeAWSConfig(cfg.CloudConfig.Aws.Cred) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to initialize AWS credentials: %w", err) + return nil, nil, fmt.Errorf("failed to initialize AWS credentials: %w", err) } awsIdentity, err := a.IdentityProvider.GetIdentity(ctx, awsConfig) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get AWS identity: %w", err) + return nil, nil, fmt.Errorf("failed to get AWS identity: %w", err) } - return registry.NewRegistry( - log, - registry.WithFetchersMap(preset.NewCisAwsFetchers(log, awsConfig, ch, awsIdentity)), - ), cloud.NewDataProvider(cloud.WithAccount(*awsIdentity)), nil, nil + return &awsConfig, awsIdentity, nil } func (a *AWS) checkDependencies() error { From 9c119fe34c5cdb6344fbe72eb4d3ed8c84064ca3 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 20 Mar 2024 12:57:18 +0100 Subject: [PATCH 2/8] improve log messages --- internal/flavors/benchmark/aws.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/flavors/benchmark/aws.go b/internal/flavors/benchmark/aws.go index c3b7fdd958..543c852ace 100644 --- a/internal/flavors/benchmark/aws.go +++ b/internal/flavors/benchmark/aws.go @@ -69,7 +69,7 @@ func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Conf awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) if err != nil && cfg.CloudConfig.Aws.Cred.DefaultRegion == "" { - log.Warn("failed to initialize; checking if running in AWSGov") + log.Warn("failed to initialize identity; retrying to check AWS Gov Cloud regions") cfg.CloudConfig.Aws.Cred.DefaultRegion = endpoints.UsGovEast1RegionID awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) } @@ -77,7 +77,7 @@ func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Conf if err != nil { return nil, nil, nil, fmt.Errorf("failed to get AWS Identity: %w", err) } - log.Info("successfully retrieved AWS Identity") + log.Infof("successfully retrieved AWS Identity: %+v", awsIdentity) return registry.NewRegistry( log, From c15ea83fa1d354e67ea61f7bc0a8e472a6b471ea Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 20 Mar 2024 15:15:56 +0100 Subject: [PATCH 3/8] make S3 fetcher partition agnostic --- internal/flavors/benchmark/aws.go | 5 +-- internal/resources/providers/awslib/aws.go | 34 +++++++++++++------ .../resources/providers/awslib/s3/provider.go | 20 ++++++++--- 3 files changed, 43 insertions(+), 16 deletions(-) diff --git a/internal/flavors/benchmark/aws.go b/internal/flavors/benchmark/aws.go index 543c852ace..5a7a6dc999 100644 --- a/internal/flavors/benchmark/aws.go +++ b/internal/flavors/benchmark/aws.go @@ -23,7 +23,6 @@ import ( "fmt" awssdk "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" "github.com/elastic/elastic-agent-libs/logp" @@ -68,9 +67,11 @@ func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Conf ) awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) + // TODO(kuba): Ask when the DefaultRegion is empty. Is there a chance it is + // set, but incorrectly - e.g. "us-east-1"? if err != nil && cfg.CloudConfig.Aws.Cred.DefaultRegion == "" { log.Warn("failed to initialize identity; retrying to check AWS Gov Cloud regions") - cfg.CloudConfig.Aws.Cred.DefaultRegion = endpoints.UsGovEast1RegionID + cfg.CloudConfig.Aws.Cred.DefaultRegion = awslib.DefaultGovRegion awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) } diff --git a/internal/resources/providers/awslib/aws.go b/internal/resources/providers/awslib/aws.go index 51942b8bf5..0a18c536d5 100644 --- a/internal/resources/providers/awslib/aws.go +++ b/internal/resources/providers/awslib/aws.go @@ -19,11 +19,14 @@ package awslib import ( "errors" + + "github.com/elastic/cloudbeat/internal/resources/utils/pointers" ) const ( - DefaultRegion = "us-east-1" - GlobalRegion = "global" + DefaultRegion = "us-east-1" + DefaultGovRegion = "us-gov-east-1" + GlobalRegion = "global" ) var ErrClientNotFound = errors.New("aws client not found") @@ -35,18 +38,29 @@ type AwsResource interface { GetRegion() string } -func GetClient[T any](region *string, list map[string]T) (T, error) { - c, ok := list[getRegion(region)] - if !ok { - return c, ErrClientNotFound +func GetDefaultClient[T any](list map[string]T) (T, error) { + c, ok := list[DefaultRegion] + if ok { + return c, nil } - return c, nil + + c, ok = list[DefaultGovRegion] + if ok { + return c, nil + } + + return c, ErrClientNotFound } -func getRegion(region *string) string { +func GetClient[T any](region *string, list map[string]T) (T, error) { if region == nil { - return DefaultRegion + return GetDefaultClient(list) } - return *region + c, ok := list[pointers.Deref(region)] + if !ok { + return c, ErrClientNotFound + } + + return c, nil } diff --git a/internal/resources/providers/awslib/s3/provider.go b/internal/resources/providers/awslib/s3/provider.go index 50d1240aef..4bdf7f6554 100644 --- a/internal/resources/providers/awslib/s3/provider.go +++ b/internal/resources/providers/awslib/s3/provider.go @@ -59,7 +59,11 @@ func NewProvider(log *logp.Logger, cfg aws.Config, factory awslib.CrossRegionFac } func (p Provider) DescribeBuckets(ctx context.Context) ([]awslib.AwsResource, error) { - clientBuckets, err := p.clients[awslib.DefaultRegion].ListBuckets(ctx, &s3Client.ListBucketsInput{}) + defaultClient, err := awslib.GetDefaultClient(p.clients) + if err != nil { + return nil, fmt.Errorf("could not select default region client: %w", err) + } + clientBuckets, err := defaultClient.ListBuckets(ctx, &s3Client.ListBucketsInput{}) if err != nil { p.log.Errorf("Could not list s3 buckets: %v", err) return nil, err @@ -225,15 +229,23 @@ func (p Provider) getBucketEncryptionAlgorithm(ctx context.Context, bucketName * } func (p Provider) getBucketRegion(ctx context.Context, bucketName *string) (string, error) { - location, err := p.clients[awslib.DefaultRegion].GetBucketLocation(ctx, &s3Client.GetBucketLocationInput{Bucket: bucketName}) + defaultClient, err := awslib.GetDefaultClient(p.clients) + if err != nil { + return "", fmt.Errorf("could not select default region client: %w", err) + } + location, err := defaultClient.GetBucketLocation(ctx, &s3Client.GetBucketLocationInput{Bucket: bucketName}) if err != nil { return "", err } region := string(location.LocationConstraint) - // Region us-east-1 have a LocationConstraint of null. + // Region us-east-1 have a LocationConstraint of null... if region == "" { - region = "us-east-1" + region = awslib.DefaultRegion + // ...but check if it's not the AWS GovCloud partition + if _, ok := p.clients[awslib.DefaultRegion]; !ok { + region = awslib.DefaultGovRegion + } } return region, nil From bb85392a82bb8f9c8f1d250bf086d5a16f7561c8 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 20 Mar 2024 15:18:48 +0100 Subject: [PATCH 4/8] make CloudTrail fetcher partition agnostic --- internal/resources/providers/awslib/cloudtrail/provider.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/resources/providers/awslib/cloudtrail/provider.go b/internal/resources/providers/awslib/cloudtrail/provider.go index 183a3c401c..acb8c7488b 100644 --- a/internal/resources/providers/awslib/cloudtrail/provider.go +++ b/internal/resources/providers/awslib/cloudtrail/provider.go @@ -19,6 +19,7 @@ package cloudtrail import ( "context" + "fmt" "github.com/aws/aws-sdk-go-v2/service/cloudtrail" "github.com/aws/aws-sdk-go-v2/service/cloudtrail/types" @@ -40,7 +41,11 @@ type Client interface { func (p Provider) DescribeTrails(ctx context.Context) ([]TrailInfo, error) { input := cloudtrail.DescribeTrailsInput{} - output, err := p.clients[awslib.DefaultRegion].DescribeTrails(ctx, &input) + defaultClient, err := awslib.GetDefaultClient(p.clients) + if err != nil { + return nil, fmt.Errorf("could not select default region client: %w", err) + } + output, err := defaultClient.DescribeTrails(ctx, &input) if err != nil { return nil, err } From c4dc93f3a9517c8038a96789a333452260c7c94c Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 20 Mar 2024 15:41:15 +0100 Subject: [PATCH 5/8] remove fine detail from info log --- internal/flavors/benchmark/aws.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/flavors/benchmark/aws.go b/internal/flavors/benchmark/aws.go index 5a7a6dc999..e9a0108a27 100644 --- a/internal/flavors/benchmark/aws.go +++ b/internal/flavors/benchmark/aws.go @@ -78,7 +78,7 @@ func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Conf if err != nil { return nil, nil, nil, fmt.Errorf("failed to get AWS Identity: %w", err) } - log.Infof("successfully retrieved AWS Identity: %+v", awsIdentity) + log.Info("successfully retrieved AWS Identity") return registry.NewRegistry( log, From 47082fb65906f2db40c4c8c633a89d160ab99d42 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Wed, 20 Mar 2024 16:17:34 +0100 Subject: [PATCH 6/8] handle IAM credentials report better --- .../resources/providers/awslib/iam/root_account.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/internal/resources/providers/awslib/iam/root_account.go b/internal/resources/providers/awslib/iam/root_account.go index 4bee6dcc4c..a2400bf23d 100644 --- a/internal/resources/providers/awslib/iam/root_account.go +++ b/internal/resources/providers/awslib/iam/root_account.go @@ -39,10 +39,15 @@ func (p Provider) getRootAccountUser(rootAccount *CredentialReport) *types.User return nil } - pwdLastUsed, err := time.Parse(time.RFC3339, rootAccount.PasswordLastUsed) - if err != nil { - p.log.Errorf("fail to parse root account password last used, error: %v", err) - return nil + pwdLastUsed := time.Time{} + // "no_information" if never used, "N/A" if user has no password + // Docs: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_getting-report.html + if rootAccount.PasswordLastUsed != "no_information" && rootAccount.PasswordLastUsed != "N/A" { + pwdLastUsed, err = time.Parse(time.RFC3339, rootAccount.PasswordLastUsed) + if err != nil { + p.log.Errorf("fail to parse root account password last used, error: %v", err) + return nil + } } return &types.User{ From 5b582f048353bc142f6535a4c5ddea2bf182c418 Mon Sep 17 00:00:00 2001 From: kubasobon Date: Mon, 25 Mar 2024 16:11:43 +0100 Subject: [PATCH 7/8] fix region picker for AWS Organizations --- internal/flavors/benchmark/aws_org.go | 40 +++++++++++++++++++++------ 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/internal/flavors/benchmark/aws_org.go b/internal/flavors/benchmark/aws_org.go index 51ce8b12a1..731b13b936 100644 --- a/internal/flavors/benchmark/aws_org.go +++ b/internal/flavors/benchmark/aws_org.go @@ -71,23 +71,30 @@ func (a *AWSOrg) initialize(ctx context.Context, log *logp.Logger, cfg *config.C return nil, nil, nil, err } - // TODO: make this mock-able - awsConfig, err := aws.InitializeAWSConfig(cfg.CloudConfig.Aws.Cred) - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to initialize AWS credentials: %w", err) - } + var ( + awsConfig *awssdk.Config + awsIdentity *cloud.Identity + err error + ) - a.IAMProvider = iam.NewIAMProvider(log, awsConfig, nil) + awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) + if err != nil && cfg.CloudConfig.Aws.Cred.DefaultRegion == "" { + log.Warn("failed to initialize identity; retrying to check AWS Gov Cloud regions") + cfg.CloudConfig.Aws.Cred.DefaultRegion = awslib.DefaultGovRegion + awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) + } - awsIdentity, err := a.IdentityProvider.GetIdentity(ctx, awsConfig) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get AWS identity: %w", err) + return nil, nil, nil, fmt.Errorf("failed to get AWS Identity: %w", err) } + log.Info("successfully retrieved AWS Identity") + + a.IAMProvider = iam.NewIAMProvider(log, *awsConfig, nil) cache := make(map[string]registry.FetchersMap) reg := registry.NewRegistry(log, registry.WithUpdater( func() (registry.FetchersMap, error) { - accounts, err := a.getAwsAccounts(ctx, log, awsConfig, awsIdentity) + accounts, err := a.getAwsAccounts(ctx, log, *awsConfig, awsIdentity) if err != nil { return nil, fmt.Errorf("failed to get AWS accounts: %w", err) } @@ -211,6 +218,21 @@ func (a *AWSOrg) pickManagementAccountRole(ctx context.Context, log *logp.Logger return config, nil } +func (a *AWSOrg) getIdentity(ctx context.Context, cfg *config.Config) (*awssdk.Config, *cloud.Identity, error) { + // TODO: make this mock-able + awsConfig, err := aws.InitializeAWSConfig(cfg.CloudConfig.Aws.Cred) + if err != nil { + return nil, nil, fmt.Errorf("failed to initialize AWS credentials: %w", err) + } + + awsIdentity, err := a.IdentityProvider.GetIdentity(ctx, awsConfig) + if err != nil { + return nil, nil, fmt.Errorf("failed to get AWS identity: %w", err) + } + + return &awsConfig, awsIdentity, nil +} + func (a *AWSOrg) checkDependencies() error { if a.IAMProvider == nil { return errors.New("aws iam provider is uninitialized") From 411cbe9884aefda45ce683866f6a7aff75ba0e7f Mon Sep 17 00:00:00 2001 From: kubasobon Date: Tue, 26 Mar 2024 11:25:24 +0100 Subject: [PATCH 8/8] remove redundant TODOs --- internal/flavors/benchmark/aws.go | 3 --- internal/flavors/benchmark/aws_org.go | 1 - 2 files changed, 4 deletions(-) diff --git a/internal/flavors/benchmark/aws.go b/internal/flavors/benchmark/aws.go index e9a0108a27..848160e51a 100644 --- a/internal/flavors/benchmark/aws.go +++ b/internal/flavors/benchmark/aws.go @@ -67,8 +67,6 @@ func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Conf ) awsConfig, awsIdentity, err = a.getIdentity(ctx, cfg) - // TODO(kuba): Ask when the DefaultRegion is empty. Is there a chance it is - // set, but incorrectly - e.g. "us-east-1"? if err != nil && cfg.CloudConfig.Aws.Cred.DefaultRegion == "" { log.Warn("failed to initialize identity; retrying to check AWS Gov Cloud regions") cfg.CloudConfig.Aws.Cred.DefaultRegion = awslib.DefaultGovRegion @@ -87,7 +85,6 @@ func (a *AWS) initialize(ctx context.Context, log *logp.Logger, cfg *config.Conf } func (a *AWS) getIdentity(ctx context.Context, cfg *config.Config) (*awssdk.Config, *cloud.Identity, error) { - // TODO: make this mock-able awsConfig, err := aws.InitializeAWSConfig(cfg.CloudConfig.Aws.Cred) if err != nil { return nil, nil, fmt.Errorf("failed to initialize AWS credentials: %w", err) diff --git a/internal/flavors/benchmark/aws_org.go b/internal/flavors/benchmark/aws_org.go index 910ab99108..b44a458847 100644 --- a/internal/flavors/benchmark/aws_org.go +++ b/internal/flavors/benchmark/aws_org.go @@ -219,7 +219,6 @@ func (a *AWSOrg) pickManagementAccountRole(ctx context.Context, log *logp.Logger } func (a *AWSOrg) getIdentity(ctx context.Context, cfg *config.Config) (*awssdk.Config, *cloud.Identity, error) { - // TODO: make this mock-able awsConfig, err := aws.InitializeAWSConfig(cfg.CloudConfig.Aws.Cred) if err != nil { return nil, nil, fmt.Errorf("failed to initialize AWS credentials: %w", err)