From 9cc275ee9922e49ffc17f09c5e6fb7d5767be3d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Farias?= Date: Mon, 18 Mar 2024 16:09:58 +0100 Subject: [PATCH 1/3] Fetch S3 buckets --- internal/inventory/asset.go | 3 + ...fetcher_ec2.go => fetcher_ec2_instance.go} | 14 +- ...2_test.go => fetcher_ec2_instance_test.go} | 10 +- internal/inventory/aws/fetcher_s3_bucket.go | 83 ++++++++++++ .../inventory/aws/fetcher_s3_bucket_test.go | 124 ++++++++++++++++++ internal/inventory/aws/fetchers.go | 1 + ...ider.go => mock_ec2_instances_provider.go} | 34 ++--- .../inventory/aws/mock_s3_bucket_provider.go | 109 +++++++++++++++ .../resources/providers/awslib/s3/provider.go | 6 +- .../providers/awslib/s3/provider_test.go | 26 ++-- internal/resources/providers/awslib/s3/s3.go | 2 +- 11 files changed, 366 insertions(+), 46 deletions(-) rename internal/inventory/aws/{fetcher_ec2.go => fetcher_ec2_instance.go} (91%) rename internal/inventory/aws/{fetcher_ec2_test.go => fetcher_ec2_instance_test.go} (96%) create mode 100644 internal/inventory/aws/fetcher_s3_bucket.go create mode 100644 internal/inventory/aws/fetcher_s3_bucket_test.go rename internal/inventory/aws/{mock_instances_provider.go => mock_ec2_instances_provider.go} (55%) create mode 100644 internal/inventory/aws/mock_s3_bucket_provider.go diff --git a/internal/inventory/asset.go b/internal/inventory/asset.go index dd2aac75a8..f6f3a19c27 100644 --- a/internal/inventory/asset.go +++ b/internal/inventory/asset.go @@ -35,6 +35,7 @@ type assetSubCategory string const ( SubCategoryCompute assetSubCategory = "compute" + SubCategoryStorage assetSubCategory = "storage" ) // assetType is used to build the document index. Use only numbers, letters and dashes (-) @@ -42,6 +43,7 @@ type assetType string const ( TypeVirtualMachine assetType = "virtual-machine" + TypeObjectStorage assetType = "object-storage" ) // assetSubType is used to build the document index. Use only numbers, letters and dashes (-) @@ -49,6 +51,7 @@ type assetSubType string const ( SubTypeEC2 assetSubType = "ec2" + SubTypeS3 assetSubType = "s3" ) type assetCloudProvider string diff --git a/internal/inventory/aws/fetcher_ec2.go b/internal/inventory/aws/fetcher_ec2_instance.go similarity index 91% rename from internal/inventory/aws/fetcher_ec2.go rename to internal/inventory/aws/fetcher_ec2_instance.go index 048203a83c..5f13480886 100644 --- a/internal/inventory/aws/fetcher_ec2.go +++ b/internal/inventory/aws/fetcher_ec2_instance.go @@ -30,16 +30,16 @@ import ( "github.com/elastic/cloudbeat/internal/resources/utils/pointers" ) -type Ec2Fetcher struct { +type Ec2InstanceFetcher struct { logger *logp.Logger - provider instancesProvider + provider ec2InstancesProvider } -type instancesProvider interface { +type ec2InstancesProvider interface { DescribeInstances(ctx context.Context) ([]*ec2.Ec2Instance, error) } -var ec2Classification = inventory.AssetClassification{ +var ec2InstanceClassification = inventory.AssetClassification{ Category: inventory.CategoryInfrastructure, SubCategory: inventory.SubCategoryCompute, Type: inventory.TypeVirtualMachine, @@ -48,13 +48,13 @@ var ec2Classification = inventory.AssetClassification{ func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher { provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{}) - return &Ec2Fetcher{ + return &Ec2InstanceFetcher{ logger: logger, provider: provider, } } -func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) { +func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) { instances, err := e.provider.DescribeInstances(ctx) if err != nil { e.logger.Errorf("Could not list ec2 instances: %v", err) @@ -84,7 +84,7 @@ func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As } assetChannel <- inventory.NewAssetEvent( - ec2Classification, + ec2InstanceClassification, instance.GetResourceArn(), instance.GetResourceName(), diff --git a/internal/inventory/aws/fetcher_ec2_test.go b/internal/inventory/aws/fetcher_ec2_instance_test.go similarity index 96% rename from internal/inventory/aws/fetcher_ec2_test.go rename to internal/inventory/aws/fetcher_ec2_instance_test.go index 06171c2630..7211508f1d 100644 --- a/internal/inventory/aws/fetcher_ec2_test.go +++ b/internal/inventory/aws/fetcher_ec2_instance_test.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/cloudbeat/internal/resources/utils/pointers" ) -func TestFetch(t *testing.T) { +func TestEC2InstanceFetcher_Fetch(t *testing.T) { instance1 := &ec2beat.Ec2Instance{ Instance: types.Instance{ IamInstanceProfile: &types.IamInstanceProfile{ @@ -75,7 +75,7 @@ func TestFetch(t *testing.T) { expected := []inventory.AssetEvent{ inventory.NewAssetEvent( - ec2Classification, + ec2InstanceClassification, "arn:aws:ec2:us-east::ec2/234567890", "test-server", inventory.WithRawAsset(instance1), @@ -107,7 +107,7 @@ func TestFetch(t *testing.T) { ), inventory.NewAssetEvent( - ec2Classification, + ec2InstanceClassification, "", "", inventory.WithRawAsset(instance2), @@ -122,10 +122,10 @@ func TestFetch(t *testing.T) { } logger := logp.NewLogger("test_fetcher_ec2") - provider := newMockInstancesProvider(t) + provider := newMockEc2InstancesProvider(t) provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil) - fetcher := Ec2Fetcher{ + fetcher := Ec2InstanceFetcher{ logger: logger, provider: provider, } diff --git a/internal/inventory/aws/fetcher_s3_bucket.go b/internal/inventory/aws/fetcher_s3_bucket.go new file mode 100644 index 0000000000..3be6990f57 --- /dev/null +++ b/internal/inventory/aws/fetcher_s3_bucket.go @@ -0,0 +1,83 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package aws + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/samber/lo" + + "github.com/elastic/cloudbeat/internal/dataprovider/providers/cloud" + "github.com/elastic/cloudbeat/internal/inventory" + "github.com/elastic/cloudbeat/internal/resources/providers/awslib" + "github.com/elastic/cloudbeat/internal/resources/providers/awslib/s3" +) + +type S3BucketFetcher struct { + logger *logp.Logger + provider s3BucketProvider +} + +var s3BucketClassification = inventory.AssetClassification{ + Category: inventory.CategoryInfrastructure, + SubCategory: inventory.SubCategoryStorage, + Type: inventory.TypeObjectStorage, + SubStype: inventory.SubTypeS3, +} + +type s3BucketProvider interface { + DescribeBuckets(ctx context.Context) ([]awslib.AwsResource, error) +} + +func NewS3BucketFetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher { + provider := s3.NewProvider(logger, cfg, &awslib.MultiRegionClientFactory[s3.Client]{}, identity.Account) + return &S3BucketFetcher{ + logger: logger, + provider: provider, + } +} + +func (s S3BucketFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) { + awsBuckets, err := s.provider.DescribeBuckets(ctx) + if err != nil { + s.logger.Errorf("Could not list s3 buckets: %v", err) + if len(awsBuckets) == 0 { + return + } + } + + buckets := lo.Map(awsBuckets, func(item awslib.AwsResource, _ int) s3.BucketDescription { + return item.(s3.BucketDescription) + }) + + for _, bucket := range buckets { + assetChannel <- inventory.NewAssetEvent( + s3BucketClassification, + bucket.GetResourceArn(), + bucket.GetResourceName(), + + inventory.WithRawAsset(bucket), + inventory.WithCloud(inventory.AssetCloud{ + Provider: inventory.AwsCloudProvider, + Region: bucket.GetRegion(), + }), + ) + } +} diff --git a/internal/inventory/aws/fetcher_s3_bucket_test.go b/internal/inventory/aws/fetcher_s3_bucket_test.go new file mode 100644 index 0000000000..0220ff8732 --- /dev/null +++ b/internal/inventory/aws/fetcher_s3_bucket_test.go @@ -0,0 +1,124 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package aws + +import ( + "context" + "testing" + "time" + + s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" + s3ctrltypes "github.com/aws/aws-sdk-go-v2/service/s3control/types" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/elastic/cloudbeat/internal/inventory" + "github.com/elastic/cloudbeat/internal/resources/providers/awslib" + "github.com/elastic/cloudbeat/internal/resources/providers/awslib/s3" + "github.com/elastic/cloudbeat/internal/resources/utils/pointers" +) + +func TestS3BucketFetcher_Fetch(t *testing.T) { + bucket1 := s3.BucketDescription{ + Name: "bucket-1", + SSEAlgorithm: nil, + BucketPolicy: nil, + BucketVersioning: &s3.BucketVersioning{ + Enabled: true, + MfaDelete: true, + }, + PublicAccessBlockConfiguration: &s3types.PublicAccessBlockConfiguration{ + BlockPublicAcls: pointers.Ref(true), + }, + AccountPublicAccessBlockConfiguration: &s3ctrltypes.PublicAccessBlockConfiguration{ + BlockPublicAcls: pointers.Ref(true), + }, + Region: "europe-west-1", + } + + bucket2 := s3.BucketDescription{ + Name: "bucket-2", + SSEAlgorithm: nil, + BucketPolicy: nil, + BucketVersioning: &s3.BucketVersioning{ + Enabled: false, + MfaDelete: false, + }, + PublicAccessBlockConfiguration: &s3types.PublicAccessBlockConfiguration{ + BlockPublicAcls: pointers.Ref(false), + }, + AccountPublicAccessBlockConfiguration: &s3ctrltypes.PublicAccessBlockConfiguration{ + BlockPublicAcls: pointers.Ref(false), + }, + Region: "europe-west-1", + } + in := []awslib.AwsResource{bucket1, bucket2} + + expected := []inventory.AssetEvent{ + inventory.NewAssetEvent( + s3BucketClassification, + "arn:aws:s3:::bucket-1", + "bucket-1", + inventory.WithRawAsset(bucket1), + inventory.WithCloud(inventory.AssetCloud{ + Provider: inventory.AwsCloudProvider, + Region: "europe-west-1", + }), + ), + inventory.NewAssetEvent( + s3BucketClassification, + "arn:aws:s3:::bucket-2", + "bucket-2", + inventory.WithRawAsset(bucket2), + inventory.WithCloud(inventory.AssetCloud{ + Provider: inventory.AwsCloudProvider, + Region: "europe-west-1", + }), + ), + } + + logger := logp.NewLogger("test_fetcher_s3_bucket") + provider := newMockS3BucketProvider(t) + provider.EXPECT().DescribeBuckets(mock.Anything).Return(in, nil) + + fetcher := S3BucketFetcher{ + logger: logger, + provider: provider, + } + + ch := make(chan inventory.AssetEvent) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + go func() { + fetcher.Fetch(ctx, ch) + }() + + received := make([]inventory.AssetEvent, 0, len(expected)) + for len(expected) != len(received) { + select { + case <-ctx.Done(): + assert.ElementsMatch(t, expected, received) + return + case event := <-ch: + received = append(received, event) + } + } + + assert.ElementsMatch(t, expected, received) +} diff --git a/internal/inventory/aws/fetchers.go b/internal/inventory/aws/fetchers.go index 6e2b561033..2643296e71 100644 --- a/internal/inventory/aws/fetchers.go +++ b/internal/inventory/aws/fetchers.go @@ -28,5 +28,6 @@ import ( func Fetchers(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) []inventory.AssetFetcher { return []inventory.AssetFetcher{ newEc2Fetcher(logger, identity, cfg), + NewS3BucketFetcher(logger, identity, cfg), } } diff --git a/internal/inventory/aws/mock_instances_provider.go b/internal/inventory/aws/mock_ec2_instances_provider.go similarity index 55% rename from internal/inventory/aws/mock_instances_provider.go rename to internal/inventory/aws/mock_ec2_instances_provider.go index 8344e5c6b0..1b6629da9c 100644 --- a/internal/inventory/aws/mock_instances_provider.go +++ b/internal/inventory/aws/mock_ec2_instances_provider.go @@ -26,21 +26,21 @@ import ( mock "github.com/stretchr/testify/mock" ) -// mockInstancesProvider is an autogenerated mock type for the instancesProvider type -type mockInstancesProvider struct { +// mockEc2InstancesProvider is an autogenerated mock type for the ec2InstancesProvider type +type mockEc2InstancesProvider struct { mock.Mock } -type mockInstancesProvider_Expecter struct { +type mockEc2InstancesProvider_Expecter struct { mock *mock.Mock } -func (_m *mockInstancesProvider) EXPECT() *mockInstancesProvider_Expecter { - return &mockInstancesProvider_Expecter{mock: &_m.Mock} +func (_m *mockEc2InstancesProvider) EXPECT() *mockEc2InstancesProvider_Expecter { + return &mockEc2InstancesProvider_Expecter{mock: &_m.Mock} } // DescribeInstances provides a mock function with given fields: ctx -func (_m *mockInstancesProvider) DescribeInstances(ctx context.Context) ([]*ec2.Ec2Instance, error) { +func (_m *mockEc2InstancesProvider) DescribeInstances(ctx context.Context) ([]*ec2.Ec2Instance, error) { ret := _m.Called(ctx) var r0 []*ec2.Ec2Instance @@ -65,41 +65,41 @@ func (_m *mockInstancesProvider) DescribeInstances(ctx context.Context) ([]*ec2. return r0, r1 } -// mockInstancesProvider_DescribeInstances_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DescribeInstances' -type mockInstancesProvider_DescribeInstances_Call struct { +// mockEc2InstancesProvider_DescribeInstances_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DescribeInstances' +type mockEc2InstancesProvider_DescribeInstances_Call struct { *mock.Call } // DescribeInstances is a helper method to define mock.On call // - ctx context.Context -func (_e *mockInstancesProvider_Expecter) DescribeInstances(ctx interface{}) *mockInstancesProvider_DescribeInstances_Call { - return &mockInstancesProvider_DescribeInstances_Call{Call: _e.mock.On("DescribeInstances", ctx)} +func (_e *mockEc2InstancesProvider_Expecter) DescribeInstances(ctx interface{}) *mockEc2InstancesProvider_DescribeInstances_Call { + return &mockEc2InstancesProvider_DescribeInstances_Call{Call: _e.mock.On("DescribeInstances", ctx)} } -func (_c *mockInstancesProvider_DescribeInstances_Call) Run(run func(ctx context.Context)) *mockInstancesProvider_DescribeInstances_Call { +func (_c *mockEc2InstancesProvider_DescribeInstances_Call) Run(run func(ctx context.Context)) *mockEc2InstancesProvider_DescribeInstances_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context)) }) return _c } -func (_c *mockInstancesProvider_DescribeInstances_Call) Return(_a0 []*ec2.Ec2Instance, _a1 error) *mockInstancesProvider_DescribeInstances_Call { +func (_c *mockEc2InstancesProvider_DescribeInstances_Call) Return(_a0 []*ec2.Ec2Instance, _a1 error) *mockEc2InstancesProvider_DescribeInstances_Call { _c.Call.Return(_a0, _a1) return _c } -func (_c *mockInstancesProvider_DescribeInstances_Call) RunAndReturn(run func(context.Context) ([]*ec2.Ec2Instance, error)) *mockInstancesProvider_DescribeInstances_Call { +func (_c *mockEc2InstancesProvider_DescribeInstances_Call) RunAndReturn(run func(context.Context) ([]*ec2.Ec2Instance, error)) *mockEc2InstancesProvider_DescribeInstances_Call { _c.Call.Return(run) return _c } -// newMockInstancesProvider creates a new instance of mockInstancesProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// newMockEc2InstancesProvider creates a new instance of mockEc2InstancesProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. -func newMockInstancesProvider(t interface { +func newMockEc2InstancesProvider(t interface { mock.TestingT Cleanup(func()) -}) *mockInstancesProvider { - mock := &mockInstancesProvider{} +}) *mockEc2InstancesProvider { + mock := &mockEc2InstancesProvider{} mock.Mock.Test(t) t.Cleanup(func() { mock.AssertExpectations(t) }) diff --git a/internal/inventory/aws/mock_s3_bucket_provider.go b/internal/inventory/aws/mock_s3_bucket_provider.go new file mode 100644 index 0000000000..1c9d19e3e0 --- /dev/null +++ b/internal/inventory/aws/mock_s3_bucket_provider.go @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mockery v2.37.1. DO NOT EDIT. + +package aws + +import ( + context "context" + + awslib "github.com/elastic/cloudbeat/internal/resources/providers/awslib" + + mock "github.com/stretchr/testify/mock" +) + +// mockS3BucketProvider is an autogenerated mock type for the s3BucketProvider type +type mockS3BucketProvider struct { + mock.Mock +} + +type mockS3BucketProvider_Expecter struct { + mock *mock.Mock +} + +func (_m *mockS3BucketProvider) EXPECT() *mockS3BucketProvider_Expecter { + return &mockS3BucketProvider_Expecter{mock: &_m.Mock} +} + +// DescribeBuckets provides a mock function with given fields: ctx +func (_m *mockS3BucketProvider) DescribeBuckets(ctx context.Context) ([]awslib.AwsResource, error) { + ret := _m.Called(ctx) + + var r0 []awslib.AwsResource + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) ([]awslib.AwsResource, error)); ok { + return rf(ctx) + } + if rf, ok := ret.Get(0).(func(context.Context) []awslib.AwsResource); ok { + r0 = rf(ctx) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]awslib.AwsResource) + } + } + + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(ctx) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// mockS3BucketProvider_DescribeBuckets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DescribeBuckets' +type mockS3BucketProvider_DescribeBuckets_Call struct { + *mock.Call +} + +// DescribeBuckets is a helper method to define mock.On call +// - ctx context.Context +func (_e *mockS3BucketProvider_Expecter) DescribeBuckets(ctx interface{}) *mockS3BucketProvider_DescribeBuckets_Call { + return &mockS3BucketProvider_DescribeBuckets_Call{Call: _e.mock.On("DescribeBuckets", ctx)} +} + +func (_c *mockS3BucketProvider_DescribeBuckets_Call) Run(run func(ctx context.Context)) *mockS3BucketProvider_DescribeBuckets_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *mockS3BucketProvider_DescribeBuckets_Call) Return(_a0 []awslib.AwsResource, _a1 error) *mockS3BucketProvider_DescribeBuckets_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *mockS3BucketProvider_DescribeBuckets_Call) RunAndReturn(run func(context.Context) ([]awslib.AwsResource, error)) *mockS3BucketProvider_DescribeBuckets_Call { + _c.Call.Return(run) + return _c +} + +// newMockS3BucketProvider creates a new instance of mockS3BucketProvider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockS3BucketProvider(t interface { + mock.TestingT + Cleanup(func()) +}) *mockS3BucketProvider { + mock := &mockS3BucketProvider{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/internal/resources/providers/awslib/s3/provider.go b/internal/resources/providers/awslib/s3/provider.go index 50d1240aef..67f64c0f82 100644 --- a/internal/resources/providers/awslib/s3/provider.go +++ b/internal/resources/providers/awslib/s3/provider.go @@ -108,7 +108,7 @@ func (p Provider) DescribeBuckets(ctx context.Context) ([]awslib.AwsResource, er BucketVersioning: bucketVersioning, PublicAccessBlockConfiguration: publicAccessBlockConfiguration, AccountPublicAccessBlockConfiguration: accountPublicAccessBlockConfig, - region: region, + Region: region, }) } } @@ -184,7 +184,7 @@ func (p Provider) getBucketsRegionMapping(ctx context.Context, buckets []types.B bucketsRegionMap := make(map[string][]types.Bucket, 0) for _, clientBucket := range buckets { region, regionErr := p.getBucketRegion(ctx, clientBucket.Name) - // If we could not get the region for a bucket, additional API calls for resources will probably fail, we should + // If we could not get the Region for a bucket, additional API calls for resources will probably fail, we should // not describe this bucket. if regionErr != nil { p.log.Errorf("Could not get bucket location for bucket %s. Not describing this bucket. Error: %v", *clientBucket.Name, regionErr) @@ -323,5 +323,5 @@ func (b BucketDescription) GetResourceType() string { } func (b BucketDescription) GetRegion() string { - return b.region + return b.Region } diff --git a/internal/resources/providers/awslib/s3/provider_test.go b/internal/resources/providers/awslib/s3/provider_test.go index 7bb850a2f3..0624ff3b9c 100644 --- a/internal/resources/providers/awslib/s3/provider_test.go +++ b/internal/resources/providers/awslib/s3/provider_test.go @@ -88,7 +88,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { regions: []string{awslib.DefaultRegion}, }, { - name: "Should not return any S3 buckets when the region can not be fetched", + name: "Should not return any S3 buckets when the Region can not be fetched", s3ClientMockReturnVals: s3ClientMockReturnVals{ "ListBuckets": {{{mock.Anything, mock.Anything}, {&s3Client.ListBucketsOutput{Buckets: []types.Bucket{{Name: &bucketName}}}, nil}}}, "GetBucketEncryption": {{{mock.Anything, mock.Anything}, {nil, errors.New("bla")}}}, @@ -120,7 +120,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { BucketVersioning: nil, PublicAccessBlockConfiguration: nil, AccountPublicAccessBlockConfiguration: nil, - region: awslib.DefaultRegion, + Region: awslib.DefaultRegion, }}, expectError: false, regions: []string{awslib.DefaultRegion}, @@ -139,7 +139,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { BucketVersioning: nil, PublicAccessBlockConfiguration: nil, AccountPublicAccessBlockConfiguration: nil, - region: string(region), + Region: string(region), }}, expectError: false, regions: []string{awslib.DefaultRegion}, @@ -168,7 +168,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { BucketVersioning: nil, PublicAccessBlockConfiguration: nil, AccountPublicAccessBlockConfiguration: nil, - region: string(region), + Region: string(region), }}, expectError: false, regions: []string{awslib.DefaultRegion, string(region)}, @@ -191,7 +191,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { BucketVersioning: nil, PublicAccessBlockConfiguration: nil, AccountPublicAccessBlockConfiguration: nil, - region: string(region), + Region: string(region), }}, expectError: false, regions: []string{awslib.DefaultRegion, string(region)}, @@ -214,7 +214,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { BucketVersioning: &BucketVersioning{true, true}, PublicAccessBlockConfiguration: nil, AccountPublicAccessBlockConfiguration: nil, - region: string(region), + Region: string(region), }}, expectError: false, regions: []string{awslib.DefaultRegion, string(region)}, @@ -249,7 +249,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { RestrictPublicBuckets: aws.Bool(false), }, AccountPublicAccessBlockConfiguration: nil, - region: string(region), + Region: string(region), }}, expectError: false, regions: []string{awslib.DefaultRegion, string(region)}, @@ -284,7 +284,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { IgnorePublicAcls: aws.Bool(false), RestrictPublicBuckets: aws.Bool(false), }, - region: string(region), + Region: string(region), }}, expectError: false, regions: []string{awslib.DefaultRegion, string(region)}, @@ -366,7 +366,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { IgnorePublicAcls: aws.Bool(false), RestrictPublicBuckets: aws.Bool(false), }, - region: string(region), + Region: string(region), }, BucketDescription{ Name: secondBucketName, @@ -385,14 +385,14 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { IgnorePublicAcls: aws.Bool(false), RestrictPublicBuckets: aws.Bool(false), }, - region: awslib.DefaultRegion, + Region: awslib.DefaultRegion, }, }, expectError: false, regions: []string{awslib.DefaultRegion, string(region)}, }, { - name: "Should return two S3 buckets from the same region", + name: "Should return two S3 buckets from the same Region", s3ClientMockReturnVals: s3ClientMockReturnVals{ "ListBuckets": {{{mock.Anything, mock.Anything}, {&s3Client.ListBucketsOutput{Buckets: []types.Bucket{{Name: &bucketName}, {Name: &secondBucketName}}}, nil}}}, "GetBucketEncryption": { @@ -468,7 +468,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { IgnorePublicAcls: aws.Bool(false), RestrictPublicBuckets: aws.Bool(false), }, - region: awslib.DefaultRegion, + Region: awslib.DefaultRegion, }, BucketDescription{ Name: secondBucketName, @@ -487,7 +487,7 @@ func (s *ProviderTestSuite) TestProvider_DescribeBuckets() { IgnorePublicAcls: aws.Bool(false), RestrictPublicBuckets: aws.Bool(false), }, - region: awslib.DefaultRegion, + Region: awslib.DefaultRegion, }, }, expectError: false, diff --git a/internal/resources/providers/awslib/s3/s3.go b/internal/resources/providers/awslib/s3/s3.go index 034355872a..c725ffdfa3 100644 --- a/internal/resources/providers/awslib/s3/s3.go +++ b/internal/resources/providers/awslib/s3/s3.go @@ -36,7 +36,7 @@ type BucketDescription struct { BucketVersioning *BucketVersioning `json:"bucket_versioning,omitempty"` PublicAccessBlockConfiguration *types.PublicAccessBlockConfiguration `json:"public_access_block_configuration"` AccountPublicAccessBlockConfiguration *s3ContorlTypes.PublicAccessBlockConfiguration `json:"account_public_access_block_configuration"` - region string + Region string } // TODO: This can be better typed, but this is a complex object. See this library for example: https://github.com/liamg/iamgo/ From c84c31d641ee8be192a726050ba196596596be02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Farias?= Date: Tue, 19 Mar 2024 14:10:54 +0100 Subject: [PATCH 2/3] Enrich cloud field --- internal/inventory/asset.go | 37 +++++++++-- .../inventory/aws/fetcher_ec2_instance.go | 62 ++++++++++++++----- .../aws/fetcher_ec2_instance_test.go | 42 +++++++++++-- internal/inventory/aws/fetcher_s3_bucket.go | 21 +++++-- .../inventory/aws/fetcher_s3_bucket_test.go | 20 +++++- internal/inventory/inventory.go | 4 +- 6 files changed, 152 insertions(+), 34 deletions(-) diff --git a/internal/inventory/asset.go b/internal/inventory/asset.go index f6f3a19c27..ec467ebd86 100644 --- a/internal/inventory/asset.go +++ b/internal/inventory/asset.go @@ -54,10 +54,8 @@ const ( SubTypeS3 assetSubType = "s3" ) -type assetCloudProvider string - const ( - AwsCloudProvider assetCloudProvider = "aws" + AwsCloudProvider = "aws" ) // AssetEvent holds the whole asset @@ -100,8 +98,37 @@ type AssetNetwork struct { // AssetCloud contains information about the cloud provider type AssetCloud struct { - Provider assetCloudProvider `json:"provider"` - Region string `json:"region"` + AvailabilityZone *string `json:"availability_zone,omitempty"` + Provider string `json:"provider,omitempty"` + Region string `json:"region,omitempty"` + Account AssetCloudAccount `json:"account"` + Instance *AssetCloudInstance `json:"instance,omitempty"` + Machine *AssetCloudMachine `json:"machine,omitempty"` + Project *AssetCloudProject `json:"project,omitempty"` + Service *AssetCloudService `json:"service,omitempty"` +} + +type AssetCloudAccount struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type AssetCloudInstance struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type AssetCloudMachine struct { + MachineType string `json:"machineType,omitempty"` +} + +type AssetCloudProject struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` +} + +type AssetCloudService struct { + Name string `json:"name,omitempty"` } // AssetHost contains information of the asset in case it is a host diff --git a/internal/inventory/aws/fetcher_ec2_instance.go b/internal/inventory/aws/fetcher_ec2_instance.go index 5f13480886..1b72c8e785 100644 --- a/internal/inventory/aws/fetcher_ec2_instance.go +++ b/internal/inventory/aws/fetcher_ec2_instance.go @@ -31,8 +31,10 @@ import ( ) type Ec2InstanceFetcher struct { - logger *logp.Logger - provider ec2InstancesProvider + logger *logp.Logger + provider ec2InstancesProvider + AccountId string + AccountName string } type ec2InstancesProvider interface { @@ -49,8 +51,10 @@ var ec2InstanceClassification = inventory.AssetClassification{ func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher { provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{}) return &Ec2InstanceFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: identity.Account, + AccountName: identity.AccountAlias, } } @@ -74,25 +78,31 @@ func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve }) } - tags := make(map[string]string, len(instance.Tags)) - for _, t := range instance.Tags { - if t.Key == nil { - continue - } - - tags[*t.Key] = pointers.Deref(t.Value) - } - assetChannel <- inventory.NewAssetEvent( ec2InstanceClassification, instance.GetResourceArn(), instance.GetResourceName(), inventory.WithRawAsset(instance), - inventory.WithTags(tags), + inventory.WithTags(e.getTags(instance)), inventory.WithCloud(inventory.AssetCloud{ - Provider: inventory.AwsCloudProvider, - Region: instance.Region, + Provider: inventory.AwsCloudProvider, + Region: instance.Region, + AvailabilityZone: e.getAvailabilityZone(instance), + Account: inventory.AssetCloudAccount{ + Id: e.AccountId, + Name: e.AccountName, + }, + Instance: &inventory.AssetCloudInstance{ + Id: pointers.Deref(instance.InstanceId), + Name: instance.GetResourceName(), + }, + Machine: &inventory.AssetCloudMachine{ + MachineType: string(instance.InstanceType), + }, + Service: &inventory.AssetCloudService{ + Name: "AWS EC2", + }, }), inventory.WithHost(inventory.AssetHost{ Architecture: string(instance.Architecture), @@ -114,3 +124,23 @@ func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve ) } } + +func (e *Ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]string { + tags := make(map[string]string, len(instance.Tags)) + for _, t := range instance.Tags { + if t.Key == nil { + continue + } + + tags[*t.Key] = pointers.Deref(t.Value) + } + return tags +} + +func (e *Ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string { + if instance.Placement == nil { + return nil + } + + return instance.Placement.AvailabilityZone +} diff --git a/internal/inventory/aws/fetcher_ec2_instance_test.go b/internal/inventory/aws/fetcher_ec2_instance_test.go index 7211508f1d..4d5f0c3a63 100644 --- a/internal/inventory/aws/fetcher_ec2_instance_test.go +++ b/internal/inventory/aws/fetcher_ec2_instance_test.go @@ -62,6 +62,9 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { PrivateIpAddress: pointers.Ref("private-ip-addre"), PublicDnsName: pointers.Ref("public-dns"), PrivateDnsName: pointers.Ref("private-dns"), + Placement: &types.Placement{ + AvailabilityZone: pointers.Ref("1a"), + }, }, Region: "us-east", } @@ -81,8 +84,23 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { inventory.WithRawAsset(instance1), inventory.WithTags(map[string]string{"Name": "test-server", "key": "value"}), inventory.WithCloud(inventory.AssetCloud{ - Provider: inventory.AwsCloudProvider, - Region: "us-east", + Provider: inventory.AwsCloudProvider, + Region: "us-east", + AvailabilityZone: pointers.Ref("1a"), + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Instance: &inventory.AssetCloudInstance{ + Id: "234567890", + Name: "test-server", + }, + Machine: &inventory.AssetCloudMachine{ + MachineType: "instance-type", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS EC2", + }, }), inventory.WithHost(inventory.AssetHost{ Architecture: string(types.ArchitectureValuesX8664), @@ -115,6 +133,20 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, Region: "us-east", + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Instance: &inventory.AssetCloudInstance{ + Id: "", + Name: "", + }, + Machine: &inventory.AssetCloudMachine{ + MachineType: "", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS EC2", + }, }), inventory.WithHost(inventory.AssetHost{}), inventory.WithNetwork(inventory.AssetNetwork{}), @@ -126,8 +158,10 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) { provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil) fetcher := Ec2InstanceFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: "123", + AccountName: "alias", } ch := make(chan inventory.AssetEvent) diff --git a/internal/inventory/aws/fetcher_s3_bucket.go b/internal/inventory/aws/fetcher_s3_bucket.go index 3be6990f57..1cf272534b 100644 --- a/internal/inventory/aws/fetcher_s3_bucket.go +++ b/internal/inventory/aws/fetcher_s3_bucket.go @@ -31,8 +31,10 @@ import ( ) type S3BucketFetcher struct { - logger *logp.Logger - provider s3BucketProvider + logger *logp.Logger + provider s3BucketProvider + AccountId string + AccountName string } var s3BucketClassification = inventory.AssetClassification{ @@ -49,8 +51,10 @@ type s3BucketProvider interface { func NewS3BucketFetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher { provider := s3.NewProvider(logger, cfg, &awslib.MultiRegionClientFactory[s3.Client]{}, identity.Account) return &S3BucketFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: identity.Account, + AccountName: identity.AccountAlias, } } @@ -76,7 +80,14 @@ func (s S3BucketFetcher) Fetch(ctx context.Context, assetChannel chan<- inventor inventory.WithRawAsset(bucket), inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, - Region: bucket.GetRegion(), + Region: bucket.Region, + Account: inventory.AssetCloudAccount{ + Id: s.AccountId, + Name: s.AccountName, + }, + Service: &inventory.AssetCloudService{ + Name: "AWS S3", + }, }), ) } diff --git a/internal/inventory/aws/fetcher_s3_bucket_test.go b/internal/inventory/aws/fetcher_s3_bucket_test.go index 0220ff8732..bdfd7ebaad 100644 --- a/internal/inventory/aws/fetcher_s3_bucket_test.go +++ b/internal/inventory/aws/fetcher_s3_bucket_test.go @@ -79,6 +79,13 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, Region: "europe-west-1", + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS S3", + }, }), ), inventory.NewAssetEvent( @@ -89,6 +96,13 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { inventory.WithCloud(inventory.AssetCloud{ Provider: inventory.AwsCloudProvider, Region: "europe-west-1", + Account: inventory.AssetCloudAccount{ + Id: "123", + Name: "alias", + }, + Service: &inventory.AssetCloudService{ + Name: "AWS S3", + }, }), ), } @@ -98,8 +112,10 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { provider.EXPECT().DescribeBuckets(mock.Anything).Return(in, nil) fetcher := S3BucketFetcher{ - logger: logger, - provider: provider, + logger: logger, + provider: provider, + AccountId: "123", + AccountName: "alias", } ch := make(chan inventory.AssetEvent) diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 6ed8742a0d..574bd06cb6 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -54,8 +54,8 @@ func NewAssetInventory(logger *logp.Logger, fetchers []AssetFetcher, publisher A fetchers: fetchers, publisher: publisher, // move to a configuration parameter - bufferFlushInterval: 15 * time.Second, - bufferMaxSize: 100, + bufferFlushInterval: 10 * time.Second, + bufferMaxSize: 1600, assetCh: make(chan AssetEvent), now: now, } From ec9cd70af17d062e6b3fc18dbdc1d937c80aeb28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Farias?= Date: Wed, 20 Mar 2024 15:12:17 +0100 Subject: [PATCH 3/3] Convert bucket policy to an indexable format --- internal/inventory/asset.go | 56 +++++++---- .../inventory/aws/fetcher_ec2_instance.go | 2 +- internal/inventory/aws/fetcher_s3_bucket.go | 96 ++++++++++++++++++- .../inventory/aws/fetcher_s3_bucket_test.go | 62 +++++++++++- internal/inventory/inventory.go | 13 +-- internal/inventory/inventory_test.go | 22 ++++- 6 files changed, 221 insertions(+), 30 deletions(-) diff --git a/internal/inventory/asset.go b/internal/inventory/asset.go index ec467ebd86..8d322728c8 100644 --- a/internal/inventory/asset.go +++ b/internal/inventory/asset.go @@ -60,19 +60,20 @@ const ( // AssetEvent holds the whole asset type AssetEvent struct { - Asset Asset - Network *AssetNetwork - Cloud *AssetCloud - Host *AssetHost - IAM *AssetIAM + Asset Asset + Network *AssetNetwork + Cloud *AssetCloud + Host *AssetHost + IAM *AssetIAM + ResourcePolicies []AssetResourcePolicy } // AssetClassification holds the taxonomy of an asset type AssetClassification struct { Category assetCategory `json:"category"` - SubCategory assetSubCategory `json:"subCategory"` + SubCategory assetSubCategory `json:"sub_category"` Type assetType `json:"type"` - SubStype assetSubType `json:"subStype"` + SubType assetSubType `json:"sub_type"` } // Asset contains the identifiers of the asset @@ -87,13 +88,13 @@ type Asset struct { // AssetNetwork contains network information type AssetNetwork struct { - NetworkId *string `json:"networkId"` - SubnetId *string `json:"subnetId"` - Ipv6Address *string `json:"ipv6Address"` - PublicIpAddress *string `json:"publicIpAddress"` - PrivateIpAddress *string `json:"privateIpAddress"` - PublicDnsName *string `json:"publicDnsName"` - PrivateDnsName *string `json:"privateDnsName"` + NetworkId *string `json:"network_id"` + SubnetId *string `json:"subnet_id"` + Ipv6Address *string `json:"ipv6_address"` + PublicIpAddress *string `json:"public_ip_address"` + PrivateIpAddress *string `json:"private_ip_address"` + PublicDnsName *string `json:"public_dns_name"` + PrivateDnsName *string `json:"private_dns_name"` } // AssetCloud contains information about the cloud provider @@ -119,7 +120,7 @@ type AssetCloudInstance struct { } type AssetCloudMachine struct { - MachineType string `json:"machineType,omitempty"` + MachineType string `json:"machine_type,omitempty"` } type AssetCloudProject struct { @@ -135,9 +136,9 @@ type AssetCloudService struct { type AssetHost struct { Architecture string `json:"architecture"` ImageId *string `json:"imageId"` - InstanceType string `json:"instanceType"` + InstanceType string `json:"instance_type"` Platform string `json:"platform"` - PlatformDetails *string `json:"platformDetails"` + PlatformDetails *string `json:"platform_details"` } type AssetIAM struct { @@ -145,6 +146,19 @@ type AssetIAM struct { Arn *string `json:"arn"` } +// AssetResourcePolicy maps security policies applied directly on resources +type AssetResourcePolicy struct { + Version *string `json:"version,omitempty"` + Id *string `json:"id,omitempty"` + Effect string `json:"effect,omitempty"` + Principal map[string]any `json:"principal,omitempty"` + Action []string `json:"action,omitempty"` + NotAction []string `json:"notAction,omitempty"` + Resource []string `json:"resource,omitempty"` + NoResource []string `json:"noResource,omitempty"` + Condition map[string]any `json:"condition,omitempty"` +} + // AssetEnricher functional builder function type AssetEnricher func(asset *AssetEvent) @@ -201,13 +215,19 @@ func WithIAM(iam AssetIAM) AssetEnricher { } } +func WithResourcePolicies(policies ...AssetResourcePolicy) AssetEnricher { + return func(a *AssetEvent) { + a.ResourcePolicies = policies + } +} + func EmptyEnricher() AssetEnricher { return func(_ *AssetEvent) {} } func generateUniqueId(c AssetClassification, resourceId string) string { hasher := sha256.New() - toBeHashed := fmt.Sprintf("%s-%s-%s-%s-%s", resourceId, c.Category, c.SubCategory, c.Type, c.SubStype) + toBeHashed := fmt.Sprintf("%s-%s-%s-%s-%s", resourceId, c.Category, c.SubCategory, c.Type, c.SubType) hasher.Write([]byte(toBeHashed)) //nolint:revive hash := hasher.Sum(nil) encoded := base64.StdEncoding.EncodeToString(hash) diff --git a/internal/inventory/aws/fetcher_ec2_instance.go b/internal/inventory/aws/fetcher_ec2_instance.go index 1b72c8e785..983f2fa14a 100644 --- a/internal/inventory/aws/fetcher_ec2_instance.go +++ b/internal/inventory/aws/fetcher_ec2_instance.go @@ -45,7 +45,7 @@ var ec2InstanceClassification = inventory.AssetClassification{ Category: inventory.CategoryInfrastructure, SubCategory: inventory.SubCategoryCompute, Type: inventory.TypeVirtualMachine, - SubStype: inventory.SubTypeEC2, + SubType: inventory.SubTypeEC2, } func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher { diff --git a/internal/inventory/aws/fetcher_s3_bucket.go b/internal/inventory/aws/fetcher_s3_bucket.go index 1cf272534b..98c0b2196d 100644 --- a/internal/inventory/aws/fetcher_s3_bucket.go +++ b/internal/inventory/aws/fetcher_s3_bucket.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/cloudbeat/internal/inventory" "github.com/elastic/cloudbeat/internal/resources/providers/awslib" "github.com/elastic/cloudbeat/internal/resources/providers/awslib/s3" + "github.com/elastic/cloudbeat/internal/resources/utils/pointers" ) type S3BucketFetcher struct { @@ -41,7 +42,7 @@ var s3BucketClassification = inventory.AssetClassification{ Category: inventory.CategoryInfrastructure, SubCategory: inventory.SubCategoryStorage, Type: inventory.TypeObjectStorage, - SubStype: inventory.SubTypeS3, + SubType: inventory.SubTypeS3, } type s3BucketProvider interface { @@ -89,6 +90,99 @@ func (s S3BucketFetcher) Fetch(ctx context.Context, assetChannel chan<- inventor Name: "AWS S3", }, }), + inventory.WithResourcePolicies(getBucketPolicies(bucket)...), ) } } + +func getBucketPolicies(bucket s3.BucketDescription) []inventory.AssetResourcePolicy { + if len(bucket.BucketPolicy) == 0 { + return nil + } + + version, hasVersion := bucket.BucketPolicy["Version"].(string) + if !hasVersion { + version = "" + } + + switch statements := bucket.BucketPolicy["Statement"].(type) { + case []map[string]any: + return convertStatements(statements, version) + case []any: + return convertAnyStatements(statements, version) + case map[string]any: + return []inventory.AssetResourcePolicy{convertStatement(statements, &version)} + } + return nil +} + +func convertAnyStatements(statements []any, version string) []inventory.AssetResourcePolicy { + policies := make([]inventory.AssetResourcePolicy, 0, len(statements)) + for _, statement := range statements { + policies = append(policies, convertStatement(statement.(map[string]any), &version)) + } + return policies +} + +func convertStatements(statements []map[string]any, version string) []inventory.AssetResourcePolicy { + policies := make([]inventory.AssetResourcePolicy, 0, len(statements)) + for _, statement := range statements { + policies = append(policies, convertStatement(statement, &version)) + } + return policies +} + +func convertStatement(statement map[string]any, version *string) inventory.AssetResourcePolicy { + p := inventory.AssetResourcePolicy{} + p.Version = version + + if sid, ok := statement["Sid"]; ok { + p.Id = pointers.Ref(sid.(string)) + } + + if effect, ok := statement["Effect"]; ok { + p.Effect = effect.(string) + } + + if anyPrincipal, ok := statement["Principal"]; ok { + switch principal := anyPrincipal.(type) { + case string: + p.Principal = map[string]any{principal: principal} + case map[string]any: + p.Principal = principal + } + } + + if action, ok := statement["Action"]; ok { + p.Action = anyToSliceString(action) + } + + if notAction, ok := statement["NotAction"]; ok { + p.NotAction = anyToSliceString(notAction) + } + + if resource, ok := statement["Resource"]; ok { + p.Resource = anyToSliceString(resource) + } + + if noResource, ok := statement["NoResource"]; ok { + p.NoResource = anyToSliceString(noResource) + } + + if condition, ok := statement["Condition"]; ok { + p.Condition = condition.(map[string]any) + } + + return p +} + +func anyToSliceString(anyString any) []string { + switch s := anyString.(type) { + case string: + return []string{s} + case []string: + return s + } + + return nil +} diff --git a/internal/inventory/aws/fetcher_s3_bucket_test.go b/internal/inventory/aws/fetcher_s3_bucket_test.go index bdfd7ebaad..5b22e57f94 100644 --- a/internal/inventory/aws/fetcher_s3_bucket_test.go +++ b/internal/inventory/aws/fetcher_s3_bucket_test.go @@ -38,7 +38,30 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { bucket1 := s3.BucketDescription{ Name: "bucket-1", SSEAlgorithm: nil, - BucketPolicy: nil, + BucketPolicy: map[string]any{ + "Version": "2012-10-17", + "Statement": []map[string]any{ + { + "Sid": "Test 1", + "Effect": "Allow", + "Principal": map[string]any{ + "AWS": "dima", + "service": "aws.com", + }, + "Action": []string{"read", "update", "delete"}, + "Resource": []string{"s3/bucket", "s3/bucket/*"}, + }, + { + "Sid": "Test 2", + "Effect": "Deny", + "Principal": map[string]any{ + "AWS": "romulo", + }, + "Action": []string{"delete"}, + "Resource": []string{"s3/bucket"}, + }, + }, + }, BucketVersioning: &s3.BucketVersioning{ Enabled: true, MfaDelete: true, @@ -55,7 +78,16 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { bucket2 := s3.BucketDescription{ Name: "bucket-2", SSEAlgorithm: nil, - BucketPolicy: nil, + BucketPolicy: map[string]any{ + "Version": "2012-10-17", + "Statement": map[string]any{ + "Sid": "Test 1", + "Effect": "Allow", + "Principal": "*", + "Action": "read", + "Resource": "s3/bucket", + }, + }, BucketVersioning: &s3.BucketVersioning{ Enabled: false, MfaDelete: false, @@ -87,6 +119,24 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { Name: "AWS S3", }, }), + inventory.WithResourcePolicies(inventory.AssetResourcePolicy{ + Version: pointers.Ref("2012-10-17"), + Id: pointers.Ref("Test 1"), + Effect: "Allow", + Principal: map[string]any{ + "AWS": "dima", + "service": "aws.com", + }, + Action: []string{"read", "update", "delete"}, + Resource: []string{"s3/bucket", "s3/bucket/*"}, + }, inventory.AssetResourcePolicy{ + Version: pointers.Ref("2012-10-17"), + Id: pointers.Ref("Test 2"), + Effect: "Deny", + Principal: map[string]any{"AWS": "romulo"}, + Action: []string{"delete"}, + Resource: []string{"s3/bucket"}, + }), ), inventory.NewAssetEvent( s3BucketClassification, @@ -104,6 +154,14 @@ func TestS3BucketFetcher_Fetch(t *testing.T) { Name: "AWS S3", }, }), + inventory.WithResourcePolicies(inventory.AssetResourcePolicy{ + Version: pointers.Ref("2012-10-17"), + Id: pointers.Ref("Test 1"), + Effect: "Allow", + Principal: map[string]any{"*": "*"}, + Action: []string{"read"}, + Resource: []string{"s3/bucket"}, + }), ), } diff --git a/internal/inventory/inventory.go b/internal/inventory/inventory.go index 574bd06cb6..721c8e80e8 100644 --- a/internal/inventory/inventory.go +++ b/internal/inventory/inventory.go @@ -105,11 +105,12 @@ func (a *AssetInventory) publish(assets []AssetEvent) { Meta: mapstr.M{libevents.FieldMetaIndex: generateIndex(e.Asset)}, Timestamp: a.now(), Fields: mapstr.M{ - "asset": e.Asset, - "cloud": e.Cloud, - "host": e.Host, - "network": e.Network, - "iam": e.IAM, + "asset": e.Asset, + "cloud": e.Cloud, + "host": e.Host, + "network": e.Network, + "iam": e.IAM, + "resource_policies": e.ResourcePolicies, }, } }) @@ -118,7 +119,7 @@ func (a *AssetInventory) publish(assets []AssetEvent) { } func generateIndex(a Asset) string { - return fmt.Sprintf("asset_inventory_%s_%s_%s_%s", a.Category, a.SubCategory, a.Type, a.SubStype) + return fmt.Sprintf("asset_inventory_%s_%s_%s_%s", a.Category, a.SubCategory, a.Type, a.SubType) } func (a *AssetInventory) Stop() { diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index 1d5ea6ddf4..70fe6f2fa2 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -48,7 +48,7 @@ func TestAssetInventory_Run(t *testing.T) { Category: CategoryInfrastructure, SubCategory: SubCategoryCompute, Type: TypeVirtualMachine, - SubStype: SubTypeEC2, + SubType: SubTypeEC2, }, Tags: map[string]string{"Name": "test-server", "key": "value"}, }, @@ -76,6 +76,16 @@ func TestAssetInventory_Run(t *testing.T) { Id: pointers.Ref("a123123"), Arn: pointers.Ref("123123:123123:123123"), }, + "resource_policies": []AssetResourcePolicy{ + { + Version: pointers.Ref("2012-10-17"), + Id: pointers.Ref("Test 1"), + Effect: "Allow", + Principal: map[string]any{"*": "*"}, + Action: []string{"read"}, + Resource: []string{"s3/bucket"}, + }, + }, }, }, } @@ -93,7 +103,7 @@ func TestAssetInventory_Run(t *testing.T) { Category: CategoryInfrastructure, SubCategory: SubCategoryCompute, Type: TypeVirtualMachine, - SubStype: SubTypeEC2, + SubType: SubTypeEC2, }, "arn:aws:ec2:us-east::ec2/234567890", "test-server", @@ -122,6 +132,14 @@ func TestAssetInventory_Run(t *testing.T) { PublicDnsName: pointers.Ref("public-dns"), PrivateDnsName: pointers.Ref("private-dns"), }), + WithResourcePolicies(AssetResourcePolicy{ + Version: pointers.Ref("2012-10-17"), + Id: pointers.Ref("Test 1"), + Effect: "Allow", + Principal: map[string]any{"*": "*"}, + Action: []string{"read"}, + Resource: []string{"s3/bucket"}, + }), ) })