Fetch S3 buckets

internal/inventory/asset.go

@@ -35,20 +35,23 @@ type assetSubCategory string
 
 const (
  SubCategoryCompute assetSubCategory = "compute"
+ SubCategoryStorage assetSubCategory = "storage"
 )
 
 // assetType is used to build the document index. Use only numbers, letters and dashes (-)
 type assetType string
 
 const (
  TypeVirtualMachine assetType = "virtual-machine"
+ TypeObjectStorage assetType = "object-storage"
 )
 
 // assetSubType is used to build the document index. Use only numbers, letters and dashes (-)
 type assetSubType string
 
 const (
  SubTypeEC2 assetSubType = "ec2"
+ SubTypeS3 assetSubType = "s3"
 )
 
 type assetCloudProvider string

internal/inventory/aws/fetcher_ec2.go → internal/inventory/aws/fetcher_ec2_instance.go

@@ -30,16 +30,16 @@ import (
  "github.com/elastic/cloudbeat/internal/resources/utils/pointers"
 )
 
-type Ec2Fetcher struct {
+type Ec2InstanceFetcher struct {
  logger *logp.Logger
- provider instancesProvider
+ provider ec2InstancesProvider
 }
 
-type instancesProvider interface {
+type ec2InstancesProvider interface {
  DescribeInstances(ctx context.Context) ([]*ec2.Ec2Instance, error)
 }
 
-var ec2Classification = inventory.AssetClassification{
+var ec2InstanceClassification = inventory.AssetClassification{
  Category: inventory.CategoryInfrastructure,
  SubCategory: inventory.SubCategoryCompute,
  Type: inventory.TypeVirtualMachine,
@@ -48,13 +48,13 @@ var ec2Classification = inventory.AssetClassification{
 
 func newEc2Fetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher {
  provider := ec2.NewEC2Provider(logger, identity.Account, cfg, &awslib.MultiRegionClientFactory[ec2.Client]{})
- return &Ec2Fetcher{
+ return &Ec2InstanceFetcher{
  logger: logger,
  provider: provider,
  }
 }
 
-func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
+func (e *Ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
  instances, err := e.provider.DescribeInstances(ctx)
  if err != nil {
  e.logger.Errorf("Could not list ec2 instances: %v", err)
@@ -84,7 +84,7 @@ func (e *Ec2Fetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.As
  }
 
  assetChannel <- inventory.NewAssetEvent(
- ec2Classification,
+ ec2InstanceClassification,
  instance.GetResourceArn(),
  instance.GetResourceName(),
 

internal/inventory/aws/fetcher_ec2_test.go → internal/inventory/aws/fetcher_ec2_instance_test.go

@@ -32,7 +32,7 @@ import (
  "github.com/elastic/cloudbeat/internal/resources/utils/pointers"
 )
 
-func TestFetch(t *testing.T) {
+func TestEC2InstanceFetcher_Fetch(t *testing.T) {
  instance1 := &ec2beat.Ec2Instance{
  Instance: types.Instance{
  IamInstanceProfile: &types.IamInstanceProfile{
@@ -75,7 +75,7 @@ func TestFetch(t *testing.T) {
 
  expected := []inventory.AssetEvent{
  inventory.NewAssetEvent(
- ec2Classification,
+ ec2InstanceClassification,
  "arn:aws:ec2:us-east::ec2/234567890",
  "test-server",
  inventory.WithRawAsset(instance1),
@@ -107,7 +107,7 @@ func TestFetch(t *testing.T) {
  ),
 
  inventory.NewAssetEvent(
- ec2Classification,
+ ec2InstanceClassification,
  "",
  "",
  inventory.WithRawAsset(instance2),
@@ -122,10 +122,10 @@ func TestFetch(t *testing.T) {
  }
 
  logger := logp.NewLogger("test_fetcher_ec2")
- provider := newMockInstancesProvider(t)
+ provider := newMockEc2InstancesProvider(t)
  provider.EXPECT().DescribeInstances(mock.Anything).Return(in, nil)
 
- fetcher := Ec2Fetcher{
+ fetcher := Ec2InstanceFetcher{
  logger: logger,
  provider: provider,
  }

internal/inventory/aws/fetcher_s3_bucket.go

@@ -0,0 +1,83 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package aws
+
+import (
+ "context"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ "github.com/elastic/elastic-agent-libs/logp"
+ "github.com/samber/lo"
+
+ "github.com/elastic/cloudbeat/internal/dataprovider/providers/cloud"
+ "github.com/elastic/cloudbeat/internal/inventory"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib/s3"
+)
+
+type S3BucketFetcher struct {
+ logger *logp.Logger
+ provider s3BucketProvider
+}
+
+var s3BucketClassification = inventory.AssetClassification{
+ Category: inventory.CategoryInfrastructure,
+ SubCategory: inventory.SubCategoryStorage,
+ Type: inventory.TypeObjectStorage,
+ SubStype: inventory.SubTypeS3,
+}
+
+type s3BucketProvider interface {
+ DescribeBuckets(ctx context.Context) ([]awslib.AwsResource, error)
+}
+
+func NewS3BucketFetcher(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) inventory.AssetFetcher {
+ provider := s3.NewProvider(logger, cfg, &awslib.MultiRegionClientFactory[s3.Client]{}, identity.Account)
+ return &S3BucketFetcher{
+ logger: logger,
+ provider: provider,
+ }
+}
+
+func (s S3BucketFetcher) Fetch(ctx context.Context, assetChannel chan<- inventory.AssetEvent) {
+ awsBuckets, err := s.provider.DescribeBuckets(ctx)
+ if err != nil {
+ s.logger.Errorf("Could not list s3 buckets: %v", err)
+ if len(awsBuckets) == 0 {
+ return
+ }
+ }
+
+ buckets := lo.Map(awsBuckets, func(item awslib.AwsResource, _ int) s3.BucketDescription {
+ return item.(s3.BucketDescription)
+ })
+
+ for _, bucket := range buckets {
+ assetChannel <- inventory.NewAssetEvent(
+ s3BucketClassification,
+ bucket.GetResourceArn(),
+ bucket.GetResourceName(),
+
+ inventory.WithRawAsset(bucket),
+ inventory.WithCloud(inventory.AssetCloud{
+ Provider: inventory.AwsCloudProvider,
+ Region: bucket.GetRegion(),
+ }),
+ )
+ }
+}

internal/inventory/aws/fetcher_s3_bucket_test.go

@@ -0,0 +1,124 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package aws
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
+ s3ctrltypes "github.com/aws/aws-sdk-go-v2/service/s3control/types"
+ "github.com/elastic/elastic-agent-libs/logp"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+
+ "github.com/elastic/cloudbeat/internal/inventory"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib"
+ "github.com/elastic/cloudbeat/internal/resources/providers/awslib/s3"
+ "github.com/elastic/cloudbeat/internal/resources/utils/pointers"
+)
+
+func TestS3BucketFetcher_Fetch(t *testing.T) {
+ bucket1 := s3.BucketDescription{
+ Name: "bucket-1",
+ SSEAlgorithm: nil,
+ BucketPolicy: nil,
+ BucketVersioning: &s3.BucketVersioning{
+ Enabled: true,
+ MfaDelete: true,
+ },
+ PublicAccessBlockConfiguration: &s3types.PublicAccessBlockConfiguration{
+ BlockPublicAcls: pointers.Ref(true),
+ },
+ AccountPublicAccessBlockConfiguration: &s3ctrltypes.PublicAccessBlockConfiguration{
+ BlockPublicAcls: pointers.Ref(true),
+ },
+ Region: "europe-west-1",
+ }
+
+ bucket2 := s3.BucketDescription{
+ Name: "bucket-2",
+ SSEAlgorithm: nil,
+ BucketPolicy: nil,
+ BucketVersioning: &s3.BucketVersioning{
+ Enabled: false,
+ MfaDelete: false,
+ },
+ PublicAccessBlockConfiguration: &s3types.PublicAccessBlockConfiguration{
+ BlockPublicAcls: pointers.Ref(false),
+ },
+ AccountPublicAccessBlockConfiguration: &s3ctrltypes.PublicAccessBlockConfiguration{
+ BlockPublicAcls: pointers.Ref(false),
+ },
+ Region: "europe-west-1",
+ }
+ in := []awslib.AwsResource{bucket1, bucket2}
+
+ expected := []inventory.AssetEvent{
+ inventory.NewAssetEvent(
+ s3BucketClassification,
+ "arn:aws:s3:::bucket-1",
+ "bucket-1",
+ inventory.WithRawAsset(bucket1),
+ inventory.WithCloud(inventory.AssetCloud{
+ Provider: inventory.AwsCloudProvider,
+ Region: "europe-west-1",
+ }),
+ ),
+ inventory.NewAssetEvent(
+ s3BucketClassification,
+ "arn:aws:s3:::bucket-2",
+ "bucket-2",
+ inventory.WithRawAsset(bucket2),
+ inventory.WithCloud(inventory.AssetCloud{
+ Provider: inventory.AwsCloudProvider,
+ Region: "europe-west-1",
+ }),
+ ),
+ }
+
+ logger := logp.NewLogger("test_fetcher_s3_bucket")
+ provider := newMockS3BucketProvider(t)
+ provider.EXPECT().DescribeBuckets(mock.Anything).Return(in, nil)
+
+ fetcher := S3BucketFetcher{
+ logger: logger,
+ provider: provider,
+ }
+
+ ch := make(chan inventory.AssetEvent)
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+ go func() {
+ fetcher.Fetch(ctx, ch)
+ }()
+
+ received := make([]inventory.AssetEvent, 0, len(expected))
+ for len(expected) != len(received) {
+ select {
+ case <-ctx.Done():
+ assert.ElementsMatch(t, expected, received)
+ return
+ case event := <-ch:
+ received = append(received, event)
+ }
+ }
+
+ assert.ElementsMatch(t, expected, received)
+}

internal/inventory/aws/fetchers.go

@@ -28,5 +28,6 @@ import (
 func Fetchers(logger *logp.Logger, identity *cloud.Identity, cfg aws.Config) []inventory.AssetFetcher {
  return []inventory.AssetFetcher{
  newEc2Fetcher(logger, identity, cfg),
+ NewS3BucketFetcher(logger, identity, cfg),
  }
 }