From 9247bf5c1d2f8c97dee628f84c635d3beaea8331 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 11:00:43 +0300 Subject: [PATCH 01/11] Remove discovery filters parameter from Discover and solely rely on filters from AwsConnection. Signed-off-by: Vasil Sirakov --- providers/aws/connection/connection.go | 22 +++-- providers/aws/connection/connection_test.go | 95 +++++++++++++++++++++ providers/aws/provider/provider.go | 2 +- providers/aws/resources/aws_ec2.go | 34 ++++++++ providers/aws/resources/aws_ec2_test.go | 75 ++++++++++++++++ providers/aws/resources/discovery.go | 47 ++-------- providers/aws/resources/discovery_test.go | 58 ------------- 7 files changed, 227 insertions(+), 106 deletions(-) create mode 100644 providers/aws/connection/connection_test.go create mode 100644 providers/aws/resources/aws_ec2_test.go diff --git a/providers/aws/connection/connection.go b/providers/aws/connection/connection.go index 7b91879c24..aa703aa299 100644 --- a/providers/aws/connection/connection.go +++ b/providers/aws/connection/connection.go @@ -53,9 +53,12 @@ type GeneralResourceDiscoveryFilters struct { } type Ec2DiscoveryFilters struct { - Regions []string - Tags map[string]string - InstanceIds []string + Regions []string + Tags map[string]string + InstanceIds []string + ExcludeRegions []string + ExcludeTags map[string]string + ExcludeInstanceIds []string } type EcrDiscoveryFilters struct { Tags []string @@ -126,9 +129,10 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config) return c, nil } +// TODO: @vasil - unit test. func parseOptsToFilters(opts map[string]string) DiscoveryFilters { d := DiscoveryFilters{ - Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}}, + Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}}, EcsDiscoveryFilters: EcsDiscoveryFilters{}, EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}}, GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}}, @@ -137,12 +141,18 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters { switch { case strings.HasPrefix(k, "ec2:tag:"): d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v - case k == "ec2:region": + case strings.HasPrefix(k, "exclude:ec2:tag:"): + d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "exclude:ec2:tag:")] = v + case strings.HasPrefix(k, "ec2:region:"): d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, v) + case strings.HasPrefix(k, "exclude:ec2:region"): + d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, v) case k == "all:region", k == "region": d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, v) - case k == "instance-id": + case strings.HasPrefix(k, "instance-id:"): d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, v) + case strings.HasPrefix(k, "exclude:instance-id:"): + d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, v) case strings.HasPrefix(k, "all:tag:"): d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v case k == "ecr:tag": diff --git a/providers/aws/connection/connection_test.go b/providers/aws/connection/connection_test.go new file mode 100644 index 0000000000..0762ee1d72 --- /dev/null +++ b/providers/aws/connection/connection_test.go @@ -0,0 +1,95 @@ +package connection + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// testParseOptsToFilters accepts a map which doesn't guarantee a deterministic iteration order. this means that slices +// in the parsed filters need to be compared individually ensuring their elements match regardless of their order. +func compareFilters(t *testing.T, expected, actual DiscoveryFilters) { + require.ElementsMatch(t, expected.Ec2DiscoveryFilters.Regions, actual.Ec2DiscoveryFilters.Regions) + require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeRegions, actual.Ec2DiscoveryFilters.ExcludeRegions) + + require.ElementsMatch(t, expected.Ec2DiscoveryFilters.InstanceIds, actual.Ec2DiscoveryFilters.InstanceIds) + require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeInstanceIds, actual.Ec2DiscoveryFilters.ExcludeInstanceIds) + + require.Equal(t, expected.Ec2DiscoveryFilters.Tags, actual.Ec2DiscoveryFilters.Tags) + require.Equal(t, expected.Ec2DiscoveryFilters.ExcludeTags, actual.Ec2DiscoveryFilters.ExcludeTags) + + require.Equal(t, expected.EcsDiscoveryFilters, actual.EcsDiscoveryFilters) + + require.Equal(t, expected.EcrDiscoveryFilters.Tags, actual.EcrDiscoveryFilters.Tags) + + require.ElementsMatch(t, expected.GeneralDiscoveryFilters.Regions, actual.GeneralDiscoveryFilters.Regions) + require.Equal(t, expected.GeneralDiscoveryFilters.Tags, actual.GeneralDiscoveryFilters.Tags) +} + +func TestParseOptsToFilters(t *testing.T) { + t.Run("all opts are mapped to discovery filters correctly", func(t *testing.T) { + opts := map[string]string{ + // Ec2DiscoveryFilters.Tags + "ec2:tag:key1": "val1", + "ec2:tag:key2": "val2", + // Ec2DiscoveryFilters.ExcludeTags + "exclude:ec2:tag:key1": "val1", + "exclude:ec2:tag:key2": "val2", + // Ec2DiscoveryFilters.Regions + "ec2:region:us-east-1": "us-east-1", + "ec2:region:us-west-1": "us-west-1", + // Ec2DiscoveryFilters.ExcludeRegions + "exclude:ec2:region:us-east-1": "us-east-1", + "exclude:ec2:region:us-west-1": "us-west-1", + // Ec2DiscoveryFilters.InstanceIds + "instance-id:iid-1": "iid-1", + "instance-id:iid-2": "iid-2", + // Ec2DiscoveryFilters.ExcludeInstanceIds + "exclude:instance-id:iid-1": "iid-1", + "exclude:instance-id:iid-2": "iid-2", + // TODO: @vasil - include others? + } + expected := DiscoveryFilters{ + Ec2DiscoveryFilters: Ec2DiscoveryFilters{ + Regions: []string{ + "us-east-1", "us-west-1", + }, + ExcludeRegions: []string{ + "us-east-1", "us-west-1", + }, + InstanceIds: []string{ + "iid-1", "iid-2", + }, + ExcludeInstanceIds: []string{ + "iid-1", "iid-2", + }, + Tags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + ExcludeTags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + }, + EcsDiscoveryFilters: EcsDiscoveryFilters{}, + EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}}, + GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}}, + } + + actual := parseOptsToFilters(opts) + compareFilters(t, expected, actual) + }) + + t.Run("empty opts are mapped to discovery filters correctly", func(t *testing.T) { + expected := DiscoveryFilters{ + Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}}, + EcsDiscoveryFilters: EcsDiscoveryFilters{}, + EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}}, + GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}}, + } + + actual := parseOptsToFilters(map[string]string{}) + compareFilters(t, expected, actual) + }) +} diff --git a/providers/aws/provider/provider.go b/providers/aws/provider/provider.go index c23cd245ef..1017eae4b2 100644 --- a/providers/aws/provider/provider.go +++ b/providers/aws/provider/provider.go @@ -269,5 +269,5 @@ func (s *Service) discover(conn *connection.AwsConnection) (*inventory.Inventory return nil, err } - return resources.Discover(runtime, conn.Filters) + return resources.Discover(runtime) } diff --git a/providers/aws/resources/aws_ec2.go b/providers/aws/resources/aws_ec2.go index e22619b0e7..64e01cc5dc 100644 --- a/providers/aws/resources/aws_ec2.go +++ b/providers/aws/resources/aws_ec2.go @@ -803,6 +803,9 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job if len(conn.Filters.Ec2DiscoveryFilters.Regions) > 0 { regions = conn.Filters.Ec2DiscoveryFilters.Regions } + for _, regionToExclude := range conn.Filters.Ec2DiscoveryFilters.ExcludeRegions { + regions = removeElement(regions, regionToExclude) + } for _, region := range regions { regionVal := region f := func() (jobpool.JobResult, error) { @@ -837,6 +840,9 @@ func (a *mqlAwsEc2) gatherInstanceInfo(instances []ec2types.Reservation, regionV res := []interface{}{} for _, reservation := range instances { for _, instance := range reservation.Instances { + if shouldExcludeInstance(instance, conn.Filters.Ec2DiscoveryFilters) { + continue + } mqlDevices := []interface{}{} for i := range instance.BlockDeviceMappings { device := instance.BlockDeviceMappings[i] @@ -1769,3 +1775,31 @@ func (a *mqlAwsEc2Vpnconnection) id() (string, error) { func (a *mqlAwsEc2Vgwtelemetry) id() (string, error) { return a.OutsideIpAddress.Data, nil } + +// true if the instance should be excluded from results. filtering for excluded regions should happen before we retrieve the EC2 instance. +func shouldExcludeInstance(instance ec2types.Instance, filters connection.Ec2DiscoveryFilters) bool { + for _, id := range filters.ExcludeInstanceIds { + if instance.InstanceId != nil && *instance.InstanceId == id { + return true + } + } + for k, v := range filters.ExcludeTags { + for _, iTag := range instance.Tags { + if iTag.Key != nil && *iTag.Key == k && + iTag.Value != nil && *iTag.Value == v { + return true + } + } + } + return false +} + +func removeElement(slice []string, value string) []string { + result := []string{} + for _, v := range slice { + if v != value { + result = append(result, v) + } + } + return result +} diff --git a/providers/aws/resources/aws_ec2_test.go b/providers/aws/resources/aws_ec2_test.go new file mode 100644 index 0000000000..df47347392 --- /dev/null +++ b/providers/aws/resources/aws_ec2_test.go @@ -0,0 +1,75 @@ +package resources + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/stretchr/testify/require" + "go.mondoo.com/cnquery/v11/providers/aws/connection" +) + +func TestShouldExcludeInstance(t *testing.T) { + instance := ec2types.Instance{ + InstanceId: aws.String("iid"), + Tags: []ec2types.Tag{ + { + Key: aws.String("key-1"), + Value: aws.String("val-1"), + }, + { + Key: aws.String("key-2"), + Value: aws.String("val-2"), + }, + }, + } + + t.Run("should exclude instance by id", func(t *testing.T) { + filters := connection.Ec2DiscoveryFilters{ + ExcludeInstanceIds: []string{ + "iid", + }, + ExcludeTags: map[string]string{ + "key-3": "val3", + }, + } + require.True(t, shouldExcludeInstance(instance, filters)) + }) + + t.Run("should exclude instance by matching tag", func(t *testing.T) { + filters := connection.Ec2DiscoveryFilters{ + ExcludeInstanceIds: []string{ + "iid-2", + }, + ExcludeTags: map[string]string{ + "key-2": "val2", + }, + } + require.False(t, shouldExcludeInstance(instance, filters)) + }) + + t.Run("should not exclude instance with only a matching tag key", func(t *testing.T) { + filters := connection.Ec2DiscoveryFilters{ + ExcludeInstanceIds: []string{ + "iid-2", + }, + ExcludeTags: map[string]string{ + "key-2": "val3", + "key-3": "val3", + }, + } + require.False(t, shouldExcludeInstance(instance, filters)) + }) + + t.Run("should not exclude instance when instance id and tags don't match", func(t *testing.T) { + filters := connection.Ec2DiscoveryFilters{ + ExcludeInstanceIds: []string{ + "iid-2", + }, + ExcludeTags: map[string]string{ + "key-3": "val3", + }, + } + require.False(t, shouldExcludeInstance(instance, filters)) + }) +} diff --git a/providers/aws/resources/discovery.go b/providers/aws/resources/discovery.go index f296852470..6b90b1d5fa 100644 --- a/providers/aws/resources/discovery.go +++ b/providers/aws/resources/discovery.go @@ -114,35 +114,6 @@ func containsInterfaceSlice(sl []interface{}, s string) bool { return false } -func instanceMatchesFilters(instance *mqlAwsEc2Instance, filters connection.DiscoveryFilters) bool { - regions := []string{} - if len(filters.GeneralDiscoveryFilters.Regions) > 0 { - regions = append(regions, filters.GeneralDiscoveryFilters.Regions...) - } - if len(filters.Ec2DiscoveryFilters.Regions) > 0 { - regions = append(regions, filters.Ec2DiscoveryFilters.Regions...) - } - if len(regions) > 0 && !contains(regions, instance.Region.Data) { - return false - } - if len(filters.Ec2DiscoveryFilters.InstanceIds) > 0 { - if !contains(filters.Ec2DiscoveryFilters.InstanceIds, instance.InstanceId.Data) { - return false - } - } - if len(filters.Ec2DiscoveryFilters.Tags) > 0 { - for k, v := range filters.Ec2DiscoveryFilters.Tags { - if instance.Tags.Data[k] == nil { - return false - } - if instance.Tags.Data[k].(string) != v { - return false - } - } - } - return true -} - func imageMatchesFilters(image *mqlAwsEcrImage, filters connection.DiscoveryFilters) bool { f := filters.EcrDiscoveryFilters if len(f.Tags) > 0 { @@ -185,9 +156,8 @@ func discoveredAssetMatchesGeneralFilters(asset *inventory.Asset, filters connec return true } -func Discover(runtime *plugin.Runtime, filters connection.DiscoveryFilters) (*inventory.Inventory, error) { +func Discover(runtime *plugin.Runtime) (*inventory.Inventory, error) { conn := runtime.Connection.(*connection.AwsConnection) - in := &inventory.Inventory{Spec: &inventory.InventorySpec{ Assets: []*inventory.Asset{}, }} @@ -202,15 +172,15 @@ func Discover(runtime *plugin.Runtime, filters connection.DiscoveryFilters) (*in targets := handleTargets(conn.Conf.Discover.Targets) for i := range targets { target := targets[i] - list, err := discover(runtime, awsAccount, target, filters) + list, err := discover(runtime, awsAccount, target, conn.Filters) if err != nil { log.Error().Err(err).Msg("error during discovery") continue } - if len(filters.GeneralDiscoveryFilters.Tags) > 0 { + if len(conn.Filters.GeneralDiscoveryFilters.Tags) > 0 { newList := []*inventory.Asset{} for i := range list { - if discoveredAssetMatchesGeneralFilters(list[i], filters.GeneralDiscoveryFilters) { + if discoveredAssetMatchesGeneralFilters(list[i], conn.Filters.GeneralDiscoveryFilters) { newList = append(newList, list[i]) } } @@ -274,6 +244,7 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, ec2 := res.(*mqlAwsEc2) + // get instances already filters out instances not matched by the filters specified in the AwsConnection ins := ec2.GetInstances() if ins == nil { return assetList, nil @@ -281,11 +252,7 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, for i := range ins.Data { instance := ins.Data[i].(*mqlAwsEc2Instance) - if !instanceMatchesFilters(instance, filters) { - continue - } assetList = append(assetList, addConnectionInfoToEc2Asset(instance, accountId, conn)) - } case DiscoverySSMInstances: res, err := NewResource(runtime, "aws.ssm", map[string]*llx.RawData{}) @@ -431,6 +398,7 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, ec2 := res.(*mqlAwsEc2) + // get instances already filters out instances not matched by the filters specified in the AwsConnection ins := ec2.GetInstances() if ins == nil { return assetList, nil @@ -438,9 +406,6 @@ func discover(runtime *plugin.Runtime, awsAccount *mqlAwsAccount, target string, for i := range ins.Data { instance := ins.Data[i].(*mqlAwsEc2Instance) - if !instanceMatchesFilters(instance, filters) { - continue - } l := mapStringInterfaceToStringString(instance.Tags.Data) assetList = append(assetList, MqlObjectToAsset(accountId, mqlObject{ diff --git a/providers/aws/resources/discovery_test.go b/providers/aws/resources/discovery_test.go index fc441089ba..9d0989c763 100644 --- a/providers/aws/resources/discovery_test.go +++ b/providers/aws/resources/discovery_test.go @@ -15,64 +15,6 @@ import ( ) func TestFilters(t *testing.T) { - require.True(t, instanceMatchesFilters(&mqlAwsEc2Instance{InstanceId: plugin.TValue[string]{Data: "i-test"}}, connection.DiscoveryFilters{ - Ec2DiscoveryFilters: connection.Ec2DiscoveryFilters{ - InstanceIds: []string{"i-test"}, - }, - }, - )) - require.True(t, instanceMatchesFilters(&mqlAwsEc2Instance{ - InstanceId: plugin.TValue[string]{Data: "i-test"}, - Tags: plugin.TValue[map[string]interface{}]{Data: map[string]interface{}{"tester2": "val2", "test-tag": "val"}}, - }, connection.DiscoveryFilters{ - Ec2DiscoveryFilters: connection.Ec2DiscoveryFilters{ - InstanceIds: []string{"i-test"}, - Tags: map[string]string{"tester2": "val2"}, - }, - }, - )) - require.True(t, instanceMatchesFilters(&mqlAwsEc2Instance{ - InstanceId: plugin.TValue[string]{Data: "i-test"}, - Tags: plugin.TValue[map[string]interface{}]{Data: map[string]interface{}{"tester2": "val2", "test-tag": "val"}}, - Region: plugin.TValue[string]{Data: "us-west-1"}, - }, connection.DiscoveryFilters{ - Ec2DiscoveryFilters: connection.Ec2DiscoveryFilters{ - Regions: []string{"us-east-2", "us-east-1", "us-west-1"}, - }, - }, - )) - require.True(t, instanceMatchesFilters(&mqlAwsEc2Instance{ - InstanceId: plugin.TValue[string]{Data: "i-test"}, - Tags: plugin.TValue[map[string]interface{}]{Data: map[string]interface{}{"tester2": "val2", "test-tag": "val"}}, - Region: plugin.TValue[string]{Data: "us-west-1"}, - }, connection.DiscoveryFilters{})) - require.False(t, instanceMatchesFilters(&mqlAwsEc2Instance{InstanceId: plugin.TValue[string]{Data: "i-test"}}, connection.DiscoveryFilters{ - Ec2DiscoveryFilters: connection.Ec2DiscoveryFilters{ - InstanceIds: []string{"i-test2"}, - }, - }, - )) - require.False(t, instanceMatchesFilters(&mqlAwsEc2Instance{ - InstanceId: plugin.TValue[string]{Data: "i-test"}, - Tags: plugin.TValue[map[string]interface{}]{Data: map[string]interface{}{"tester2": "val2", "test-tag": "val"}}, - }, connection.DiscoveryFilters{ - Ec2DiscoveryFilters: connection.Ec2DiscoveryFilters{ - InstanceIds: []string{"i-test"}, - Tags: map[string]string{"test-tag": "val2"}, - }, - }, - )) - require.False(t, instanceMatchesFilters(&mqlAwsEc2Instance{ - InstanceId: plugin.TValue[string]{Data: "i-test"}, - Tags: plugin.TValue[map[string]interface{}]{Data: map[string]interface{}{"tester2": "val2", "test-tag": "val"}}, - Region: plugin.TValue[string]{Data: "us-west-2"}, - }, connection.DiscoveryFilters{ - Ec2DiscoveryFilters: connection.Ec2DiscoveryFilters{ - Regions: []string{"us-east-2", "us-east-1", "us-west-1"}, - }, - }, - )) - require.True(t, imageMatchesFilters(&mqlAwsEcrImage{ Tags: plugin.TValue[[]interface{}]{Data: []interface{}{"latest"}}, }, connection.DiscoveryFilters{})) From f983f6661176e668b4a7323ef0643ca844115fab Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 11:05:17 +0300 Subject: [PATCH 02/11] Added license header to new files. Signed-off-by: Vasil Sirakov --- providers/aws/connection/connection_test.go | 3 +++ providers/aws/resources/aws_ec2_test.go | 3 +++ 2 files changed, 6 insertions(+) diff --git a/providers/aws/connection/connection_test.go b/providers/aws/connection/connection_test.go index 0762ee1d72..c922046e0e 100644 --- a/providers/aws/connection/connection_test.go +++ b/providers/aws/connection/connection_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package connection import ( diff --git a/providers/aws/resources/aws_ec2_test.go b/providers/aws/resources/aws_ec2_test.go index df47347392..2f528564a5 100644 --- a/providers/aws/resources/aws_ec2_test.go +++ b/providers/aws/resources/aws_ec2_test.go @@ -1,3 +1,6 @@ +// Copyright (c) Mondoo, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package resources import ( From 78a0956eaca01bb5cc2b8b184523bb56044f7a60 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 11:53:57 +0300 Subject: [PATCH 03/11] Added ECS discovery filters parsing from inventory config. Signed-off-by: Vasil Sirakov --- providers/aws/connection/connection.go | 21 ++++++++++-- providers/aws/connection/connection_test.go | 37 ++++++++++++++++++--- providers/aws/resources/discovery.go | 9 ----- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/providers/aws/connection/connection.go b/providers/aws/connection/connection.go index aa703aa299..b1b8ca4a71 100644 --- a/providers/aws/connection/connection.go +++ b/providers/aws/connection/connection.go @@ -6,6 +6,7 @@ package connection import ( "context" "errors" + "strconv" "strings" "github.com/aws/aws-sdk-go-v2/aws" @@ -129,7 +130,6 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config) return c, nil } -// TODO: @vasil - unit test. func parseOptsToFilters(opts map[string]string) DiscoveryFilters { d := DiscoveryFilters{ Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}}, @@ -147,7 +147,7 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters { d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, v) case strings.HasPrefix(k, "exclude:ec2:region"): d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, v) - case k == "all:region", k == "region": + case strings.HasPrefix(k, "all:region:"), strings.HasPrefix(k, "region:"): d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, v) case strings.HasPrefix(k, "instance-id:"): d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, v) @@ -155,8 +155,23 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters { d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, v) case strings.HasPrefix(k, "all:tag:"): d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v - case k == "ecr:tag": + case strings.HasPrefix(k, "ecr:tag:"): d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, v) + case k == "ecs:only-running-containers": + parsed, err := strconv.ParseBool(v) + if err == nil { + d.EcsDiscoveryFilters.OnlyRunningContainers = parsed + } + case k == "ecs:discover-instances": + parsed, err := strconv.ParseBool(v) + if err == nil { + d.EcsDiscoveryFilters.DiscoverInstances = parsed + } + case k == "ecs:discover-images": + parsed, err := strconv.ParseBool(v) + if err == nil { + d.EcsDiscoveryFilters.DiscoverImages = parsed + } } } return d diff --git a/providers/aws/connection/connection_test.go b/providers/aws/connection/connection_test.go index c922046e0e..85ed7833d8 100644 --- a/providers/aws/connection/connection_test.go +++ b/providers/aws/connection/connection_test.go @@ -23,7 +23,7 @@ func compareFilters(t *testing.T, expected, actual DiscoveryFilters) { require.Equal(t, expected.EcsDiscoveryFilters, actual.EcsDiscoveryFilters) - require.Equal(t, expected.EcrDiscoveryFilters.Tags, actual.EcrDiscoveryFilters.Tags) + require.ElementsMatch(t, expected.EcrDiscoveryFilters.Tags, actual.EcrDiscoveryFilters.Tags) require.ElementsMatch(t, expected.GeneralDiscoveryFilters.Regions, actual.GeneralDiscoveryFilters.Regions) require.Equal(t, expected.GeneralDiscoveryFilters.Tags, actual.GeneralDiscoveryFilters.Tags) @@ -50,7 +50,20 @@ func TestParseOptsToFilters(t *testing.T) { // Ec2DiscoveryFilters.ExcludeInstanceIds "exclude:instance-id:iid-1": "iid-1", "exclude:instance-id:iid-2": "iid-2", - // TODO: @vasil - include others? + // GeneralDiscoveryFilters.Regions + "all:region:us-east-1": "us-east-1", + "all:region:us-west-1": "us-west-1", + "region:eu-west-1": "eu-west-1", + // GeneralDiscoveryFilters.Tags + "all:tag:key1": "val1", + "all:tag:key2": "val2", + // EcrDiscoveryFilters.Tags + "ecr:tag:tag1": "tag1", + "ecr:tag:tag2": "tag2", + // EcsDiscoveryFilters + "ecs:only-running-containers": "true", + "ecs:discover-images": "T", + "ecs:discover-instances": "false", } expected := DiscoveryFilters{ Ec2DiscoveryFilters: Ec2DiscoveryFilters{ @@ -75,9 +88,23 @@ func TestParseOptsToFilters(t *testing.T) { "key2": "val2", }, }, - EcsDiscoveryFilters: EcsDiscoveryFilters{}, - EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}}, - GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}}, + EcsDiscoveryFilters: EcsDiscoveryFilters{ + OnlyRunningContainers: true, + DiscoverImages: true, + DiscoverInstances: false, + }, + EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{ + "tag1", "tag2", + }}, + GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{ + Regions: []string{ + "us-east-1", "us-west-1", "eu-west-1", + }, + Tags: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + }, } actual := parseOptsToFilters(opts) diff --git a/providers/aws/resources/discovery.go b/providers/aws/resources/discovery.go index 6b90b1d5fa..d799d3668c 100644 --- a/providers/aws/resources/discovery.go +++ b/providers/aws/resources/discovery.go @@ -96,15 +96,6 @@ var AllAPIResources = []string{ DiscoverySagemakerNotebookInstances, } -func contains(sl []string, s string) bool { - for i := range sl { - if sl[i] == s { - return true - } - } - return false -} - func containsInterfaceSlice(sl []interface{}, s string) bool { for i := range sl { if sl[i].(string) == s { From dbd4d868802410f0a03fe140824a44bcd4591269 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 12:58:21 +0300 Subject: [PATCH 04/11] Added 'ec2' prefix in front of instance id ec2 filter tag name. Signed-off-by: Vasil Sirakov --- providers/aws/connection/connection.go | 4 ++-- providers/aws/connection/connection_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/providers/aws/connection/connection.go b/providers/aws/connection/connection.go index b1b8ca4a71..17c3c8dba5 100644 --- a/providers/aws/connection/connection.go +++ b/providers/aws/connection/connection.go @@ -149,9 +149,9 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters { d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, v) case strings.HasPrefix(k, "all:region:"), strings.HasPrefix(k, "region:"): d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, v) - case strings.HasPrefix(k, "instance-id:"): + case strings.HasPrefix(k, "ec2:instance-id:"): d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, v) - case strings.HasPrefix(k, "exclude:instance-id:"): + case strings.HasPrefix(k, "exclude:ec2:instance-id:"): d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, v) case strings.HasPrefix(k, "all:tag:"): d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v diff --git a/providers/aws/connection/connection_test.go b/providers/aws/connection/connection_test.go index 85ed7833d8..80bd8a3a17 100644 --- a/providers/aws/connection/connection_test.go +++ b/providers/aws/connection/connection_test.go @@ -45,11 +45,11 @@ func TestParseOptsToFilters(t *testing.T) { "exclude:ec2:region:us-east-1": "us-east-1", "exclude:ec2:region:us-west-1": "us-west-1", // Ec2DiscoveryFilters.InstanceIds - "instance-id:iid-1": "iid-1", - "instance-id:iid-2": "iid-2", + "ec2:instance-id:iid-1": "iid-1", + "ec2:instance-id:iid-2": "iid-2", // Ec2DiscoveryFilters.ExcludeInstanceIds - "exclude:instance-id:iid-1": "iid-1", - "exclude:instance-id:iid-2": "iid-2", + "exclude:ec2:instance-id:iid-1": "iid-1", + "exclude:ec2:instance-id:iid-2": "iid-2", // GeneralDiscoveryFilters.Regions "all:region:us-east-1": "us-east-1", "all:region:us-west-1": "us-west-1", From 4396bbb2b237b48a346afc3803851e18fae24fb3 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 15:00:25 +0300 Subject: [PATCH 05/11] Extract function to determine applicable regions from which instances should be queried. Signed-off-by: Vasil Sirakov --- providers/aws/resources/aws_ec2.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/providers/aws/resources/aws_ec2.go b/providers/aws/resources/aws_ec2.go index 64e01cc5dc..1f972c6a24 100644 --- a/providers/aws/resources/aws_ec2.go +++ b/providers/aws/resources/aws_ec2.go @@ -800,12 +800,7 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job if err != nil { return []*jobpool.Job{{Err: err}} } - if len(conn.Filters.Ec2DiscoveryFilters.Regions) > 0 { - regions = conn.Filters.Ec2DiscoveryFilters.Regions - } - for _, regionToExclude := range conn.Filters.Ec2DiscoveryFilters.ExcludeRegions { - regions = removeElement(regions, regionToExclude) - } + regions = determineApplicableRegions(regions, conn.Filters.Ec2DiscoveryFilters.Regions, conn.Filters.Ec2DiscoveryFilters.ExcludeRegions) for _, region := range regions { regionVal := region f := func() (jobpool.JobResult, error) { @@ -1794,6 +1789,19 @@ func shouldExcludeInstance(instance ec2types.Instance, filters connection.Ec2Dis return false } +// given an initial set of regions, applies the allowed regions filter and the excluded regions filter on it +// returning a resulting slice of only applicable region to which queries should be targeted +func determineApplicableRegions(regions, regionsToInclude, regionsToExclude []string) []string { + res := regions + if len(regionsToInclude) > 0 { + res = regionsToInclude + } + for _, regionToExclude := range regionsToExclude { + res = removeElement(res, regionToExclude) + } + return res +} + func removeElement(slice []string, value string) []string { result := []string{} for _, v := range slice { From f530e7b14129f1852f86485923e16c1806db0cff Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 15:06:08 +0300 Subject: [PATCH 06/11] Add unit test to the function that determines which regions are applicable. Signed-off-by: Vasil Sirakov --- providers/aws/resources/aws_ec2_test.go | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/providers/aws/resources/aws_ec2_test.go b/providers/aws/resources/aws_ec2_test.go index 2f528564a5..a8a375d94e 100644 --- a/providers/aws/resources/aws_ec2_test.go +++ b/providers/aws/resources/aws_ec2_test.go @@ -76,3 +76,35 @@ func TestShouldExcludeInstance(t *testing.T) { require.False(t, shouldExcludeInstance(instance, filters)) }) } + +func TestDetermineApplicableRegions(t *testing.T) { + t.Run("allow regions override initial region list", func(t *testing.T) { + initialRegions := []string{"a", "b"} + allowedRegions := []string{"b", "c"} + excludedRegions := []string{} + + expected := []string{"b", "c"} + actual := determineApplicableRegions(initialRegions, allowedRegions, excludedRegions) + require.ElementsMatch(t, expected, actual) + }) + + t.Run("excluded regions work correctly", func(t *testing.T) { + initialRegions := []string{"a", "b"} + allowedRegions := []string{} + excludedRegions := []string{"b"} + + expected := []string{"a"} + actual := determineApplicableRegions(initialRegions, allowedRegions, excludedRegions) + require.ElementsMatch(t, expected, actual) + }) + + t.Run("excluded regions not present in the initial slice are ignored", func(t *testing.T) { + initialRegions := []string{"a", "b"} + allowedRegions := []string{} + excludedRegions := []string{"b", "c", "d", "e"} + + expected := []string{"a"} + actual := determineApplicableRegions(initialRegions, allowedRegions, excludedRegions) + require.ElementsMatch(t, expected, actual) + }) +} From 0dab40ba7a08dd78b465dcd22482bd2cec2ef3f0 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 15:11:36 +0300 Subject: [PATCH 07/11] Simplified function to determine applicable regions. Signed-off-by: Vasil Sirakov --- providers/aws/resources/aws_ec2.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/providers/aws/resources/aws_ec2.go b/providers/aws/resources/aws_ec2.go index 1f972c6a24..2ab4fda539 100644 --- a/providers/aws/resources/aws_ec2.go +++ b/providers/aws/resources/aws_ec2.go @@ -26,6 +26,7 @@ import ( "go.mondoo.com/cnquery/v11/providers-sdk/v1/util/jobpool" "go.mondoo.com/cnquery/v11/providers/aws/connection" "go.mondoo.com/cnquery/v11/types" + "go.mondoo.com/cnquery/v11/utils/stringx" ) func (e *mqlAwsEc2) id() (string, error) { @@ -1792,22 +1793,15 @@ func shouldExcludeInstance(instance ec2types.Instance, filters connection.Ec2Dis // given an initial set of regions, applies the allowed regions filter and the excluded regions filter on it // returning a resulting slice of only applicable region to which queries should be targeted func determineApplicableRegions(regions, regionsToInclude, regionsToExclude []string) []string { - res := regions if len(regionsToInclude) > 0 { - res = regionsToInclude + return regionsToInclude } - for _, regionToExclude := range regionsToExclude { - res = removeElement(res, regionToExclude) - } - return res -} - -func removeElement(slice []string, value string) []string { - result := []string{} - for _, v := range slice { - if v != value { - result = append(result, v) + res := []string{} + for _, r := range regions { + if !stringx.Contains(regionsToExclude, r) { + res = append(res, r) } } - return result + + return res } From d4a059e370ee07c255050ea4bf2ae7e02a742a3f Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 16:24:15 +0300 Subject: [PATCH 08/11] Use comma-separated values instead of individual keys in the filters map and implement parsing of CLI flags to opts matching them to known filter prefixes. Signed-off-by: Vasil Sirakov --- providers/aws/connection/connection.go | 20 +++++++++--------- providers/aws/provider/provider.go | 28 ++++++++++++++++++-------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/providers/aws/connection/connection.go b/providers/aws/connection/connection.go index 17c3c8dba5..03b8fc21a6 100644 --- a/providers/aws/connection/connection.go +++ b/providers/aws/connection/connection.go @@ -143,16 +143,16 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters { d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v case strings.HasPrefix(k, "exclude:ec2:tag:"): d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "exclude:ec2:tag:")] = v - case strings.HasPrefix(k, "ec2:region:"): - d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, v) - case strings.HasPrefix(k, "exclude:ec2:region"): - d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, v) - case strings.HasPrefix(k, "all:region:"), strings.HasPrefix(k, "region:"): - d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, v) - case strings.HasPrefix(k, "ec2:instance-id:"): - d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, v) - case strings.HasPrefix(k, "exclude:ec2:instance-id:"): - d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, v) + case strings.HasPrefix(k, "ec2:regions"): + d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, strings.Split(v, ",")...) + case strings.HasPrefix(k, "exclude:ec2:regions"): + d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, strings.Split(v, ",")...) + case strings.HasPrefix(k, "all:regions"), strings.HasPrefix(k, "regions"): + d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, strings.Split(v, ",")...) + case strings.HasPrefix(k, "ec2:instance-ids"): + d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, strings.Split(v, ",")...) + case strings.HasPrefix(k, "exclude:ec2:instance-ids"): + d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, strings.Split(v, ",")...) case strings.HasPrefix(k, "all:tag:"): d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v case strings.HasPrefix(k, "ecr:tag:"): diff --git a/providers/aws/provider/provider.go b/providers/aws/provider/provider.go index 1017eae4b2..4aea2bd34c 100644 --- a/providers/aws/provider/provider.go +++ b/providers/aws/provider/provider.go @@ -83,15 +83,27 @@ func parseFlagsToFiltersOpts(m map[string]*llx.Primitive) map[string]string { o := make(map[string]string, 0) if x, ok := m["filters"]; ok && len(x.Map) != 0 { + knownTagPrefixes := []string{ + "ec2:tag:", + "exclude:ec2:tag:", + "ec2:regions", + "exclude:ec2:regions", + "all:regions", + "regions", + "ec2:instance-ids", + "exclude:ec2:instance-ids", + "all:tag:", + "ecr:tag:", + "ecs:only-running-containers", + "ecs:discover-instances", + "ecs:discover-images", + } for k, v := range x.Map { - if strings.Contains(k, "tag:") { - o[k] = string(v.Value) - } - if k == "instance-id" { - o[k] = string(v.Value) - } - if strings.Contains(k, "region") { - o[k] = string(v.Value) + for _, prefix := range knownTagPrefixes { + if strings.HasPrefix(k, prefix) { + o[k] = string(v.Value) + break + } } } } From 19042d96388bde59561e0ee8b20936c513d4da32 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 18:27:19 +0300 Subject: [PATCH 09/11] Fix unit test for parsing options to discovery filters. Signed-off-by: Vasil Sirakov --- providers/aws/connection/connection_test.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/providers/aws/connection/connection_test.go b/providers/aws/connection/connection_test.go index 80bd8a3a17..82b9939c5d 100644 --- a/providers/aws/connection/connection_test.go +++ b/providers/aws/connection/connection_test.go @@ -39,21 +39,15 @@ func TestParseOptsToFilters(t *testing.T) { "exclude:ec2:tag:key1": "val1", "exclude:ec2:tag:key2": "val2", // Ec2DiscoveryFilters.Regions - "ec2:region:us-east-1": "us-east-1", - "ec2:region:us-west-1": "us-west-1", + "ec2:regions": "us-east-1,us-west-1", // Ec2DiscoveryFilters.ExcludeRegions - "exclude:ec2:region:us-east-1": "us-east-1", - "exclude:ec2:region:us-west-1": "us-west-1", + "exclude:ec2:regions": "us-east-1,us-west-1", // Ec2DiscoveryFilters.InstanceIds - "ec2:instance-id:iid-1": "iid-1", - "ec2:instance-id:iid-2": "iid-2", + "ec2:instance-ids": "iid-1,iid-2", // Ec2DiscoveryFilters.ExcludeInstanceIds - "exclude:ec2:instance-id:iid-1": "iid-1", - "exclude:ec2:instance-id:iid-2": "iid-2", + "exclude:ec2:instance-ids": "iid-1,iid-2", // GeneralDiscoveryFilters.Regions - "all:region:us-east-1": "us-east-1", - "all:region:us-west-1": "us-west-1", - "region:eu-west-1": "eu-west-1", + "all:regions": "us-east-1,us-west-1,eu-west-1", // GeneralDiscoveryFilters.Tags "all:tag:key1": "val1", "all:tag:key2": "val2", From adbfee4266024a67304ab4c1c24df1ba1916f5e0 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Wed, 25 Sep 2024 20:04:36 +0300 Subject: [PATCH 10/11] Improve cases for each tag to equals instead of HasPrefix where we no longer embed the value into the key. Signed-off-by: Vasil Sirakov --- providers/aws/connection/connection.go | 14 +++++++------- providers/aws/connection/connection_test.go | 3 +-- providers/aws/provider/provider.go | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/providers/aws/connection/connection.go b/providers/aws/connection/connection.go index 03b8fc21a6..c4caef50fe 100644 --- a/providers/aws/connection/connection.go +++ b/providers/aws/connection/connection.go @@ -143,20 +143,20 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters { d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v case strings.HasPrefix(k, "exclude:ec2:tag:"): d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "exclude:ec2:tag:")] = v - case strings.HasPrefix(k, "ec2:regions"): + case k == "ec2:regions": d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, strings.Split(v, ",")...) - case strings.HasPrefix(k, "exclude:ec2:regions"): + case k == "exclude:ec2:regions": d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, strings.Split(v, ",")...) - case strings.HasPrefix(k, "all:regions"), strings.HasPrefix(k, "regions"): + case k == "all:regions", k == "regions": d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, strings.Split(v, ",")...) - case strings.HasPrefix(k, "ec2:instance-ids"): + case k == "ec2:instance-ids": d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, strings.Split(v, ",")...) - case strings.HasPrefix(k, "exclude:ec2:instance-ids"): + case k == "exclude:ec2:instance-ids": d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, strings.Split(v, ",")...) case strings.HasPrefix(k, "all:tag:"): d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v - case strings.HasPrefix(k, "ecr:tag:"): - d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, v) + case k == "ecr:tags": + d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, strings.Split(v, ",")...) case k == "ecs:only-running-containers": parsed, err := strconv.ParseBool(v) if err == nil { diff --git a/providers/aws/connection/connection_test.go b/providers/aws/connection/connection_test.go index 82b9939c5d..dc9561eb03 100644 --- a/providers/aws/connection/connection_test.go +++ b/providers/aws/connection/connection_test.go @@ -52,8 +52,7 @@ func TestParseOptsToFilters(t *testing.T) { "all:tag:key1": "val1", "all:tag:key2": "val2", // EcrDiscoveryFilters.Tags - "ecr:tag:tag1": "tag1", - "ecr:tag:tag2": "tag2", + "ecr:tags": "tag1,tag2", // EcsDiscoveryFilters "ecs:only-running-containers": "true", "ecs:discover-images": "T", diff --git a/providers/aws/provider/provider.go b/providers/aws/provider/provider.go index 4aea2bd34c..497b0c5a57 100644 --- a/providers/aws/provider/provider.go +++ b/providers/aws/provider/provider.go @@ -93,7 +93,7 @@ func parseFlagsToFiltersOpts(m map[string]*llx.Primitive) map[string]string { "ec2:instance-ids", "exclude:ec2:instance-ids", "all:tag:", - "ecr:tag:", + "ecr:tags", "ecs:only-running-containers", "ecs:discover-instances", "ecs:discover-images", From 8a9f7839236a0bd22289be24d63b92dca8938610 Mon Sep 17 00:00:00 2001 From: Vasil Sirakov Date: Thu, 26 Sep 2024 10:57:21 +0300 Subject: [PATCH 11/11] Fix bug where an instance with a specific id gets retrieved from a given region but propagate an error when we attempt to find it from another region it is not in. Signed-off-by: Vasil Sirakov --- providers/aws/resources/aws.go | 10 ++++++++++ providers/aws/resources/aws_ec2.go | 7 +++++++ 2 files changed, 17 insertions(+) diff --git a/providers/aws/resources/aws.go b/providers/aws/resources/aws.go index cc1fbc79fd..5a9474c473 100644 --- a/providers/aws/resources/aws.go +++ b/providers/aws/resources/aws.go @@ -70,6 +70,16 @@ func Is400AccessDeniedError(err error) bool { return false } +func Is400InstanceNotFoundError(err error) bool { + var respErr *http.ResponseError + if errors.As(err, &respErr) { + if respErr.HTTPStatusCode() == 400 && (strings.Contains(respErr.Error(), "InvalidInstanceID.NotFound") || strings.Contains(respErr.Error(), "InvalidInstanceID.Malformed")) { + return true + } + } + return false +} + func strMapToInterface(m map[string]string) map[string]interface{} { res := map[string]interface{}{} for k, v := range m { diff --git a/providers/aws/resources/aws_ec2.go b/providers/aws/resources/aws_ec2.go index 2ab4fda539..3109038f32 100644 --- a/providers/aws/resources/aws_ec2.go +++ b/providers/aws/resources/aws_ec2.go @@ -817,6 +817,13 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job log.Warn().Str("region", regionVal).Msg("error accessing region for AWS API") return res, nil } + // AWS returns an error response when trying to find an instance with a specific identifier if it cannot find it in some region. + // we do not propagate this error upward because an instance can be found in one region and return an error for all others which + // would be the expected behavior. + if Is400InstanceNotFoundError(err) { + log.Debug().Str("region", regionVal).Msg("could not find instance in region") + return res, nil + } return nil, err } res, err = a.gatherInstanceInfo(instances, regionVal)