🧹 Remove discovery filters parameter from Discover

providers/aws/connection/connection.go

@@ -6,6 +6,7 @@ package connection
 import (
  "context"
  "errors"
+ "strconv"
  "strings"
 
  "github.com/aws/aws-sdk-go-v2/aws"
@@ -53,9 +54,12 @@ type GeneralResourceDiscoveryFilters struct {
 }
 
 type Ec2DiscoveryFilters struct {
- Regions []string
- Tags map[string]string
- InstanceIds []string
+ Regions []string
+ Tags map[string]string
+ InstanceIds []string
+ ExcludeRegions []string
+ ExcludeTags map[string]string
+ ExcludeInstanceIds []string
 }
 type EcrDiscoveryFilters struct {
  Tags []string
@@ -128,7 +132,7 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
 
 func parseOptsToFilters(opts map[string]string) DiscoveryFilters {
  d := DiscoveryFilters{
- Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}},
+ Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}},
  EcsDiscoveryFilters: EcsDiscoveryFilters{},
  EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}},
  GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}},
@@ -137,16 +141,37 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters {
  switch {
  case strings.HasPrefix(k, "ec2:tag:"):
  d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v
- case k == "ec2:region":
+ case strings.HasPrefix(k, "exclude:ec2:tag:"):
+ d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "exclude:ec2:tag:")] = v
+ case strings.HasPrefix(k, "ec2:region:"):
  d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, v)
- case k == "all:region", k == "region":
+ case strings.HasPrefix(k, "exclude:ec2:region"):
+ d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, v)
+ case strings.HasPrefix(k, "all:region:"), strings.HasPrefix(k, "region:"):
  d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, v)
- case k == "instance-id":
+ case strings.HasPrefix(k, "ec2:instance-id:"):
  d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, v)
+ case strings.HasPrefix(k, "exclude:ec2:instance-id:"):
+ d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, v)
  case strings.HasPrefix(k, "all:tag:"):
  d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v
- case k == "ecr:tag":
+ case strings.HasPrefix(k, "ecr:tag:"):
  d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, v)
+ case k == "ecs:only-running-containers":
+ parsed, err := strconv.ParseBool(v)
+ if err == nil {
+ d.EcsDiscoveryFilters.OnlyRunningContainers = parsed
+ }
+ case k == "ecs:discover-instances":
+ parsed, err := strconv.ParseBool(v)
+ if err == nil {
+ d.EcsDiscoveryFilters.DiscoverInstances = parsed
+ }
+ case k == "ecs:discover-images":
+ parsed, err := strconv.ParseBool(v)
+ if err == nil {
+ d.EcsDiscoveryFilters.DiscoverImages = parsed
+ }
  }
  }
  return d

providers/aws/connection/connection_test.go

@@ -0,0 +1,125 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package connection
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// testParseOptsToFilters accepts a map which doesn't guarantee a deterministic iteration order. this means that slices
+// in the parsed filters need to be compared individually ensuring their elements match regardless of their order.
+func compareFilters(t *testing.T, expected, actual DiscoveryFilters) {
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.Regions, actual.Ec2DiscoveryFilters.Regions)
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeRegions, actual.Ec2DiscoveryFilters.ExcludeRegions)
+
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.InstanceIds, actual.Ec2DiscoveryFilters.InstanceIds)
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeInstanceIds, actual.Ec2DiscoveryFilters.ExcludeInstanceIds)
+
+ require.Equal(t, expected.Ec2DiscoveryFilters.Tags, actual.Ec2DiscoveryFilters.Tags)
+ require.Equal(t, expected.Ec2DiscoveryFilters.ExcludeTags, actual.Ec2DiscoveryFilters.ExcludeTags)
+
+ require.Equal(t, expected.EcsDiscoveryFilters, actual.EcsDiscoveryFilters)
+
+ require.ElementsMatch(t, expected.EcrDiscoveryFilters.Tags, actual.EcrDiscoveryFilters.Tags)
+
+ require.ElementsMatch(t, expected.GeneralDiscoveryFilters.Regions, actual.GeneralDiscoveryFilters.Regions)
+ require.Equal(t, expected.GeneralDiscoveryFilters.Tags, actual.GeneralDiscoveryFilters.Tags)
+}
+
+func TestParseOptsToFilters(t *testing.T) {
+ t.Run("all opts are mapped to discovery filters correctly", func(t *testing.T) {
+ opts := map[string]string{
+ // Ec2DiscoveryFilters.Tags
+ "ec2:tag:key1": "val1",
+ "ec2:tag:key2": "val2",
+ // Ec2DiscoveryFilters.ExcludeTags
+ "exclude:ec2:tag:key1": "val1",
+ "exclude:ec2:tag:key2": "val2",
+ // Ec2DiscoveryFilters.Regions
+ "ec2:region:us-east-1": "us-east-1",
+ "ec2:region:us-west-1": "us-west-1",
+ // Ec2DiscoveryFilters.ExcludeRegions
+ "exclude:ec2:region:us-east-1": "us-east-1",
+ "exclude:ec2:region:us-west-1": "us-west-1",
+ // Ec2DiscoveryFilters.InstanceIds
+ "ec2:instance-id:iid-1": "iid-1",
+ "ec2:instance-id:iid-2": "iid-2",
+ // Ec2DiscoveryFilters.ExcludeInstanceIds
+ "exclude:ec2:instance-id:iid-1": "iid-1",
+ "exclude:ec2:instance-id:iid-2": "iid-2",
+ // GeneralDiscoveryFilters.Regions
+ "all:region:us-east-1": "us-east-1",
+ "all:region:us-west-1": "us-west-1",
+ "region:eu-west-1": "eu-west-1",
+ // GeneralDiscoveryFilters.Tags
+ "all:tag:key1": "val1",
+ "all:tag:key2": "val2",
+ // EcrDiscoveryFilters.Tags
+ "ecr:tag:tag1": "tag1",
+ "ecr:tag:tag2": "tag2",
+ // EcsDiscoveryFilters
+ "ecs:only-running-containers": "true",
+ "ecs:discover-images": "T",
+ "ecs:discover-instances": "false",
+ }
+ expected := DiscoveryFilters{
+ Ec2DiscoveryFilters: Ec2DiscoveryFilters{
+ Regions: []string{
+ "us-east-1", "us-west-1",
+ },
+ ExcludeRegions: []string{
+ "us-east-1", "us-west-1",
+ },
+ InstanceIds: []string{
+ "iid-1", "iid-2",
+ },
+ ExcludeInstanceIds: []string{
+ "iid-1", "iid-2",
+ },
+ Tags: map[string]string{
+ "key1": "val1",
+ "key2": "val2",
+ },
+ ExcludeTags: map[string]string{
+ "key1": "val1",
+ "key2": "val2",
+ },
+ },
+ EcsDiscoveryFilters: EcsDiscoveryFilters{
+ OnlyRunningContainers: true,
+ DiscoverImages: true,
+ DiscoverInstances: false,
+ },
+ EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{
+ "tag1", "tag2",
+ }},
+ GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{
+ Regions: []string{
+ "us-east-1", "us-west-1", "eu-west-1",
+ },
+ Tags: map[string]string{
+ "key1": "val1",
+ "key2": "val2",
+ },
+ },
+ }
+
+ actual := parseOptsToFilters(opts)
+ compareFilters(t, expected, actual)
+ })
+
+ t.Run("empty opts are mapped to discovery filters correctly", func(t *testing.T) {
+ expected := DiscoveryFilters{
+ Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}},
+ EcsDiscoveryFilters: EcsDiscoveryFilters{},
+ EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}},
+ GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}},
+ }
+
+ actual := parseOptsToFilters(map[string]string{})
+ compareFilters(t, expected, actual)
+ })
+}

providers/aws/provider/provider.go

@@ -269,5 +269,5 @@ func (s *Service) discover(conn *connection.AwsConnection) (*inventory.Inventory
  return nil, err
  }
 
- return resources.Discover(runtime, conn.Filters)
+ return resources.Discover(runtime)
 }

providers/aws/resources/aws_ec2.go

@@ -803,6 +803,9 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job
  if len(conn.Filters.Ec2DiscoveryFilters.Regions) > 0 {
  regions = conn.Filters.Ec2DiscoveryFilters.Regions
  }
+ for _, regionToExclude := range conn.Filters.Ec2DiscoveryFilters.ExcludeRegions {
+ regions = removeElement(regions, regionToExclude)
+ }
  for _, region := range regions {
  regionVal := region
  f := func() (jobpool.JobResult, error) {
@@ -837,6 +840,9 @@ func (a *mqlAwsEc2) gatherInstanceInfo(instances []ec2types.Reservation, regionV
  res := []interface{}{}
  for _, reservation := range instances {
  for _, instance := range reservation.Instances {
+ if shouldExcludeInstance(instance, conn.Filters.Ec2DiscoveryFilters) {
+ continue
+ }
  mqlDevices := []interface{}{}
  for i := range instance.BlockDeviceMappings {
  device := instance.BlockDeviceMappings[i]
@@ -1769,3 +1775,31 @@ func (a *mqlAwsEc2Vpnconnection) id() (string, error) {
 func (a *mqlAwsEc2Vgwtelemetry) id() (string, error) {
  return a.OutsideIpAddress.Data, nil
 }
+
+// true if the instance should be excluded from results. filtering for excluded regions should happen before we retrieve the EC2 instance.
+func shouldExcludeInstance(instance ec2types.Instance, filters connection.Ec2DiscoveryFilters) bool {
+ for _, id := range filters.ExcludeInstanceIds {
+ if instance.InstanceId != nil && *instance.InstanceId == id {
+ return true
+ }
+ }
+ for k, v := range filters.ExcludeTags {
+ for _, iTag := range instance.Tags {
+ if iTag.Key != nil && *iTag.Key == k &&
+ iTag.Value != nil && *iTag.Value == v {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func removeElement(slice []string, value string) []string {
+ result := []string{}
+ for _, v := range slice {
+ if v != value {
+ result = append(result, v)
+ }
+ }
+ return result
+}

providers/aws/resources/aws_ec2_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package resources
+
+import (
+ "testing"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
+ "github.com/stretchr/testify/require"
+ "go.mondoo.com/cnquery/v11/providers/aws/connection"
+)
+
+func TestShouldExcludeInstance(t *testing.T) {
+ instance := ec2types.Instance{
+ InstanceId: aws.String("iid"),
+ Tags: []ec2types.Tag{
+ {
+ Key: aws.String("key-1"),
+ Value: aws.String("val-1"),
+ },
+ {
+ Key: aws.String("key-2"),
+ Value: aws.String("val-2"),
+ },
+ },
+ }
+
+ t.Run("should exclude instance by id", func(t *testing.T) {
+ filters := connection.Ec2DiscoveryFilters{
+ ExcludeInstanceIds: []string{
+ "iid",
+ },
+ ExcludeTags: map[string]string{
+ "key-3": "val3",
+ },
+ }
+ require.True(t, shouldExcludeInstance(instance, filters))
+ })
+
+ t.Run("should exclude instance by matching tag", func(t *testing.T) {
+ filters := connection.Ec2DiscoveryFilters{
+ ExcludeInstanceIds: []string{
+ "iid-2",
+ },
+ ExcludeTags: map[string]string{
+ "key-2": "val2",
+ },
+ }
+ require.False(t, shouldExcludeInstance(instance, filters))
+ })
+
+ t.Run("should not exclude instance with only a matching tag key", func(t *testing.T) {
+ filters := connection.Ec2DiscoveryFilters{
+ ExcludeInstanceIds: []string{
+ "iid-2",
+ },
+ ExcludeTags: map[string]string{
+ "key-2": "val3",
+ "key-3": "val3",
+ },
+ }
+ require.False(t, shouldExcludeInstance(instance, filters))
+ })
+
+ t.Run("should not exclude instance when instance id and tags don't match", func(t *testing.T) {
+ filters := connection.Ec2DiscoveryFilters{
+ ExcludeInstanceIds: []string{
+ "iid-2",
+ },
+ ExcludeTags: map[string]string{
+ "key-3": "val3",
+ },
+ }
+ require.False(t, shouldExcludeInstance(instance, filters))
+ })
+}