🧹 Remove discovery filters parameter from Discover

providers/aws/connection/connection.go

@@ -6,6 +6,7 @@ package connection
 import (
  "context"
  "errors"
+ "strconv"
  "strings"
 
  "github.com/aws/aws-sdk-go-v2/aws"
@@ -53,9 +54,12 @@ type GeneralResourceDiscoveryFilters struct {
 }
 
 type Ec2DiscoveryFilters struct {
- Regions []string
- Tags map[string]string
- InstanceIds []string
+ Regions []string
+ Tags map[string]string
+ InstanceIds []string
+ ExcludeRegions []string
+ ExcludeTags map[string]string
+ ExcludeInstanceIds []string
 }
 type EcrDiscoveryFilters struct {
  Tags []string
@@ -128,7 +132,7 @@ func NewAwsConnection(id uint32, asset *inventory.Asset, conf *inventory.Config)
 
 func parseOptsToFilters(opts map[string]string) DiscoveryFilters {
  d := DiscoveryFilters{
- Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}},
+ Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}},
  EcsDiscoveryFilters: EcsDiscoveryFilters{},
  EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}},
  GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}},
@@ -137,16 +141,37 @@ func parseOptsToFilters(opts map[string]string) DiscoveryFilters {
  switch {
  case strings.HasPrefix(k, "ec2:tag:"):
  d.Ec2DiscoveryFilters.Tags[strings.TrimPrefix(k, "ec2:tag:")] = v
- case k == "ec2:region":
- d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, v)
- case k == "all:region", k == "region":
- d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, v)
- case k == "instance-id":
- d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, v)
+ case strings.HasPrefix(k, "exclude:ec2:tag:"):
+ d.Ec2DiscoveryFilters.ExcludeTags[strings.TrimPrefix(k, "exclude:ec2:tag:")] = v
+ case k == "ec2:regions":
+ d.Ec2DiscoveryFilters.Regions = append(d.Ec2DiscoveryFilters.Regions, strings.Split(v, ",")...)
+ case k == "exclude:ec2:regions":
+ d.Ec2DiscoveryFilters.ExcludeRegions = append(d.Ec2DiscoveryFilters.ExcludeRegions, strings.Split(v, ",")...)
+ case k == "all:regions", k == "regions":
+ d.GeneralDiscoveryFilters.Regions = append(d.GeneralDiscoveryFilters.Regions, strings.Split(v, ",")...)
+ case k == "ec2:instance-ids":
+ d.Ec2DiscoveryFilters.InstanceIds = append(d.Ec2DiscoveryFilters.InstanceIds, strings.Split(v, ",")...)
+ case k == "exclude:ec2:instance-ids":
+ d.Ec2DiscoveryFilters.ExcludeInstanceIds = append(d.Ec2DiscoveryFilters.ExcludeInstanceIds, strings.Split(v, ",")...)
  case strings.HasPrefix(k, "all:tag:"):
  d.GeneralDiscoveryFilters.Tags[strings.TrimPrefix(k, "all:tag:")] = v
- case k == "ecr:tag":
- d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, v)
+ case k == "ecr:tags":
+ d.EcrDiscoveryFilters.Tags = append(d.EcrDiscoveryFilters.Tags, strings.Split(v, ",")...)
+ case k == "ecs:only-running-containers":
+ parsed, err := strconv.ParseBool(v)
+ if err == nil {
+ d.EcsDiscoveryFilters.OnlyRunningContainers = parsed
+ }
+ case k == "ecs:discover-instances":
+ parsed, err := strconv.ParseBool(v)
+ if err == nil {
+ d.EcsDiscoveryFilters.DiscoverInstances = parsed
+ }
+ case k == "ecs:discover-images":
+ parsed, err := strconv.ParseBool(v)
+ if err == nil {
+ d.EcsDiscoveryFilters.DiscoverImages = parsed
+ }
  }
  }
  return d

providers/aws/connection/connection_test.go

@@ -0,0 +1,118 @@
+// Copyright (c) Mondoo, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package connection
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// testParseOptsToFilters accepts a map which doesn't guarantee a deterministic iteration order. this means that slices
+// in the parsed filters need to be compared individually ensuring their elements match regardless of their order.
+func compareFilters(t *testing.T, expected, actual DiscoveryFilters) {
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.Regions, actual.Ec2DiscoveryFilters.Regions)
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeRegions, actual.Ec2DiscoveryFilters.ExcludeRegions)
+
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.InstanceIds, actual.Ec2DiscoveryFilters.InstanceIds)
+ require.ElementsMatch(t, expected.Ec2DiscoveryFilters.ExcludeInstanceIds, actual.Ec2DiscoveryFilters.ExcludeInstanceIds)
+
+ require.Equal(t, expected.Ec2DiscoveryFilters.Tags, actual.Ec2DiscoveryFilters.Tags)
+ require.Equal(t, expected.Ec2DiscoveryFilters.ExcludeTags, actual.Ec2DiscoveryFilters.ExcludeTags)
+
+ require.Equal(t, expected.EcsDiscoveryFilters, actual.EcsDiscoveryFilters)
+
+ require.ElementsMatch(t, expected.EcrDiscoveryFilters.Tags, actual.EcrDiscoveryFilters.Tags)
+
+ require.ElementsMatch(t, expected.GeneralDiscoveryFilters.Regions, actual.GeneralDiscoveryFilters.Regions)
+ require.Equal(t, expected.GeneralDiscoveryFilters.Tags, actual.GeneralDiscoveryFilters.Tags)
+}
+
+func TestParseOptsToFilters(t *testing.T) {
+ t.Run("all opts are mapped to discovery filters correctly", func(t *testing.T) {
+ opts := map[string]string{
+ // Ec2DiscoveryFilters.Tags
+ "ec2:tag:key1": "val1",
+ "ec2:tag:key2": "val2",
+ // Ec2DiscoveryFilters.ExcludeTags
+ "exclude:ec2:tag:key1": "val1",
+ "exclude:ec2:tag:key2": "val2",
+ // Ec2DiscoveryFilters.Regions
+ "ec2:regions": "us-east-1,us-west-1",
+ // Ec2DiscoveryFilters.ExcludeRegions
+ "exclude:ec2:regions": "us-east-1,us-west-1",
+ // Ec2DiscoveryFilters.InstanceIds
+ "ec2:instance-ids": "iid-1,iid-2",
+ // Ec2DiscoveryFilters.ExcludeInstanceIds
+ "exclude:ec2:instance-ids": "iid-1,iid-2",
+ // GeneralDiscoveryFilters.Regions
+ "all:regions": "us-east-1,us-west-1,eu-west-1",
+ // GeneralDiscoveryFilters.Tags
+ "all:tag:key1": "val1",
+ "all:tag:key2": "val2",
+ // EcrDiscoveryFilters.Tags
+ "ecr:tags": "tag1,tag2",
+ // EcsDiscoveryFilters
+ "ecs:only-running-containers": "true",
+ "ecs:discover-images": "T",
+ "ecs:discover-instances": "false",
+ }
+ expected := DiscoveryFilters{
+ Ec2DiscoveryFilters: Ec2DiscoveryFilters{
+ Regions: []string{
+ "us-east-1", "us-west-1",
+ },
+ ExcludeRegions: []string{
+ "us-east-1", "us-west-1",
+ },
+ InstanceIds: []string{
+ "iid-1", "iid-2",
+ },
+ ExcludeInstanceIds: []string{
+ "iid-1", "iid-2",
+ },
+ Tags: map[string]string{
+ "key1": "val1",
+ "key2": "val2",
+ },
+ ExcludeTags: map[string]string{
+ "key1": "val1",
+ "key2": "val2",
+ },
+ },
+ EcsDiscoveryFilters: EcsDiscoveryFilters{
+ OnlyRunningContainers: true,
+ DiscoverImages: true,
+ DiscoverInstances: false,
+ },
+ EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{
+ "tag1", "tag2",
+ }},
+ GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{
+ Regions: []string{
+ "us-east-1", "us-west-1", "eu-west-1",
+ },
+ Tags: map[string]string{
+ "key1": "val1",
+ "key2": "val2",
+ },
+ },
+ }
+
+ actual := parseOptsToFilters(opts)
+ compareFilters(t, expected, actual)
+ })
+
+ t.Run("empty opts are mapped to discovery filters correctly", func(t *testing.T) {
+ expected := DiscoveryFilters{
+ Ec2DiscoveryFilters: Ec2DiscoveryFilters{Tags: map[string]string{}, ExcludeTags: map[string]string{}},
+ EcsDiscoveryFilters: EcsDiscoveryFilters{},
+ EcrDiscoveryFilters: EcrDiscoveryFilters{Tags: []string{}},
+ GeneralDiscoveryFilters: GeneralResourceDiscoveryFilters{Tags: map[string]string{}},
+ }
+
+ actual := parseOptsToFilters(map[string]string{})
+ compareFilters(t, expected, actual)
+ })
+}

providers/aws/provider/provider.go

@@ -83,15 +83,27 @@ func parseFlagsToFiltersOpts(m map[string]*llx.Primitive) map[string]string {
  o := make(map[string]string, 0)
 
  if x, ok := m["filters"]; ok && len(x.Map) != 0 {
+ knownTagPrefixes := []string{
+ "ec2:tag:",
+ "exclude:ec2:tag:",
+ "ec2:regions",
+ "exclude:ec2:regions",
+ "all:regions",
+ "regions",
+ "ec2:instance-ids",
+ "exclude:ec2:instance-ids",
+ "all:tag:",
+ "ecr:tags",
+ "ecs:only-running-containers",
+ "ecs:discover-instances",
+ "ecs:discover-images",
+ }
  for k, v := range x.Map {
- if strings.Contains(k, "tag:") {
- o[k] = string(v.Value)
- }
- if k == "instance-id" {
- o[k] = string(v.Value)
- }
- if strings.Contains(k, "region") {
- o[k] = string(v.Value)
+ for _, prefix := range knownTagPrefixes {
+ if strings.HasPrefix(k, prefix) {
+ o[k] = string(v.Value)
+ break
+ }
  }
  }
  }
@@ -269,5 +281,5 @@ func (s *Service) discover(conn *connection.AwsConnection) (*inventory.Inventory
  return nil, err
  }
 
- return resources.Discover(runtime, conn.Filters)
+ return resources.Discover(runtime)
 }

providers/aws/resources/aws.go

@@ -70,6 +70,16 @@ func Is400AccessDeniedError(err error) bool {
  return false
 }
 
+func Is400InstanceNotFoundError(err error) bool {
+ var respErr *http.ResponseError
+ if errors.As(err, &respErr) {
+ if respErr.HTTPStatusCode() == 400 && (strings.Contains(respErr.Error(), "InvalidInstanceID.NotFound") || strings.Contains(respErr.Error(), "InvalidInstanceID.Malformed")) {
+ return true
+ }
+ }
+ return false
+}
+
 func strMapToInterface(m map[string]string) map[string]interface{} {
  res := map[string]interface{}{}
  for k, v := range m {

providers/aws/resources/aws_ec2.go

@@ -26,6 +26,7 @@ import (
  "go.mondoo.com/cnquery/v11/providers-sdk/v1/util/jobpool"
  "go.mondoo.com/cnquery/v11/providers/aws/connection"
  "go.mondoo.com/cnquery/v11/types"
+ "go.mondoo.com/cnquery/v11/utils/stringx"
 )
 
 func (e *mqlAwsEc2) id() (string, error) {
@@ -800,9 +801,7 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job
  if err != nil {
  return []*jobpool.Job{{Err: err}}
  }
- if len(conn.Filters.Ec2DiscoveryFilters.Regions) > 0 {
- regions = conn.Filters.Ec2DiscoveryFilters.Regions
- }
+ regions = determineApplicableRegions(regions, conn.Filters.Ec2DiscoveryFilters.Regions, conn.Filters.Ec2DiscoveryFilters.ExcludeRegions)
  for _, region := range regions {
  regionVal := region
  f := func() (jobpool.JobResult, error) {
@@ -818,6 +817,13 @@ func (a *mqlAwsEc2) getInstances(conn *connection.AwsConnection) []*jobpool.Job
  log.Warn().Str("region", regionVal).Msg("error accessing region for AWS API")
  return res, nil
  }
+ // AWS returns an error response when trying to find an instance with a specific identifier if it cannot find it in some region.
+ // we do not propagate this error upward because an instance can be found in one region and return an error for all others which
+ // would be the expected behavior.
+ if Is400InstanceNotFoundError(err) {
+ log.Debug().Str("region", regionVal).Msg("could not find instance in region")
+ return res, nil
+ }
  return nil, err
  }
  res, err = a.gatherInstanceInfo(instances, regionVal)
@@ -837,6 +843,9 @@ func (a *mqlAwsEc2) gatherInstanceInfo(instances []ec2types.Reservation, regionV
  res := []interface{}{}
  for _, reservation := range instances {
  for _, instance := range reservation.Instances {
+ if shouldExcludeInstance(instance, conn.Filters.Ec2DiscoveryFilters) {
+ continue
+ }
  mqlDevices := []interface{}{}
  for i := range instance.BlockDeviceMappings {
  device := instance.BlockDeviceMappings[i]
@@ -1769,3 +1778,37 @@ func (a *mqlAwsEc2Vpnconnection) id() (string, error) {
 func (a *mqlAwsEc2Vgwtelemetry) id() (string, error) {
  return a.OutsideIpAddress.Data, nil
 }
+
+// true if the instance should be excluded from results. filtering for excluded regions should happen before we retrieve the EC2 instance.
+func shouldExcludeInstance(instance ec2types.Instance, filters connection.Ec2DiscoveryFilters) bool {
+ for _, id := range filters.ExcludeInstanceIds {
+ if instance.InstanceId != nil && *instance.InstanceId == id {
+ return true
+ }
+ }
+ for k, v := range filters.ExcludeTags {
+ for _, iTag := range instance.Tags {
+ if iTag.Key != nil && *iTag.Key == k &&
+ iTag.Value != nil && *iTag.Value == v {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+// given an initial set of regions, applies the allowed regions filter and the excluded regions filter on it
+// returning a resulting slice of only applicable region to which queries should be targeted
+func determineApplicableRegions(regions, regionsToInclude, regionsToExclude []string) []string {
+ if len(regionsToInclude) > 0 {
+ return regionsToInclude
+ }
+ res := []string{}
+ for _, r := range regions {
+ if !stringx.Contains(regionsToExclude, r) {
+ res = append(res, r)
+ }
+ }
+
+ return res
+}