From fb5b87f71acb1337849256e1b4baea721e414a53 Mon Sep 17 00:00:00 2001 From: isan_rivkin Date: Sat, 18 Mar 2023 16:55:24 +0200 Subject: [PATCH] add support for multiple aws sessions (#22) --- cmd/acm.go | 224 +++++++++++++++++--------------- cmd/ddb.go | 80 +++++++----- cmd/r53.go | 56 +++++--- cmd/root.go | 2 +- cmd/s3.go | 168 +++++++++++++++--------- lib/awsu/auth.go | 26 +++- lib/search/s3search/searcher.go | 14 +- 7 files changed, 347 insertions(+), 223 deletions(-) diff --git a/cmd/acm.go b/cmd/acm.go index f220a67..a3fefd9 100644 --- a/cmd/acm.go +++ b/cmd/acm.go @@ -30,6 +30,7 @@ import ( ) var ( + acmMultiAWSProfile *[]string awsRegion string filterQuery string acmFilterDomains *bool @@ -55,139 +56,156 @@ var acmCmd = &cobra.Command{ - Certificate ID surf acm -q some-acm-id --filter-id + + - Multiple AWS Profiles/Regions + + surf acm -q my-comain.com --aws-session profile1,region1 --aws-session profile2,region2 + `, Run: func(cmd *cobra.Command, args []string) { tui := buildTUI() - auth, err := awsu.NewSessionInput(awsProfile, awsRegion) + sessionInputs, err := resolveAWSSessions(acmMultiAWSProfile, awsProfile, awsRegion) if err != nil { - log.Fatalf("failed creating session in AWS %s", err.Error()) + log.WithError(err).Fatalf("failed building input for AWS session") } - - acmClient, err := awsu.NewACM(auth) + auths, err := awsu.NewSessionInputMatrix(sessionInputs) if err != nil { - log.Fatalf("failed creating ACM client %s", err.Error()) + log.WithError(err).Fatalf("failed creating session in AWS") } + for _, auth := range auths { + // auth, err := awsu.NewSessionInput(awsProfile, awsRegion) - api := awsu.NewAcmClient(acmClient) - parallel := 30 - m := search.NewDefaultRegexMatcher() + if err != nil { + log.Fatalf("failed creating session in AWS %s", err.Error()) + } - tui.GetLoader().Start("searching acm", "", "green") + acmClient, err := awsu.NewACM(auth) - result, err := api.ListAndFilter(parallel, true, func(c *acm.CertificateDetail) bool { - if *acmFilterAllOptions { - *acmFilterAttachedResources = true - *acmFilterDomains = true - *acmFilterID = true + if err != nil { + log.Fatalf("failed creating ACM client %s", err.Error()) } - if *acmFilterDomains { - domains := aws.StringValueSlice(c.SubjectAlternativeNames) - for _, d := range domains { - if isMatch, _ := m.IsMatch(filterQuery, d); isMatch { - return true + api := awsu.NewAcmClient(acmClient) + parallel := 30 + m := search.NewDefaultRegexMatcher() + + tui.GetLoader().Start("searching acm", "", "green") + + result, err := api.ListAndFilter(parallel, true, func(c *acm.CertificateDetail) bool { + if *acmFilterAllOptions { + *acmFilterAttachedResources = true + *acmFilterDomains = true + *acmFilterID = true + } + + if *acmFilterDomains { + domains := aws.StringValueSlice(c.SubjectAlternativeNames) + for _, d := range domains { + if isMatch, _ := m.IsMatch(filterQuery, d); isMatch { + return true + } } } - } - if *acmFilterAttachedResources { - usedBy := aws.StringValueSlice(c.InUseBy) - for _, arn := range usedBy { - if isMatch, _ := m.IsMatch(filterQuery, arn); isMatch { - return true + if *acmFilterAttachedResources { + usedBy := aws.StringValueSlice(c.InUseBy) + for _, arn := range usedBy { + if isMatch, _ := m.IsMatch(filterQuery, arn); isMatch { + return true + } } } - } - if *acmFilterID { - if isMatch, _ := m.IsMatch(filterQuery, aws.StringValue(c.CertificateArn)); isMatch { - return true + if *acmFilterID { + if isMatch, _ := m.IsMatch(filterQuery, aws.StringValue(c.CertificateArn)); isMatch { + return true + } } - } - return false - }) + return false + }) - tui.GetLoader().Stop() + tui.GetLoader().Stop() - if err != nil { - log.WithError(err).Fatal("failed listing acm certificates") - } + if err != nil { + log.WithError(err).Fatal("failed listing acm certificates") + } - certs := result.Certificates - sort.SliceStable(certs, func(i, j int) bool { - c1 := certs[i] - c2 := certs[j] - - c1Create := aws.TimeValue(c1.CreatedAt) - c2Create := aws.TimeValue(c2.CreatedAt) - - return c2Create.After(c1Create) - - }) - - for _, c := range result.Certificates { - - arn := aws.StringValue(c.CertificateArn) - splitted := strings.Split(arn, "/") - id := splitted[len(splitted)-1] - url := awsu.GenerateACMWebURL(auth.EffectiveRegion, id) - status := aws.StringValue(c.Status) - domain := aws.StringValue(c.DomainName) - inUseBy := aws.StringValueSlice(c.InUseBy) - created := aws.TimeValue(c.CreatedAt) - notAfter := aws.TimeValue(c.NotAfter) - - // date expiration - - expireDays := time.Until(notAfter).Hours() / 24 - // status pretty output consolidation - validationMethodsMapper := map[string]bool{} - validationStatusMapper := map[string]bool{} - validationMethods := "" - validationStatus := "" - if c.DomainValidationOptions != nil { - for _, o := range c.DomainValidationOptions { - m := aws.StringValue(o.ValidationMethod) - validationMethodsMapper[m] = true - - s := aws.StringValue(o.ValidationStatus) - validationStatusMapper[s] = true + certs := result.Certificates + sort.SliceStable(certs, func(i, j int) bool { + c1 := certs[i] + c2 := certs[j] + + c1Create := aws.TimeValue(c1.CreatedAt) + c2Create := aws.TimeValue(c2.CreatedAt) + + return c2Create.After(c1Create) + + }) + + for _, c := range result.Certificates { + + arn := aws.StringValue(c.CertificateArn) + splitted := strings.Split(arn, "/") + id := splitted[len(splitted)-1] + url := awsu.GenerateACMWebURL(auth.EffectiveRegion, id) + status := aws.StringValue(c.Status) + domain := aws.StringValue(c.DomainName) + inUseBy := aws.StringValueSlice(c.InUseBy) + created := aws.TimeValue(c.CreatedAt) + notAfter := aws.TimeValue(c.NotAfter) + + // date expiration + + expireDays := time.Until(notAfter).Hours() / 24 + // status pretty output consolidation + validationMethodsMapper := map[string]bool{} + validationStatusMapper := map[string]bool{} + validationMethods := "" + validationStatus := "" + if c.DomainValidationOptions != nil { + for _, o := range c.DomainValidationOptions { + m := aws.StringValue(o.ValidationMethod) + validationMethodsMapper[m] = true + + s := aws.StringValue(o.ValidationStatus) + validationStatusMapper[s] = true + } } - } - if len(validationStatusMapper) > 1 { - validationStatus = "Partial" - } else { - for s := range validationStatusMapper { - validationStatus = s + if len(validationStatusMapper) > 1 { + validationStatus = "Partial" + } else { + for s := range validationStatusMapper { + validationStatus = s + } } - } - for m := range validationMethodsMapper { - validationMethods += m + " |" - } + for m := range validationMethodsMapper { + validationMethods += m + " |" + } - labelsOrder := []string{"Domain", "URL", "Status"} + labelsOrder := []string{"Domain", "URL", "Status"} - certInfo := map[string]string{ - "Domain": domain, - "URL": url, - "Status": status, - } + certInfo := map[string]string{ + "Domain": domain, + "URL": url, + "Status": status, + } - if getLogLevelFromVerbosity() >= log.DebugLevel { - labelsOrder = append(labelsOrder, []string{"Created", "Expire In", "Validation"}...) - certInfo["Created"] = created.String() - certInfo["Expire In"] = fmt.Sprintf("%d", int(expireDays)) - certInfo["Validation"] = fmt.Sprintf("%s [%s]", validationMethods, validationStatus) - for i, arn := range inUseBy { - useByLabel := fmt.Sprintf("Used By %d", i) - certInfo[useByLabel] = arn - labelsOrder = append(labelsOrder, useByLabel) + if getLogLevelFromVerbosity() >= log.DebugLevel { + labelsOrder = append(labelsOrder, []string{"Created", "Expire In", "Validation"}...) + certInfo["Created"] = created.String() + certInfo["Expire In"] = fmt.Sprintf("%d", int(expireDays)) + certInfo["Validation"] = fmt.Sprintf("%s [%s]", validationMethods, validationStatus) + for i, arn := range inUseBy { + useByLabel := fmt.Sprintf("Used By %d", i) + certInfo[useByLabel] = arn + labelsOrder = append(labelsOrder, useByLabel) + } } - } - tui.GetTable().PrintInfoBox(certInfo, labelsOrder, false) + tui.GetTable().PrintInfoBox(certInfo, labelsOrder, false) + } } }, } @@ -198,7 +216,7 @@ func init() { acmCmd.PersistentFlags().StringVarP(&awsProfile, "profile", "p", getDefaultProfileEnvVar(), "~/.aws/credentials chosen account") acmCmd.PersistentFlags().StringVarP(&awsRegion, "region", "r", "", "~/.aws/config default region if empty") acmCmd.PersistentFlags().StringVarP(&filterQuery, "query", "q", "", "filter query regex supported") - + acmMultiAWSProfile = acmCmd.PersistentFlags().StringArray("aws-session", []string{}, "search in multiple aws profiles & regions (comma separated: --aws-session default,us-east-1 --aws-session dev-account,us-west-2) - overrides --profile and --region") acmFilterDomains = acmCmd.PersistentFlags().Bool("filter-domains", true, "compare query input against all subject names i.e domains") acmFilterID = acmCmd.PersistentFlags().Bool("filter-id", false, "compare query input against all acm arn's") acmFilterAttachedResources = acmCmd.PersistentFlags().Bool("filter-used-by", false, "compare query input against arn's using the acm certificate i.e load balancer") diff --git a/cmd/ddb.go b/cmd/ddb.go index b322577..c116826 100644 --- a/cmd/ddb.go +++ b/cmd/ddb.go @@ -39,6 +39,7 @@ var ( ddbStopOnFirstMatch *bool ddbFailFast *bool sanitizeOutput *bool + ddbMultiAWSProfile *[]string ) var validDDBOutputs = map[string]bool{ @@ -62,6 +63,10 @@ Search free text patterns inside Bytes, Binary, Protobuf, Base64 and Json format $surf ddb -q val -t table -p my-aws-profile -r us-east-1 +=== use --aws-sessiohn to search multiple aws profiles/region === + + $surf ddb -t table -q val --aws-session "profile1,region2" --aws-session "profile2,region3" + === search all tables with production in their name, where the data containing the pattern val === $surf ddb -q val --all-tables -t production @@ -95,48 +100,59 @@ Search free text patterns inside Bytes, Binary, Protobuf, Base64 and Json format } tui := buildTUI() - // MARSHAL ATTRIBUTES UTILITY https://docs.aws.amazon.com/sdk-for-go/api/service/dynamodb/dynamodbattribute/ - auth, err := awsu.NewSessionInput(awsProfile, awsRegion) + sessionInputs, err := resolveAWSSessions(ddbMultiAWSProfile, awsProfile, awsRegion) if err != nil { - log.WithError(err).Fatalf("failed creating session in AWS") + log.WithError(err).Fatalf("failed building input for AWS session") } - awsRegion = auth.EffectiveRegion - - client, err := awsu.NewDDB(auth) + auths, err := awsu.NewSessionInputMatrix(sessionInputs) if err != nil { - log.WithError(err).Fatalf("failed creating ddb session") + log.WithError(err).Fatalf("failed creating session in AWS") } - ddb := awsu.NewDDBClient(client) - if *ddbListTables { - tui.GetLoader().Start("listing dynamodb tables", "", "green") - if err := listDDBTables(ddb, true, *ddbIncludeGlobalTables, tui); err != nil { - log.WithError(err).Error("failed listing tables") - } - return - } else { - if *ddbMatchAll { - ddbQuery = "\\..*" - } + for _, auth := range auths { - parallel := 30 - m := common.NewDefaultRegexMatcher() - p := search.NewParserFactory() - s := search.NewSearcher[awsu.DDBApi, common.Matcher](ddb, m, p) - i, err := search.NewSearchInput(tableNamePattern, ddbQuery, *ddbFailFast, *ddbIncludeGlobalTables, *ddbStopOnFirstMatch, search.ObjectMatch, parallel) - if err != nil { - log.WithError(err).Error("failed creating search input") - } - tui.GetLoader().Start("searching dynamodb", "", "green") - output, err := s.Search(i) - tui.GetLoader().Stop() + // MARSHAL ATTRIBUTES UTILITY https://docs.aws.amazon.com/sdk-for-go/api/service/dynamodb/dynamodbattribute/ + //auth, err := awsu.NewSessionInput(awsProfile, awsRegion) + + awsRegion = auth.EffectiveRegion + + client, err := awsu.NewDDB(auth) if err != nil { - log.WithError(err).Fatalf("failed running search on dynamodb") + log.WithError(err).Fatalf("failed creating ddb session") + } + ddb := awsu.NewDDBClient(client) + if *ddbListTables { + tui.GetLoader().Start("listing dynamodb tables", "", "green") + if err := listDDBTables(ddb, true, *ddbIncludeGlobalTables, tui); err != nil { + log.WithError(err).Error("failed listing tables") + } + return + } else { + + if *ddbMatchAll { + ddbQuery = "\\..*" + } + + parallel := 30 + m := common.NewDefaultRegexMatcher() + p := search.NewParserFactory() + s := search.NewSearcher[awsu.DDBApi, common.Matcher](ddb, m, p) + i, err := search.NewSearchInput(tableNamePattern, ddbQuery, *ddbFailFast, *ddbIncludeGlobalTables, *ddbStopOnFirstMatch, search.ObjectMatch, parallel) + if err != nil { + log.WithError(err).Error("failed creating search input") + } + tui.GetLoader().Start("searching dynamodb", "", "green") + output, err := s.Search(i) + tui.GetLoader().Stop() + + if err != nil { + log.WithError(err).Fatalf("failed running search on dynamodb") + } + printDDBSearchOutput(i, output, tui) } - printDDBSearchOutput(i, output, tui) } }, } @@ -176,6 +192,7 @@ func printDDBSearchOutputAsJSON(input *search.Input, output *search.Output) { log.WithError(err).Fatalf("failed parsing map to json container") } fmt.Println(printer.PrettyJson(obj.String())) + } func printDDBSearchOutput(input *search.Input, output *search.Output, tui printer.TuiController[printer.Loader, printer.Table]) { @@ -252,6 +269,7 @@ func init() { ddbCmd.PersistentFlags().StringVarP(&ddbQuery, "query", "q", "", "filter query regex supported (if used with --all will error)") ddbCmd.PersistentFlags().StringVarP(&tableNamePattern, "table", "t", "", "regex table pattern name to match") ddbCmd.PersistentFlags().StringVarP(&ddbOutputType, "out", "o", "pretty", "output format [json, pretty]") + ddbMultiAWSProfile = ddbCmd.PersistentFlags().StringArray("aws-session", []string{}, "search in multiple aws profiles & regions (comma separated: --aws-session default,us-east-1 --aws-session dev-account,us-west-2) - overrides --profile and --region") ddbFailFast = ddbCmd.Flags().Bool("fail-fast", false, "fail on first error seen") ddbListTables = ddbCmd.Flags().Bool("list-tables", false, "list all available tables") ddbMatchAll = ddbCmd.Flags().Bool("all", false, "match all data (same as using -q '\\\\..*') if used with --query will error") diff --git a/cmd/r53.go b/cmd/r53.go index 5e20a31..1d89e80 100644 --- a/cmd/r53.go +++ b/cmd/r53.go @@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -16,7 +16,7 @@ limitations under the License. package cmd import ( - "fmt" + "strings" "github.com/aws/aws-sdk-go/aws" "github.com/isan-rivkin/surf/lib/awsu" @@ -25,6 +25,7 @@ import ( ) var ( + r53MultipleProfiles string recusiveSearchMaxDepth *int recordInput *string awsProfile string @@ -34,27 +35,51 @@ var ( // r53Cmd represents the r53 command var r53Cmd = &cobra.Command{ - Use: "r53 -q '*.some.dns.record.com'\n", + Use: `r53 -q `, Short: "Query route53 to get your dns record values", - Long: `Query Route53 to get all sorts of information about a dns record. - r53 will use your default AWS credentials`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("r53 called") + Long: ` + Query Route53 to get all sorts of information about a dns record. + + === search the target of '*.address.com' === + + $surf r53 -q '*.address.com' + + === skip Name Server validation (NS records) === + + $surf r53 -q '*.address.com' --ns-skip + + === search the target of '*.address.com' in multiple AWS profiles === + + $surf r53 -q '*.address.com' --aws-profiles default,prod,dev + `, + Run: func(cmd *cobra.Command, args []string) { debug := false recurse := true - in, err := awsu.NewR53Input(aws.StringValue(recordInput), awsProfile, debug, muteR53Logs, skipNSVerification, recurse, *recusiveSearchMaxDepth) + // TODO(multiple aws account not consolidated) + var awsProfiles []string + if r53MultipleProfiles != "" { + awsProfiles = strings.Split(r53MultipleProfiles, ",") + } else { + awsProfiles = append(awsProfiles, awsProfile) + } + if len(awsProfiles) < 1 { + log.Fatal("no aws profiles provided") + } - if err != nil { + for _, awsProf := range awsProfiles { + in, err := awsu.NewR53Input(aws.StringValue(recordInput), awsProf, debug, muteR53Logs, skipNSVerification, recurse, *recusiveSearchMaxDepth) - log.WithError(err).Fatal("failed creating r53 input") - } - _, err = awsu.SearchRoute53(in) + if err != nil { - if err != nil { - log.WithError(err).Fatal("failed searching r53") - } + log.WithError(err).Fatal("failed creating r53 input") + } + _, err = awsu.SearchRoute53(in) + if err != nil { + log.WithError(err).Errorf("failed searching r53 in profile %s", awsProf) + } + } }, } @@ -64,6 +89,7 @@ func init() { r53Cmd.PersistentFlags().StringVarP(&awsProfile, "profile", "p", getDefaultProfileEnvVar(), "~/.aws/credentials chosen account") r53Cmd.PersistentFlags().BoolVar(&muteR53Logs, "mute-logs", false, "if flag set then logs from route53-cli sdk will be muted") r53Cmd.PersistentFlags().BoolVar(&skipNSVerification, "ns-skip", false, "if set then nameservers will not be verified against the hosted zone result") + r53Cmd.PersistentFlags().StringVar(&r53MultipleProfiles, "aws-profiles", "", "search in multiple aws profiles (comma separated: --aws-profiles prod,dev,staging) overrides --profile") maxDepth := 3 r53Cmd.PersistentFlags().IntVarP(&maxDepth, "max-depth", "d", maxDepth, "if -R is used then specifies when to stop recursive search depth") recusiveSearchMaxDepth = &maxDepth diff --git a/cmd/root.go b/cmd/root.go index e975463..dea5859 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -31,7 +31,7 @@ import ( ) const ( - AppVersion = "2.1.2" + AppVersion = "2.2.2" AppName = "surf" ) diff --git a/cmd/s3.go b/cmd/s3.go index 63f4eab..721e30f 100644 --- a/cmd/s3.go +++ b/cmd/s3.go @@ -17,6 +17,7 @@ package cmd import ( "fmt" + "strings" "github.com/isan-rivkin/surf/lib/awsu" common "github.com/isan-rivkin/surf/lib/search" @@ -27,10 +28,11 @@ import ( ) var ( - bucketName string - keyPrefix string - s3WebOutput *bool - allowAllBuckets *bool + s3MultiAWSProfile *[]string + bucketName string + keyPrefix string + s3WebOutput *bool + allowAllBuckets *bool ) // s3Cmd represents the s3 command @@ -48,96 +50,139 @@ var s3Cmd = &cobra.Command{ $surf s3 -q my-key --prefix prefix-key -b my-bucket +=== use --aws-session to search in multiple aws profiles/regions === + + $surf s3 -q my-key --aws-session profile1,region1 --aws-session profile2,region2 + === Regex on bucket names to search in === $surf s3 -q '\.json$' -b '^(prod)(.*)-public' + ` + getEnvVarConfig("s3"), Run: func(cmd *cobra.Command, args []string) { tui := buildTUI() - - auth, err := awsu.NewSessionInput(awsProfile, awsRegion) + sessionInputs, err := resolveAWSSessions(s3MultiAWSProfile, awsProfile, awsRegion) + if err != nil { + log.WithError(err).Fatalf("failed building input for AWS session") + } + auths, err := awsu.NewSessionInputMatrix(sessionInputs) if err != nil { log.WithError(err).Fatalf("failed creating session in AWS") } - s3Client, err := awsu.NewS3(auth) + for _, auth := range auths { - if err != nil { - log.WithError(err).Fatalf("failed creating S3 client") - } + s3Client, err := awsu.NewS3(auth) - api := awsu.NewS3Client(s3Client) - parallel := 30 + if err != nil { + log.WithError(err).Fatalf("failed creating S3 client") + } - bucketName = *getEnvOrOverride(&bucketName, EnvKeyS3DefaultBucket) + api := awsu.NewS3Client(s3Client) + parallel := 30 - input := search.NewSearchInput(bucketName, keyPrefix, filterQuery, parallel, *allowAllBuckets) - m := common.NewDefaultRegexMatcher() - s := search.NewSearcher[awsu.S3API, common.Matcher](api, m) + bucketName = *getEnvOrOverride(&bucketName, EnvKeyS3DefaultBucket) - tui.GetLoader().Start("searching s3", "", "green") + input := search.NewSearchInput(bucketName, keyPrefix, filterQuery, parallel, *allowAllBuckets) + m := common.NewDefaultRegexMatcher() + s := search.NewSearcher[awsu.S3API, common.Matcher](api, m) - output, err := s.Search(input) + tui.GetLoader().Start("searching s3", "", "green") - tui.GetLoader().Stop() + output, err := s.Search(input) - if err != nil { - msg := "error while searching keys" - if err.Error() == search.TooManyBucketsErr { - msg = "too many buckets, use --bucket to filter buckets or use --all-buckets to allow anyway (discouraged)" + tui.GetLoader().Stop() + + if err != nil { + msg := "error while searching keys" + if err.Error() == search.TooManyBucketsErr { + msg = "too many buckets, use --bucket to filter buckets or use --all-buckets to allow anyway (discouraged)" + } + log.WithError(err).Fatalf(msg) + } + + if !*s3WebOutput { + for bucketName, matchedKeys := range output.BucketToMatches { + for _, k := range matchedKeys { + fmt.Printf("s3://%s/%s\n", bucketName, k) + } + } + return + } + labelsOrder := []string{"Match", "Bucket", "AWS Session", "Num #"} + labelsOrderSummary := []string{"Bucket", "Query"} + tables := []map[string]string{} + summaryTable := map[string]string{ + "Bucket": "Num #", + "Query": filterQuery, + } + if keyPrefix != "" { + summaryTable["Prefix"] = keyPrefix } - log.WithError(err).Fatalf(msg) - } - if !*s3WebOutput { for bucketName, matchedKeys := range output.BucketToMatches { + bucketInfo := map[string]string{} + matches := fmt.Sprintf("%d", len(matchedKeys)) + bucketInfo["Bucket"] = bucketName + bucketInfo["Num #"] = matches + bucketInfo["AWS Session"] = fmt.Sprintf("%s %s", auth.EffectiveProfile, auth.EffectiveRegion) + summaryTable[bucketName] = matches + labelsOrderSummary = append(labelsOrderSummary, bucketName) + + if len(matchedKeys) == 0 { + continue + } + for _, k := range matchedKeys { - fmt.Printf("s3://%s/%s\n", bucketName, k) + url := awsu.GenerateS3WebURL(bucketName, auth.EffectiveRegion, k) + url = printer.FmtURL(url) + val := bucketInfo["Match"] + bucketInfo["Match"] = fmt.Sprintf("%s\n%s", val, url) } + tables = append(tables, bucketInfo) } - return - } - labelsOrder := []string{"Match", "Bucket", "Num #"} - labelsOrderSummary := []string{"Bucket", "Query"} - tables := []map[string]string{} - summaryTable := map[string]string{ - "Bucket": "Num #", - "Query": filterQuery, - } - if keyPrefix != "" { - summaryTable["Prefix"] = keyPrefix - } - - for bucketName, matchedKeys := range output.BucketToMatches { - bucketInfo := map[string]string{} - matches := fmt.Sprintf("%d", len(matchedKeys)) - bucketInfo["Bucket"] = bucketName - bucketInfo["Num #"] = matches - summaryTable[bucketName] = matches - labelsOrderSummary = append(labelsOrderSummary, bucketName) - if len(matchedKeys) == 0 { - continue + for _, t := range tables { + tui.GetTable().PrintInfoBox(t, labelsOrder, false) } - for _, k := range matchedKeys { - url := awsu.GenerateS3WebURL(bucketName, auth.EffectiveRegion, k) - url = printer.FmtURL(url) - val := bucketInfo["Match"] - bucketInfo["Match"] = fmt.Sprintf("%s\n%s", val, url) + if getLogLevelFromVerbosity() >= log.DebugLevel { + tui.GetTable().PrintInfoBox(summaryTable, labelsOrderSummary, false) } - tables = append(tables, bucketInfo) } + }, +} - for _, t := range tables { - tui.GetTable().PrintInfoBox(t, labelsOrder, false) - } +func resolveAWSSessions(multiple *[]string, profile, region string) ([]*awsu.AWSSessionInput, error) { + if multiple != nil && len(*multiple) > 0 { + log.Debugf("using multiple aws sessions, got %v", *multiple) + return inputToMultipleAWSSessions(*multiple) + } + log.Debugf("using since aws sessions profile=%s region=%s", profile, region) + return []*awsu.AWSSessionInput{ + { + Profile: profile, + Region: region, + }, + }, nil +} - if getLogLevelFromVerbosity() >= log.DebugLevel { - tui.GetTable().PrintInfoBox(summaryTable, labelsOrderSummary, false) +func inputToMultipleAWSSessions(input []string) ([]*awsu.AWSSessionInput, error) { + var sessions []*awsu.AWSSessionInput + for _, pair := range input { + tuple := strings.Split(pair, ",") + if len(tuple) != 2 { + return nil, fmt.Errorf("invalid input, must be in pairs of profile,region; got %v", input) } - }, + profile := tuple[0] + region := tuple[1] + sessions = append(sessions, &awsu.AWSSessionInput{ + Profile: profile, + Region: region, + }) + } + return sessions, nil } func init() { @@ -147,6 +192,7 @@ func init() { s3Cmd.PersistentFlags().StringVarP(&keyPrefix, "prefix", "k", "", "key prefix to start search from") s3Cmd.PersistentFlags().StringVarP(&filterQuery, "query", "q", "", "filter query regex supported") s3Cmd.PersistentFlags().StringVarP(&bucketName, "bucket", "b", "", "bucket query to start from search") + s3MultiAWSProfile = s3Cmd.PersistentFlags().StringArray("aws-session", []string{}, "search in multiple aws profiles & regions (comma separated: --aws-session default,us-east-1 --aws-session dev-account,us-west-2) - overrides --profile and --region") s3WebOutput = s3Cmd.PersistentFlags().Bool("output-url", true, "Output the results with clickable URL links") allowAllBuckets = s3Cmd.PersistentFlags().Bool("all-buckets", false, "when not providing --bucket pattern this flag required to allow all buckets search") s3Cmd.MarkPersistentFlagRequired("query") diff --git a/lib/awsu/auth.go b/lib/awsu/auth.go index 3526e3a..f2dee23 100644 --- a/lib/awsu/auth.go +++ b/lib/awsu/auth.go @@ -14,9 +14,27 @@ import ( ) type AuthInput struct { - Provider client.ConfigProvider - Configs []*aws.Config - EffectiveRegion string + Provider client.ConfigProvider + Configs []*aws.Config + EffectiveRegion string + EffectiveProfile string +} + +type AWSSessionInput struct { + Profile string + Region string +} + +func NewSessionInputMatrix(inputs []*AWSSessionInput) ([]*AuthInput, error) { + var out []*AuthInput + for _, input := range inputs { + auth, err := NewSessionInput(input.Profile, input.Region) + if err != nil { + return nil, err + } + out = append(out, auth) + } + return out, nil } func NewSessionInput(profile, region string) (*AuthInput, error) { @@ -33,7 +51,7 @@ func NewSessionInput(profile, region string) (*AuthInput, error) { } conf := []*aws.Config{c} - return &AuthInput{Provider: sess, Configs: conf, EffectiveRegion: effectiveRegion}, nil + return &AuthInput{Provider: sess, Configs: conf, EffectiveRegion: effectiveRegion, EffectiveProfile: profile}, nil } func NewACM(in *AuthInput) (*acm.ACM, error) { diff --git a/lib/search/s3search/searcher.go b/lib/search/s3search/searcher.go index eaf6c7f..5290acd 100644 --- a/lib/search/s3search/searcher.go +++ b/lib/search/s3search/searcher.go @@ -126,8 +126,8 @@ func (s *DefaultSearcher[CC, Matcher]) Search(i *Input) (*Output, error) { res.Keys = append(res.Keys, aws.StringValue(k.Key)) } } - asyncResults <- res } + asyncResults <- res }) } @@ -135,19 +135,17 @@ func (s *DefaultSearcher[CC, Matcher]) Search(i *Input) (*Output, error) { size := len(targetBuckets) counter := 1 for r := range asyncResults { + counter++ + if r.Err != nil { - log.WithError(err).WithField("bucket", r.Bucket).Error("failed describing keys") - continue + log.WithError(r.Err).WithField("bucket", r.Bucket).Error("failed searching keys in bucket (potential fix: sure the target bucket is in the target region)") + } else { + filteredResult.BucketToMatches[r.Bucket] = r.Keys } - filteredResult.BucketToMatches[r.Bucket] = r.Keys - if counter >= size { break } - - counter++ } - return filteredResult, nil }