diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c230840c03b..8b01cf6c662 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -148,6 +148,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Improve error reporting and fix IPv6 handling of TCP and UDP metric collection. {pull}35996[35996] - Fix handling of NUL-terminated log lines in Fortinet Firewall module. {issue}36026[36026] {pull}36027[36027] - Make redact field configuration recommended in CEL input and log warning if missing. {pull}36008[36008] +- Fix handling of region name configuration in awss3 input {pull}36034[36034] *Heartbeat* diff --git a/x-pack/filebeat/input/awss3/input.go b/x-pack/filebeat/input/awss3/input.go index 4f45f914144..221084881f8 100644 --- a/x-pack/filebeat/input/awss3/input.go +++ b/x-pack/filebeat/input/awss3/input.go @@ -119,10 +119,10 @@ func (in *s3Input) Run(inputContext v2.Context, pipeline beat.Pipeline) error { if err != nil && in.config.RegionName == "" { return fmt.Errorf("failed to get AWS region from queue_url: %w", err) } - if regionName != in.config.RegionName { - inputContext.Logger.Warnf("configured region disagrees with queue_url region: %q != %q: using %[1]q", - in.config.RegionName, regionName) - regionName = in.config.RegionName + var warn regionMismatchError + if errors.As(err, &warn) { + // Warn of mismatch, but go ahead with configured region name. + inputContext.Logger.Warnf("%v: using %q", err, regionName) } in.awsConfig.Region = regionName @@ -306,7 +306,7 @@ func (in *s3Input) createS3Lister(ctx v2.Context, cancelCtx context.Context, cli var errBadQueueURL = errors.New("QueueURL is not in format: https://sqs.{REGION_ENDPOINT}.{ENDPOINT}/{ACCOUNT_NUMBER}/{QUEUE_NAME}") -func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (string, error) { +func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (region string, err error) { // get region from queueURL // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs u, err := url.Parse(queueURL) @@ -317,7 +317,11 @@ func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (str queueHostSplit := strings.SplitN(u.Host, ".", 3) if len(queueHostSplit) == 3 { if queueHostSplit[2] == endpoint || (endpoint == "" && strings.HasPrefix(queueHostSplit[2], "amazonaws.")) { - return queueHostSplit[1], nil + region = queueHostSplit[1] + if defaultRegion != "" && region != defaultRegion { + return defaultRegion, regionMismatchError{queueURLRegion: region, defaultRegion: defaultRegion} + } + return region, nil } } else if defaultRegion != "" { return defaultRegion, nil @@ -326,6 +330,15 @@ func getRegionFromQueueURL(queueURL string, endpoint, defaultRegion string) (str return "", errBadQueueURL } +type regionMismatchError struct { + queueURLRegion string + defaultRegion string +} + +func (e regionMismatchError) Error() string { + return fmt.Sprintf("configured region disagrees with queue_url region: %q != %q", e.queueURLRegion, e.defaultRegion) +} + func getRegionForBucket(ctx context.Context, s3Client *s3.Client, bucketName string) (string, error) { getBucketLocationOutput, err := s3Client.GetBucketLocation(ctx, &s3.GetBucketLocationInput{ Bucket: awssdk.String(bucketName), diff --git a/x-pack/filebeat/input/awss3/input_test.go b/x-pack/filebeat/input/awss3/input_test.go index 02a91022f5e..8a195eb3084 100644 --- a/x-pack/filebeat/input/awss3/input_test.go +++ b/x-pack/filebeat/input/awss3/input_test.go @@ -81,6 +81,19 @@ func TestGetRegionFromQueueURL(t *testing.T) { endpoint: "googlecloud.com", wantErr: errBadQueueURL, }, + { + name: "mismatch_regions_no_default", + queueURL: "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs", + deflt: "", + want: "us-east-1", + }, + { + name: "mismatch_regions", + queueURL: "https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs", + deflt: "ap-west-1", + want: "ap-west-1", + wantErr: regionMismatchError{queueURLRegion: "us-east-1", defaultRegion: "ap-west-1"}, + }, { name: "localstack", queueURL: "http://localhost:4566/000000000000/filebeat-s3-integtest-d9clk9",