-
Notifications
You must be signed in to change notification settings - Fork 4.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
* [libbeat] Add lowercase_fields and uppercase_fields processors * [libbeat]: Add lowercase_key processor * only loweercase * changelog * addressed comments minus documentation * addressed comments & documentation * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/alterFieldProcessor.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/alterFieldProcessor.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/alterFieldProcessor.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/lowercase.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/alterFieldProcessor.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/alterFieldProcessor.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/alterFieldProcessor.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/lowercase.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * case insensitive search * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * case insensitive search * using flattenkeys * changelog * cosmetic changes * lowercase with flattenKeys * lowercase with foldvalue * Update CHANGELOG.next.asciidoc Co-authored-by: Denis <denis@rdner.de> * resolved conflicts * added license * full_path search * cosmetic changes * using alterpath * update version * refactor * linter issues * fix * fix * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/alterFieldProcessor.go Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * Update libbeat/processors/actions/docs/lowercase.asciidoc Co-authored-by: Denis <denis@rdner.de> * skip mandatory fields with test * skip mandatory fields * additonal logic to skipping mandatory field * optimization * fix test case * fix flaky test --------- Co-authored-by: davidifr <davidfr.mail@gmail.com> Co-authored-by: Denis <denis@rdner.de> (cherry picked from commit bfde79f) Co-authored-by: Khushi Jain <khushi.jain@elastic.co>
- Loading branch information
1 parent
701c8f4
commit 0003eaf
Showing
8 changed files
with
629 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,135 @@ | ||
| // Licensed to Elasticsearch B.V. under one or more contributor | ||
| // license agreements. See the NOTICE file distributed with | ||
| // this work for additional information regarding copyright | ||
| // ownership. Elasticsearch B.V. licenses this file to you under | ||
| // the Apache License, Version 2.0 (the "License"); you may | ||
| // not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| package actions | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/elastic/beats/v7/libbeat/beat" | ||
| "github.com/elastic/beats/v7/libbeat/processors" | ||
| conf "github.com/elastic/elastic-agent-libs/config" | ||
| "github.com/elastic/elastic-agent-libs/mapstr" | ||
| ) | ||
|
|
||
| type alterFieldProcessor struct { | ||
| Fields []string | ||
| IgnoreMissing bool | ||
| FailOnError bool | ||
| AlterFullField bool | ||
|
|
||
| processorName string | ||
| alterFunc mapstr.AlterFunc | ||
| } | ||
|
|
||
| // NewAlterFieldProcessor is an umbrella method for processing events based on provided fields. Such as converting event keys to uppercase/lowercase | ||
| func NewAlterFieldProcessor(c *conf.C, processorName string, alterFunc mapstr.AlterFunc) (beat.Processor, error) { | ||
| config := struct { | ||
| Fields []string `config:"fields"` | ||
| IgnoreMissing bool `config:"ignore_missing"` | ||
| FailOnError bool `config:"fail_on_error"` | ||
| AlterFullField bool `config:"alter_full_field"` | ||
| }{ | ||
| IgnoreMissing: false, | ||
| FailOnError: true, | ||
| AlterFullField: true, | ||
| } | ||
|
|
||
| if err := c.Unpack(&config); err != nil { | ||
| return nil, fmt.Errorf("failed to unpack the %s fields configuration: %w", processorName, err) | ||
| } | ||
|
|
||
| // Skip mandatory fields | ||
| var configFields []string | ||
| var lowerField string | ||
| for _, readOnly := range processors.MandatoryExportedFields { | ||
| readOnly = strings.ToLower(readOnly) | ||
| for _, field := range config.Fields { | ||
| // Skip fields that match "readOnly" or start with "readOnly." | ||
| lowerField = strings.ToLower(field) | ||
| if strings.HasPrefix(lowerField, readOnly+".") || lowerField == readOnly { | ||
| continue | ||
| } | ||
| // Add fields that do not match "readOnly" criteria | ||
| configFields = append(configFields, field) | ||
| } | ||
| } | ||
| return &alterFieldProcessor{ | ||
| Fields: configFields, | ||
| IgnoreMissing: config.IgnoreMissing, | ||
| FailOnError: config.FailOnError, | ||
| processorName: processorName, | ||
| AlterFullField: config.AlterFullField, | ||
| alterFunc: alterFunc, | ||
| }, nil | ||
|
|
||
| } | ||
|
|
||
| func (a *alterFieldProcessor) String() string { | ||
| return fmt.Sprintf("%s fields=%+v", a.processorName, *a) | ||
| } | ||
|
|
||
| func (a *alterFieldProcessor) Run(event *beat.Event) (*beat.Event, error) { | ||
| var backup *beat.Event | ||
| if a.FailOnError { | ||
| backup = event.Clone() | ||
| } | ||
|
|
||
| for _, field := range a.Fields { | ||
| err := a.alter(event, field) | ||
| if err != nil { | ||
| if a.IgnoreMissing && errors.Is(err, mapstr.ErrKeyNotFound) { | ||
| continue | ||
| } | ||
| if a.FailOnError { | ||
| event = backup | ||
| _, _ = event.PutValue("error.message", err.Error()) | ||
| return event, err | ||
| } | ||
| } | ||
| } | ||
|
|
||
| return event, nil | ||
| } | ||
|
|
||
| func (a *alterFieldProcessor) alter(event *beat.Event, field string) error { | ||
|
|
||
| // modify all segments of the key | ||
| if a.AlterFullField { | ||
| err := event.Fields.AlterPath(field, mapstr.CaseInsensitiveMode, a.alterFunc) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } else { | ||
| // modify only the last segment | ||
| segmentCount := strings.Count(field, ".") | ||
| err := event.Fields.AlterPath(field, mapstr.CaseInsensitiveMode, func(key string) (string, error) { | ||
| if segmentCount > 0 { | ||
| segmentCount-- | ||
| return key, nil | ||
| } | ||
| return a.alterFunc(key) | ||
| }) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| [[lowercase]] | ||
| === Lowercase fields in events | ||
|
|
||
| ++++ | ||
| <titleabbrev>lowercase</titleabbrev> | ||
| ++++ | ||
|
|
||
| The `lowercase` processor specifies a list of fields that should be converted to lowercase. This transformation applies to keys that match the specified fields. Matching is performed case-insensitively. | ||
|
|
||
|
|
||
| ==== Examples: | ||
|
|
||
| 1. Default scenario | ||
|
|
||
| [source,yaml] | ||
| ---- | ||
| processors: | ||
| - rename: | ||
| fields: | ||
| - "ab.cd" | ||
| ignore_missing: false | ||
| fail_on_error: true | ||
| full_path: true | ||
| ---- | ||
| [source,json] | ||
| ---- | ||
| // Input | ||
| { | ||
| "AB": {"CD":"data"}, | ||
| "CD": {"ef":"data"} | ||
| } | ||
| // output | ||
| { | ||
| "ab": {"cd":"data"}, // `AB.CD` -> `ab.cd` | ||
| "CD": {"ef":"data"} | ||
| } | ||
| ---- | ||
|
|
||
| [start=2] | ||
| 2. When `full_path` is false | ||
|
|
||
| [source,yaml] | ||
| ---- | ||
| processors: | ||
| - rename: | ||
| fields: | ||
| - "ab.cd" | ||
| ignore_missing: false | ||
| fail_on_error: true | ||
| alter_full_field: false | ||
| ---- | ||
|
|
||
| [source,json] | ||
| ---- | ||
| // Input | ||
| { | ||
| "AB": {"CD":"data"}, | ||
| "CD": {"ef":"data"} | ||
| } | ||
| // output | ||
| { | ||
| "AB": {"cd":"data"}, // `AB.CD` -> `AB.cd` (only `cd` is lowercased) | ||
| "CD": {"ef":"data"} | ||
| } | ||
| ---- | ||
|
|
||
| [start=2] | ||
| 2. In case of non unique path to the key | ||
|
|
||
| [source,yaml] | ||
| ---- | ||
| processors: | ||
| - rename: | ||
| fields: | ||
| - "ab" | ||
| ignore_missing: false | ||
| fail_on_error: true | ||
| alter_full_field: true | ||
| ---- | ||
|
|
||
| [source,json] | ||
| ---- | ||
| // Input | ||
| { | ||
| "ab": "first", | ||
| "aB": "second" | ||
| } | ||
| // Output | ||
| { | ||
| "ab": "first", | ||
| "aB": "second", | ||
| "err": "... Error: key collision" | ||
| } | ||
| ---- | ||
|
|
||
| ==== Configuration: | ||
|
|
||
| The `lowercase` processor has the following configuration settings: | ||
|
|
||
| `fields`:: The field names to lowercase. The match is case-insensitive, e.g. `a.b.c.d` would match `A.b.C.d` or `A.B.C.D`. | ||
| `ignore_missing`:: (Optional) Indicates whether to ignore events that lack the source field. | ||
| The default is `false`, which will fail processing of an event if a field is missing. | ||
| `fail_on_error`:: (Optional) If set to `true` and an error occurs, the changes are reverted and the original event is returned. | ||
| If set to `false`, processing continues if an error occurs. Default is `true`. | ||
| `alter_full_field`:: (Optional) If set to `true`, the entire key path is lowercased. If set to `false` only the final part of the key path is lowercased. Default is true | ||
|
|
||
|
|
||
|
|
||
| See <<conditions>> for a list of supported conditions. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| // Licensed to Elasticsearch B.V. under one or more contributor | ||
| // license agreements. See the NOTICE file distributed with | ||
| // this work for additional information regarding copyright | ||
| // ownership. Elasticsearch B.V. licenses this file to you under | ||
| // the Apache License, Version 2.0 (the "License"); you may | ||
| // not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, | ||
| // software distributed under the License is distributed on an | ||
| // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| // KIND, either express or implied. See the License for the | ||
| // specific language governing permissions and limitations | ||
| // under the License. | ||
|
|
||
| package actions | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
| "github.com/elastic/beats/v7/libbeat/beat" | ||
| "github.com/elastic/beats/v7/libbeat/processors" | ||
| "github.com/elastic/beats/v7/libbeat/processors/checks" | ||
| conf "github.com/elastic/elastic-agent-libs/config" | ||
| ) | ||
|
|
||
| func init() { | ||
| processors.RegisterPlugin( | ||
| "lowercase", | ||
| checks.ConfigChecked( | ||
| NewLowerCaseProcessor, | ||
| checks.RequireFields("fields"), | ||
| checks.AllowedFields("fields", "when", "ignore_missing", "fail_on_error", "alter_full_field"), | ||
| ), | ||
| ) | ||
| } | ||
|
|
||
| // NewLowerCaseProcessor converts event keys matching the provided fields to lowercase | ||
| func NewLowerCaseProcessor(c *conf.C) (beat.Processor, error) { | ||
| return NewAlterFieldProcessor(c, "lowercase", lowerCase) | ||
| } | ||
|
|
||
| func lowerCase(field string) (string, error) { | ||
| return strings.ToLower(field), nil | ||
| } |
Oops, something went wrong.