From e489739f15704c2d5ea2ecca2e47337347029224 Mon Sep 17 00:00:00 2001 From: Vishal Charpe Date: Mon, 16 Dec 2024 10:51:11 +0530 Subject: [PATCH] update-bearer: update-bearer-token-handling-impl --- README.md | 30 ++++++------ examples/fluent-bit-http.conf | 6 +-- examples/fluent-bit-multiple-config.conf | 6 +-- examples/fluent-bit-tcp.conf | 6 +-- examples/fluent-bit.conf | 6 +-- examples/fluent-bit_multiline.conf | 7 ++- output/client.go | 24 ++++++---- output/out_lm.go | 60 ++++++++++++++---------- output/out_lm_test.go | 2 +- 9 files changed, 80 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 4e1d700..6f1beea 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,10 @@ Create a custom `fluent-bit.conf` or edit the existing one to specify which logs Workers 1 accessKey accessID - bearerToken + bearerToken Bearer resourceMapping {"": ""} - include_metadata - lm_debug + includeMetadata + lmDebug ``` For more configuration examples, please refer to the examples folder, or see the [Fluentbit configuration documentation](https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/configuration-file) @@ -59,17 +59,17 @@ Produces this event: ## LogicMonitor properties -| Property | Description | -|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `Name` | Name of the input plugin. | -| `lmCompanyName` | LogicMonitor account name with domain. For example, test.logicmonitor.com . | -| `Match` | A pattern to match against the tags of incoming records. For example, * will match everything. | -| `Workers` | Number of workers to operate. | -| `accessID` | LM API Token access ID. | -| `accessKey` | LM API Token access key. | -| `bearerToken` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. | -| `resourceMapping` | The mapping that defines the source of the log event to the LM resource. In this case, the `` in the incoming event is mapped to the value of ``. | -| `include_metadata` | When `true`, appends additional metadata to the log. default `false`. | -| `lm_debug` | When `true`, logs more information to the fluent-bit console. | +| Property | Description | +|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `Name` | Name of the input plugin. | +| `lmCompanyName` | LogicMonitor account name with domain. For example, test.logicmonitor.com . | +| `Match` | A pattern to match against the tags of incoming records. For example, * will match everything. | +| `Workers` | Number of workers to operate. | +| `accessID` | LM API Token access ID. If not provided, omit setting the key entirely. | +| `accessKey` | LM API Token access key. If not provided, omit setting the key entirely. | +| `bearerToken` | LM API Bearer Token. Either specify `access_id` and `access_key` both or `bearer_token`. If all specified, LMv1 token(`access_id` and `access_key`) will be used for authentication with Logicmonitor. | +| `resourceMapping` | The mapping that defines the source of the log event to the LM resource. In this case, the `` in the incoming event is mapped to the value of ``. | +| `includeMetadata` | When `true`, appends additional metadata to the log. default `false`. | +| `lmDebug` | When `true`, logs more information to the fluent-bit console. | diff --git a/examples/fluent-bit-http.conf b/examples/fluent-bit-http.conf index e4dae6b..ed5b2db 100644 --- a/examples/fluent-bit-http.conf +++ b/examples/fluent-bit-http.conf @@ -10,8 +10,8 @@ Workers 1 accessKey accessID - bearerToken + bearerToken Bearer resourceMapping {"host":"system.ips"} - include_metadata + includeMetadata id - lm_debug \ No newline at end of file + lmDebug \ No newline at end of file diff --git a/examples/fluent-bit-multiple-config.conf b/examples/fluent-bit-multiple-config.conf index c2c054b..394298f 100644 --- a/examples/fluent-bit-multiple-config.conf +++ b/examples/fluent-bit-multiple-config.conf @@ -22,7 +22,7 @@ Workers 1 accessKey accessID - bearerToken - include_metadata + bearerToken Bearer + includeMetadata id - lm_debug + lmDebug diff --git a/examples/fluent-bit-tcp.conf b/examples/fluent-bit-tcp.conf index 26d01d1..cdd0f12 100644 --- a/examples/fluent-bit-tcp.conf +++ b/examples/fluent-bit-tcp.conf @@ -16,8 +16,8 @@ Workers 1 accessKey accessID - bearerToken + bearerToken Bearer resourceMapping {"host":"system.ips"} - include_metadata + includeMetadata id - lm_debug \ No newline at end of file + lmDebug \ No newline at end of file diff --git a/examples/fluent-bit.conf b/examples/fluent-bit.conf index 13b4f5c..bee05d9 100644 --- a/examples/fluent-bit.conf +++ b/examples/fluent-bit.conf @@ -12,7 +12,7 @@ Workers 1 accessKey accessID - bearerToken + bearerToken Bearer resourceMapping {"host":"system.ips"} - include_metadata - lm_debug \ No newline at end of file + includeMetadata + lmDebug \ No newline at end of file diff --git a/examples/fluent-bit_multiline.conf b/examples/fluent-bit_multiline.conf index 570e46f..7c9d087 100644 --- a/examples/fluent-bit_multiline.conf +++ b/examples/fluent-bit_multiline.conf @@ -12,9 +12,8 @@ Workers 1 accessKey accessID - bearerToken + bearerToken Bearer resourceMapping {"host":"system.ips"} - include_metadata true - include_metadata + includeMetadata id - lm_debug \ No newline at end of file + lmDebug \ No newline at end of file diff --git a/output/client.go b/output/client.go index bb52a73..198637f 100644 --- a/output/client.go +++ b/output/client.go @@ -30,6 +30,8 @@ type LogicmonitorClient struct { resourceMapping string bulk []model.LogInput includeMetadata bool + logSource string + versionId string logger *Logger sizeThresholdInBytes int } @@ -38,8 +40,8 @@ type LogicmonitorClient struct { type ClientOptionFunc func(*LogicmonitorClient) error // NewClient is a constructor for Logicmonitor http client -func NewClient( lmCompanyName string, accessID string, accessKey string, bearerToken string, useBearerTokenforAuth bool, resourceMapping string, includeMetadata bool, logger *Logger ) (*LogicmonitorClient) { - +func NewClient( lmCompanyName string, accessID string, accessKey string, bearerToken string, useBearerTokenforAuth bool, resourceMapping string, includeMetadata bool, logSource string, versionId string, logger *Logger ) (*LogicmonitorClient) { + logicmonitorClient := &LogicmonitorClient{ lmCompanyName: lmCompanyName, accessID: accessID, @@ -49,13 +51,15 @@ func NewClient( lmCompanyName string, accessID string, accessKey string, bearerT resourceMapping: resourceMapping, bulk: nil, includeMetadata: includeMetadata, + logSource: logSource, + versionId: versionId, logger: logger, sizeThresholdInBytes: maxRequestBodySizeInBytes, } - + return logicmonitorClient - - + + } func NewLogIngester(logicmonitorClient *LogicmonitorClient) (*logs.LMLogIngest){ @@ -64,11 +68,12 @@ func NewLogIngester(logicmonitorClient *LogicmonitorClient) (*logs.LMLogIngest){ BearerToken: logicmonitorClient.bearerToken} companyName := logicmonitorClient.lmCompanyName + userAgent := fmt.Sprintf("%s/%s", logicmonitorClient.logSource, logicmonitorClient.versionId) options := []logs.Option{ logs.WithLogBatchingDisabled(), logs.WithAuthentication(auth), logs.WithEndpoint("https://"+companyName+"/rest"), - logs.WithUserAgent("lm-logs-fluentbit"), + logs.WithUserAgent(userAgent), } lmLog, err := logs.NewLMLogIngest(context.Background(), options...) @@ -128,15 +133,14 @@ func (logicmonitorClient *LogicmonitorClient) Send(log []byte, logIngestor *logs for k, v := range resourceMapReceived { if(k != ""){ - resourceMap[v] = jsonMap[k] + resourceMap[k] = v } - } if (jsonMap["host"].(string) != "" && len(resourceMap)==0){ resourceMap["system.hostname"] = jsonMap["host"] } - - if(logicmonitorClient.includeMetadata){ + logger.Debug(fmt.Sprintf("include metadata: %s",logicmonitorClient.includeMetadata)) + if(logicmonitorClient.includeMetadata){ metadata = getMetadata(jsonMap) } diff --git a/output/out_lm.go b/output/out_lm.go index 9237edf..2510eb0 100644 --- a/output/out_lm.go +++ b/output/out_lm.go @@ -6,15 +6,15 @@ package main import ( "C" "fmt" - "github.com/fluent/fluent-bit-go/output" - jsoniter "github.com/json-iterator/go" - "github.com/logicmonitor/lm-data-sdk-go/api/logs" "os" "reflect" "strconv" "time" "unsafe" + "github.com/fluent/fluent-bit-go/output" + jsoniter "github.com/json-iterator/go" + "github.com/logicmonitor/lm-data-sdk-go/api/logs" ) const ( @@ -28,11 +28,11 @@ var ( ) type LogicmonitorOutput struct { - plugin Plugin - logger *Logger - client *LogicmonitorClient - logIngestor *logs.LMLogIngest - id string + plugin Plugin + logger *Logger + client *LogicmonitorClient + logIngestor *logs.LMLogIngest + id string } var ( @@ -145,7 +145,7 @@ func FLBPluginFlushCtx(ctx, data unsafe.Pointer, length C.int, tag *C.char) int break } - log, err := serializeRecord(ts, C.GoString(tag), record,id) + log, err := serializeRecord(ts, C.GoString(tag), record, id) if err != nil { continue } @@ -160,12 +160,12 @@ func FLBPluginFlushCtx(ctx, data unsafe.Pointer, length C.int, tag *C.char) int // //export FLBPluginExit func FLBPluginExit() int { - plugin.Flush(nil,nil) + plugin.Flush(nil, nil) return output.FLB_OK } func initConfigParams(ctx unsafe.Pointer) error { - debug, err := strconv.ParseBool(output.FLBPluginConfigKey(ctx, "lm_debug")) + debug, err := strconv.ParseBool(output.FLBPluginConfigKey(ctx, "lmDebug")) if err != nil { debug = false } @@ -186,38 +186,47 @@ func initConfigParams(ctx unsafe.Pointer) error { } logger = NewLogger(outputName+"_"+outputId, debug) - lmCompanyName := output.FLBPluginConfigKey(ctx, "lmCompanyName") if lmCompanyName == "" { - return fmt.Errorf("LM Company name is not specified. Please specify the company name in the configuration") + return fmt.Errorf("LM Company name is not specified. Please specify the company name in the configuration") } accessKey := output.FLBPluginConfigKey(ctx, "accessKey") accessID := output.FLBPluginConfigKey(ctx, "accessID") - useBearerTokenforAuth := true - if (accessID == "" || accessKey == "") { + useBearerTokenforAuth := false + if accessID == "" || accessKey == "" { logger.Log("accessID or accessKey is empty. Using bearer Token for authentication") - useBearerTokenforAuth = false + useBearerTokenforAuth = true } bearerToken := output.FLBPluginConfigKey(ctx, "bearerToken") - if(bearerToken == "" && useBearerTokenforAuth){ - return fmt.Errorf("Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.") + if accessID == "" || accessKey == "" { + if bearerToken == "" { + return fmt.Errorf("Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.") + } } resourceMapping := output.FLBPluginConfigKey(ctx, "resourceMapping") - includeMetadata, err := strconv.ParseBool(output.FLBPluginConfigKey(ctx, "include_metadata")) - - client := NewClient(lmCompanyName ,accessID , accessKey, bearerToken, useBearerTokenforAuth, resourceMapping, includeMetadata, logger) + includeMetadata, err := strconv.ParseBool(output.FLBPluginConfigKey(ctx, "includeMetadata")) + logSource := output.FLBPluginConfigKey(ctx, "logSource") + if logSource == "" { + logSource = "lm-logs-fluentbit" + } + versionId := output.FLBPluginConfigKey(ctx, "versionId") + if versionId == "" { + versionId = "1.0.0" + } + + client := NewClient(lmCompanyName, accessID, accessKey, bearerToken, useBearerTokenforAuth, resourceMapping, includeMetadata, logSource, versionId, logger) logIngestor := NewLogIngester(client) outputs[outputId] = LogicmonitorOutput{ - logger: logger, - client: client, - logIngestor: logIngestor, - id: outputId, + logger: logger, + client: client, + logIngestor: logIngestor, + id: outputId, } return nil @@ -238,6 +247,7 @@ func serializeRecord(ts interface{}, tag string, record map[interface{}]interfac body["timestamp"] = formatTimestamp(ts) body["fluentbit_tag"] = tag + body["_resource.type"] = "Fluentbit" serialized, err := jsoniter.Marshal(body) if err != nil { diff --git a/output/out_lm_test.go b/output/out_lm_test.go index ca12c8c..93ff4e6 100644 --- a/output/out_lm_test.go +++ b/output/out_lm_test.go @@ -56,7 +56,7 @@ func (p *TestPlugin) Environment(ctx unsafe.Pointer, key string) string { return p.token case "lm_url": return p.url - case "lm_debug": + case "lmDebug": return p.debug } return "not found"