From 0acc013707af23985bd2e819aaff11b6f853b96a Mon Sep 17 00:00:00 2001 From: Natalia Angulo Date: Tue, 28 Apr 2020 09:03:06 +0200 Subject: [PATCH 1/4] Add prometheus metrics --- Dockerfile | 2 +- logdna/adapter/adapter.go | 416 +++++++++++++++++--------------- logdna/adapter/instrumenting.go | 184 ++++++++++++++ logdna/adapter/types.go | 68 +++--- logdna/logdna.go | 155 ++++++------ modules.go | 2 +- 6 files changed, 531 insertions(+), 296 deletions(-) create mode 100644 logdna/adapter/instrumenting.go diff --git a/Dockerfile b/Dockerfile index b2a3ac1..9acde5f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1,2 @@ FROM gliderlabs/logspout:latest -ENV BUILD_VERSION 1.2.0 \ No newline at end of file +ENV BUILD_VERSION 1.2.1 diff --git a/logdna/adapter/adapter.go b/logdna/adapter/adapter.go index 8298be5..477a2ce 100644 --- a/logdna/adapter/adapter.go +++ b/logdna/adapter/adapter.go @@ -2,223 +2,257 @@ package adapter import ( - "bytes" - "encoding/json" - "fmt" - "log" - "net/http" - "net/url" - "os" - "regexp" - "strings" - "text/template" - "time" - - "github.com/gliderlabs/logspout/router" - "github.com/gojektech/heimdall" - "github.com/gojektech/heimdall/httpclient" + "bytes" + "encoding/json" + "fmt" + "log" + "net/http" + "net/url" + "os" + "regexp" + "strings" + "text/template" + "time" + + "github.com/gliderlabs/logspout/router" + "github.com/gojektech/heimdall" + "github.com/gojektech/heimdall/httpclient" ) // New method of Adapter: func New(config Configuration) *Adapter { - backoff := heimdall.NewConstantBackoff(config.BackoffInterval, config.JitterInterval) - retrier := heimdall.NewRetrier(backoff) - httpClient := httpclient.NewClient( - httpclient.WithHTTPTimeout(config.HTTPTimeout), - httpclient.WithRetrier(retrier), - httpclient.WithRetryCount(int(config.RequestRetryCount)), - ) - - adapter := &Adapter{ - Config: config, - HTTPClient: httpClient, - Logger: log.New(os.Stdout, config.Hostname + " ", log.LstdFlags), - Queue: make(chan Line), - } - - go adapter.readQueue() - return adapter + backoff := heimdall.NewConstantBackoff(config.BackoffInterval, config.JitterInterval) + retrier := heimdall.NewRetrier(backoff) + httpClient := httpclient.NewClient( + httpclient.WithHTTPTimeout(config.HTTPTimeout), + httpclient.WithRetrier(retrier), + httpclient.WithRetryCount(int(config.RequestRetryCount)), + ) + adapterVersion := os.Getenv("BUILD_VERSION") + + adapter := &Adapter{ + Config: config, + HTTPClient: httpClient, + Logger: log.New(os.Stdout, config.Hostname+" ", log.LstdFlags), + Queue: make(chan Line), + instrumenting: newInstrumentingAdapter(), + version: adapterVersion, + } + + go adapter.instrumenting.fireBuildInfo(adapterVersion) + + go adapter.readQueue() + return adapter } // getLevel method is for specifying the Level: func (adapter *Adapter) getLevel(source string) string { - switch source { - case "stdout": - return "INFO" - case "stderr": - return "ERROR" - } - return "" + switch source { + case "stdout": + return "INFO" + case "stderr": + return "ERROR" + } + return "" } // getHost method is for deciding what to choose as a hostname: func (adapter *Adapter) getHost(containerHostname string) string { - host := containerHostname - if (adapter.Config.Hostname != "") { - host = adapter.Config.Hostname - } - return host + host := containerHostname + if adapter.Config.Hostname != "" { + host = adapter.Config.Hostname + } + return host } // getTags method is for extracting the tags from templates: func (adapter *Adapter) getTags(m *router.Message) string { - - if adapter.Config.Tags == "" { - return "" - } - - splitTags := strings.Split(adapter.Config.Tags, ",") - var listTags []string - existenceMap := map[string]bool{} - for _, t := range splitTags { - parsed := false - if matched, error := regexp.Match(`{{.+}}`, []byte(t)); matched && error == nil { - var parsedTagBytes bytes.Buffer - tmp, e := template.New("parsedTag").Parse(t) - if e == nil { - err := tmp.ExecuteTemplate(&parsedTagBytes, "parsedTag", m) - if err == nil { - parsedTag := parsedTagBytes.String() - for _, p := range strings.Split(parsedTag, ":") { - if !existenceMap[p] { - listTags = append(listTags, p) - existenceMap[p] = true - } - } - parsed = true - } - } - } - - if !parsed && !existenceMap[t] { - listTags = append(listTags, t) - existenceMap[t] = true - } - } - - return strings.Join(listTags, ",") + + if adapter.Config.Tags == "" { + return "" + } + + splitTags := strings.Split(adapter.Config.Tags, ",") + var listTags []string + existenceMap := map[string]bool{} + for _, t := range splitTags { + parsed := false + if matched, error := regexp.Match(`{{.+}}`, []byte(t)); matched && error == nil { + var parsedTagBytes bytes.Buffer + tmp, e := template.New("parsedTag").Parse(t) + if e == nil { + err := tmp.ExecuteTemplate(&parsedTagBytes, "parsedTag", m) + if err == nil { + parsedTag := parsedTagBytes.String() + for _, p := range strings.Split(parsedTag, ":") { + if !existenceMap[p] { + listTags = append(listTags, p) + existenceMap[p] = true + } + } + parsed = true + } + } + } + + if !parsed && !existenceMap[t] { + listTags = append(listTags, t) + existenceMap[t] = true + } + } + + return strings.Join(listTags, ",") } // Stream method is for streaming the messages: func (adapter *Adapter) Stream(logstream chan *router.Message) { - for m := range logstream { - if m.Data == "" { - continue - } - - messageStr, err := json.Marshal(Message{ - Message: m.Data, - Container: ContainerInfo{ - Name: strings.Trim(m.Container.Name, "/"), - ID: m.Container.ID, - Config: ContainerConfig{ - Image: m.Container.Config.Image, - Hostname: m.Container.Config.Hostname, - Labels: m.Container.Config.Labels, - }, - }, - Level: adapter.getLevel(m.Source), - Hostname: adapter.getHost(m.Container.Config.Hostname), - Tags: adapter.getTags(m), - }) - - if err != nil { - adapter.Logger.Println( - fmt.Errorf( - "JSON Marshalling Error: %s", - err.Error(), - ), - ) - } else { - adapter.Queue <- Line{ - Line: string(messageStr), - File: m.Container.Name, - Timestamp: time.Now().Unix(), - } - } - } + defer func(begin time.Time) { + adapter.instrumenting.fireAddRequestCount(streamActionLabel) + adapter.instrumenting.fireAddRequestLatency(begin, streamActionLabel) + }(time.Now()) + + for m := range logstream { + if m.Data == "" { + continue + } + + messageStr, err := json.Marshal(Message{ + Message: m.Data, + Container: ContainerInfo{ + Name: strings.Trim(m.Container.Name, "/"), + ID: m.Container.ID, + Config: ContainerConfig{ + Image: m.Container.Config.Image, + Hostname: m.Container.Config.Hostname, + Labels: m.Container.Config.Labels, + }, + }, + Level: adapter.getLevel(m.Source), + Hostname: adapter.getHost(m.Container.Config.Hostname), + Tags: adapter.getTags(m), + }) + + if err != nil { + go adapter.instrumenting.fireAddStreamMarshalDataError() + adapter.Logger.Println( + fmt.Errorf( + "JSON Marshalling Error: %s. Missing message data: %s", + err.Error(), + m.Data, + ), + ) + } else { + adapter.Queue <- Line{ + Line: string(messageStr), + File: m.Container.Name, + Timestamp: time.Now().Unix(), + } + } + } } // readQueue is a method for reading from queue: func (adapter *Adapter) readQueue() { - buffer := make([]Line, 0) - bufferSize := 0 - - timeout := time.NewTimer(adapter.Config.FlushInterval) - - for { - select { - case msg := <-adapter.Queue: - if bufferSize >= int(adapter.Config.MaxBufferSize) { - timeout.Stop() - adapter.flushBuffer(buffer) - buffer = make([]Line, 0) - bufferSize = 0 - } - - buffer = append(buffer, msg) - bufferSize += len(msg.Line) - - case <-timeout.C: - if bufferSize > 0 { - adapter.flushBuffer(buffer) - buffer = make([]Line, 0) - bufferSize = 0 - } - } - - timeout.Reset(adapter.Config.FlushInterval) - } + defer func(begin time.Time) { + adapter.instrumenting.fireAddRequestCount(readQueueActionLabel) + adapter.instrumenting.fireAddRequestLatency(begin, readQueueActionLabel) + }(time.Now()) + + buffer := make([]Line, 0) + bufferSize := 0 + + timeout := time.NewTimer(adapter.Config.FlushInterval) + + for { + select { + case msg := <-adapter.Queue: + if bufferSize >= int(adapter.Config.MaxBufferSize) { + timeout.Stop() + adapter.flushBuffer(buffer) + buffer = make([]Line, 0) + bufferSize = 0 + } + + buffer = append(buffer, msg) + bufferSize += len(msg.Line) + + case <-timeout.C: + if bufferSize > 0 { + adapter.flushBuffer(buffer) + buffer = make([]Line, 0) + bufferSize = 0 + } + } + + timeout.Reset(adapter.Config.FlushInterval) + } } // flushBuffer is a method for flushing the lines: func (adapter *Adapter) flushBuffer(buffer []Line) { - var data bytes.Buffer - - body := struct { - Lines []Line `json:"lines"` - }{ - Lines: buffer, - } - - if error := json.NewEncoder(&data).Encode(body); error != nil { - adapter.Logger.Println( - fmt.Errorf( - "JSON Encoding Error: %s", - error.Error(), - ), - ) - return - } - - urlValues := url.Values{} - urlValues.Add("hostname", "logdna_logspout") - url := "https://" + adapter.Config.LogDNAURL + "?" + urlValues.Encode() - req, _ := http.NewRequest(http.MethodPost, url, &data) - req.Header.Set("user-agent", "logspout/" + os.Getenv("BUILD_VERSION")) - req.Header.Set("Content-Type", "application/json; charset=UTF-8") - req.SetBasicAuth(adapter.Config.LogDNAKey, "") - resp, err := adapter.HTTPClient.Do(req) - - if err != nil { - adapter.Logger.Println( - fmt.Errorf( - "HTTP Client Post Request Error: %s", - err.Error(), - ), - ) - return - } - - if resp != nil { - if resp.StatusCode != http.StatusOK { - adapter.Logger.Println( - fmt.Errorf( - "Received Status Code: %s While Sending Message", - resp.StatusCode, - ), - ) - } - defer resp.Body.Close() - } + defer func(begin time.Time) { + adapter.instrumenting.fireAddRequestCount(flushBufferActionLabel) + adapter.instrumenting.fireAddRequestLatency(begin, flushBufferActionLabel) + }(time.Now()) + + var data bytes.Buffer + + body := struct { + Lines []Line `json:"lines"` + }{ + Lines: buffer, + } + + if error := json.NewEncoder(&data).Encode(body); error != nil { + go adapter.instrumenting.fireAddFlushBufferEncodeDataError() + adapter.Logger.Println( + fmt.Errorf( + "JSON Encoding Error: %s", + error.Error(), + ), + ) + return + } + + urlValues := url.Values{} + urlValues.Add("hostname", "logdna_logspout") + url := "https://" + adapter.Config.LogDNAURL + "?" + urlValues.Encode() + req, _ := http.NewRequest(http.MethodPost, url, &data) + req.Header.Set("user-agent", "logspout/"+adapter.version) + req.Header.Set("Content-Type", "application/json; charset=UTF-8") + req.SetBasicAuth(adapter.Config.LogDNAKey, "") + resp, err := adapter.HTTPClient.Do(req) + + if err != nil { + go adapter.instrumenting.fireAddFlushBufferLogDNACodeError() + adapter.Logger.Println( + fmt.Errorf( + "HTTP Client Post Request Error: %s", + err.Error(), + ), + ) + return + } + + if resp != nil { + go adapter.instrumenting.fireAddLogDNAClientRequest(flushBufferActionLabel, fmt.Sprint(resp.StatusCode)) + + if resp.StatusCode != http.StatusOK { + adapter.Logger.Println( + fmt.Errorf( + "Received Status Code: %d While Sending Message", + resp.StatusCode, + ), + ) + } + defer resp.Body.Close() + } else { + go adapter.instrumenting.fireAddFlushBufferLogDNACodeError() + adapter.Logger.Println( + fmt.Errorf( + "HTTP Client Post Request Error: %s", + "request response is nil", + ), + ) + } } diff --git a/logdna/adapter/instrumenting.go b/logdna/adapter/instrumenting.go new file mode 100644 index 0000000..483bb05 --- /dev/null +++ b/logdna/adapter/instrumenting.go @@ -0,0 +1,184 @@ +package adapter + +import ( + "fmt" + "log" + "os" + "runtime" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + statusLabel = "status" + actionLabel = "action" + + // actions + streamActionLabel = "stream" + readQueueActionLabel = "read_queue" + flushBufferActionLabel = "flush_buffer" +) + +var ( + emptyPrometheusLabels = prometheus.Labels{} + + actionLabelsValues = []string{ + streamActionLabel, + readQueueActionLabel, + flushBufferActionLabel, + } + // logDNAStatusCodes are the possible response codes from logDNA server when shipping logs + logDNAStatusCodes = []string{"200", "400", "403", "500", "501", "502", "503"} +) + +type instrumentingAdapter struct { + // logdna adapter metrics + // general metrics + buildInfo *prometheus.GaugeVec + // performance metrics + requestCount *prometheus.CounterVec + requestLatency *prometheus.HistogramVec + logDNARequestCount *prometheus.CounterVec + + // error metrics + streamMarshalDataErrorCount *prometheus.CounterVec + flushBufferEncodeDataErrorCount *prometheus.CounterVec + flushBufferLogDNACodeErrorCount *prometheus.CounterVec + + logger *log.Logger +} + +// newInstrumentingAdapter returns an instance of an instrumenting Adapter. +func newInstrumentingAdapter() *instrumentingAdapter { + logger := log.New(os.Stdout, "Instrumenting ", log.LstdFlags) + + // ## Define metrics + // general metrics + buildInfo := promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "logspout_logdna", + Subsystem: "adapter", + Name: "build_info", + Help: "Number of logspout logdna adapter builds.", + }, + []string{"go_version", "adapter_version"}, + ) + + // performance metrics + requestCount := promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "logspout_logdna", + Subsystem: "adapter", + Name: "request_total", + Help: "Number of stream requests received.", + }, []string{actionLabel}) + requestLatency := promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "logspout_logdna", + Subsystem: "adapter", + Name: "request_latency_seconds", + Help: "Total duration of stream requests in seconds.", + }, []string{actionLabel}) + logDNARequestCount := promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "logspout_logdna", + Subsystem: "adapter", + Name: "logdna_request_total", + Help: "Number of requests to LogDNA service while flushing message.", + }, []string{actionLabel, statusLabel}) + + // error metrics + streamMarshalDataErrorCount := promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "logspout_logdna", + Subsystem: "adapter", + Name: "stream_marshal_data_error_total", + Help: "Number of errors when marshalling the received stream data.", + }, []string{}) + flushBufferEncodeDataErrorCount := promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "logspout_logdna", + Subsystem: "adapter", + Name: "flush_buffer_encode_data_error_total", + Help: "Number of errors when encoding the data while flushing message.", + }, []string{}) + flushBufferLogDNACodeErrorCount := promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "logspout_logdna", + Subsystem: "adapter", + Name: "flush_buffer_logdna_code_error_total", + Help: "Number of errors when posting the message to LogDNA because of a code error when performing the request while flushing message.", + }, []string{}) + + // ## Initialize metrics: why? https://www.robustperception.io/existential-issues-with-metrics + // performance metrics + for _, action := range actionLabelsValues { + if _, err := requestCount.GetMetricWith(prometheus.Labels{ + actionLabel: action, + }); err != nil { + logger.Println(fmt.Errorf("cannot initialize request counter metric: %s", err)) + } + } + for _, action := range actionLabelsValues { + if _, err := requestLatency.GetMetricWith(prometheus.Labels{ + actionLabel: action, + }); err != nil { + logger.Println(fmt.Errorf("cannot initialize request latency metric: %s", err)) + } + } + + for _, status := range logDNAStatusCodes { + for _, action := range actionLabelsValues { + if _, err := logDNARequestCount.GetMetricWith(prometheus.Labels{ + actionLabel: action, + statusLabel: status, + }); err != nil { + logger.Println(fmt.Errorf("cannot initialize flush buffer logdna client error counter metric: %s", err)) + } + } + } + + return &instrumentingAdapter{ + buildInfo: buildInfo, + + requestCount: requestCount, + requestLatency: requestLatency, + logDNARequestCount: logDNARequestCount, + + streamMarshalDataErrorCount: streamMarshalDataErrorCount, + flushBufferEncodeDataErrorCount: flushBufferEncodeDataErrorCount, + flushBufferLogDNACodeErrorCount: flushBufferLogDNACodeErrorCount, + + logger: logger, + } +} + +func (inst *instrumentingAdapter) fireBuildInfo(adapterVersion string) { + inst.buildInfo.WithLabelValues(runtime.Version(), adapterVersion).Set(1) +} + +func (inst *instrumentingAdapter) fireAddRequestCount(action string) { + inst.requestCount.With(prometheus.Labels{ + actionLabel: action, + }).Add(1) +} + +func (inst *instrumentingAdapter) fireAddRequestLatency(begin time.Time, action string) { + inst.requestLatency.With(prometheus.Labels{ + actionLabel: action, + }).Observe(time.Since(begin).Seconds()) +} + +func (inst *instrumentingAdapter) fireAddStreamMarshalDataError() { + inst.streamMarshalDataErrorCount.With(emptyPrometheusLabels).Add(1) +} + +func (inst *instrumentingAdapter) fireAddFlushBufferEncodeDataError() { + inst.flushBufferEncodeDataErrorCount.With(emptyPrometheusLabels).Add(1) +} + +func (inst *instrumentingAdapter) fireAddFlushBufferLogDNACodeError() { + inst.flushBufferLogDNACodeErrorCount.With(emptyPrometheusLabels).Add(1) +} + +func (inst *instrumentingAdapter) fireAddLogDNAClientRequest(action string, statusCode string) { + inst.logDNARequestCount.With(prometheus.Labels{ + statusLabel: statusCode, + actionLabel: action, + }).Add(1) +} diff --git a/logdna/adapter/types.go b/logdna/adapter/types.go index d59fe7f..bba9ca7 100644 --- a/logdna/adapter/types.go +++ b/logdna/adapter/types.go @@ -2,64 +2,66 @@ package adapter import ( - "log" - "time" + "log" + "time" - "github.com/gojektech/heimdall" + "github.com/gojektech/heimdall" ) /* - Definitions of All Structs + Definitions of All Structs */ // Configuration is Configuration Struct for LogDNA Adapter: type Configuration struct { - BackoffInterval time.Duration - FlushInterval time.Duration - Hostname string - HTTPTimeout time.Duration - JitterInterval time.Duration - LogDNAKey string - LogDNAURL string - MaxBufferSize uint64 - RequestRetryCount uint64 - Tags string + BackoffInterval time.Duration + FlushInterval time.Duration + Hostname string + HTTPTimeout time.Duration + JitterInterval time.Duration + LogDNAKey string + LogDNAURL string + MaxBufferSize uint64 + RequestRetryCount uint64 + Tags string } // Adapter structure: type Adapter struct { - Config Configuration - HTTPClient heimdall.Client - Logger *log.Logger - Queue chan Line + Config Configuration + HTTPClient heimdall.Client + Logger *log.Logger + Queue chan Line + instrumenting *instrumentingAdapter + version string } // Line structure for the queue of Adapter: type Line struct { - Timestamp int64 `json:"timestamp"` - Line string `json:"line"` - File string `json:"file"` + Timestamp int64 `json:"timestamp"` + Line string `json:"line"` + File string `json:"file"` } // Message structure: type Message struct { - Message string `json:"message"` - Container ContainerInfo `json:"container"` - Level string `json:"level"` - Hostname string `json:"hostname"` - Tags string `json:"tags"` + Message string `json:"message"` + Container ContainerInfo `json:"container"` + Level string `json:"level"` + Hostname string `json:"hostname"` + Tags string `json:"tags"` } // ContainerInfo structure for the Container of Message: type ContainerInfo struct { - Name string `json:"name"` - ID string `json:"id"` - Config ContainerConfig `json:"config"` + Name string `json:"name"` + ID string `json:"id"` + Config ContainerConfig `json:"config"` } // ContainerConfig structure for the Config of ContainerInfo: type ContainerConfig struct { - Image string `json:"image"` - Hostname string `json:"hostname"` - Labels map[string]string `json:"labels"` -} \ No newline at end of file + Image string `json:"image"` + Hostname string `json:"hostname"` + Labels map[string]string `json:"labels"` +} diff --git a/logdna/logdna.go b/logdna/logdna.go index 7fe3196..b120b43 100644 --- a/logdna/logdna.go +++ b/logdna/logdna.go @@ -1,4 +1,4 @@ -// Package logdna is LogSpout Adapter to Forward Logs to LogDNA +// Package logdna is LogSpout Adapter to Forward Logs to LogDNA package logdna // The MIT License (MIT) @@ -25,94 +25,109 @@ package logdna // SOFTWARE. import ( - "errors" - "log" - "os" - "strconv" - "strings" - "time" - - "github.com/gliderlabs/logspout/router" - "github.com/logdna/logspout/logdna/adapter" + "errors" + "fmt" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/gliderlabs/logspout/adapters/logdna/adapter" + "github.com/gliderlabs/logspout/router" + "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus/promhttp" ) /* - Common Functions + Common Functions */ // Getting Uint Variable from Environment: func getUintOpt(name string, dfault uint64) uint64 { - if result, err := strconv.ParseUint(os.Getenv(name), 10, 64); err == nil { - return result - } - return dfault + if result, err := strconv.ParseUint(os.Getenv(name), 10, 64); err == nil { + return result + } + return dfault } // Getting Duration Variable from Environment: func getDurationOpt(name string, dfault time.Duration) time.Duration { - if result, err := strconv.ParseInt(os.Getenv(name), 10, 64); err == nil { - return time.Duration(result) - } - return dfault + if result, err := strconv.ParseInt(os.Getenv(name), 10, 64); err == nil { + return time.Duration(result) + } + return dfault } // Getting String Variable from Environment: func getStringOpt(name, dfault string) string { - if value := os.Getenv(name); value != "" { - return value - } - return dfault + if value := os.Getenv(name); value != "" { + return value + } + return dfault +} + +// metrics returns a http.Handler for the prometheus metrics +func metrics() http.Handler { + r := mux.NewRouter() + r.Handle("/metrics", promhttp.Handler()) + + return r } func init() { - router.AdapterFactories.Register(NewLogDNAAdapter, "logdna") - - filterLabels := make([]string, 0) - if filterLabelsValue := os.Getenv("FILTER_LABELS"); filterLabelsValue != "" { - filterLabels = strings.Split(filterLabelsValue, ",") - } - - filterSources := make([]string, 0) - if filterSourcesValue := os.Getenv("FILTER_SOURCES"); filterSourcesValue != "" { - filterSources = strings.Split(filterSourcesValue, ",") - } - - r := &router.Route{ - Adapter: "logdna", - FilterName: getStringOpt("FILTER_NAME", ""), - FilterID: getStringOpt("FILTER_ID", ""), - FilterLabels: filterLabels, - FilterSources: filterSources, - } - - if err := router.Routes.Add(r); err != nil { - log.Fatal("Cannot Add New Route: ", err.Error()) - } + router.HTTPHandlers.Register(metrics, "metrics") + router.AdapterFactories.Register(NewLogDNAAdapter, "logdna") + + fmt.Println(router.HTTPHandlers.All()) + + filterLabels := make([]string, 0) + if filterLabelsValue := os.Getenv("FILTER_LABELS"); filterLabelsValue != "" { + filterLabels = strings.Split(filterLabelsValue, ",") + } + + filterSources := make([]string, 0) + if filterSourcesValue := os.Getenv("FILTER_SOURCES"); filterSourcesValue != "" { + filterSources = strings.Split(filterSourcesValue, ",") + } + + r := &router.Route{ + Adapter: "logdna", + FilterName: getStringOpt("FILTER_NAME", ""), + FilterID: getStringOpt("FILTER_ID", ""), + FilterLabels: filterLabels, + FilterSources: filterSources, + } + + if err := router.Routes.Add(r); err != nil { + log.Fatal("Cannot Add New Route: ", err.Error()) + } } // NewLogDNAAdapter creates adapter: func NewLogDNAAdapter(route *router.Route) (router.LogAdapter, error) { - logdnaKey := os.Getenv("LOGDNA_KEY") - if logdnaKey == "" { - return nil, errors.New("Cannot Find Environment Variable \"LOGDNA_KEY\"") - } - - if os.Getenv("INACTIVITY_TIMEOUT") == "" { - os.Setenv("INACTIVITY_TIMEOUT", "1m") - } - - config := adapter.Configuration{ - BackoffInterval: getDurationOpt("HTTP_CLIENT_BACKOFF", 2) * time.Millisecond, - FlushInterval: getDurationOpt("FLUSH_INTERVAL", 250) * time.Millisecond, - Hostname: os.Getenv("HOSTNAME"), - HTTPTimeout: getDurationOpt("HTTP_CLIENT_TIMEOUT", 30) * time.Second, - JitterInterval: getDurationOpt("HTTP_CLIENT_JITTER", 5) * time.Millisecond, - LogDNAKey: logdnaKey, - LogDNAURL: getStringOpt("LOGDNA_URL", "logs.logdna.com/logs/ingest"), - MaxBufferSize: getUintOpt("MAX_BUFFER_SIZE", 2) * 1024 * 1024, - RequestRetryCount: getUintOpt("MAX_REQUEST_RETRY", 5), - Tags: os.Getenv("TAGS"), - } - - return adapter.New(config), nil -} \ No newline at end of file + logdnaKey := os.Getenv("LOGDNA_KEY") + if logdnaKey == "" { + return nil, errors.New("Cannot Find Environment Variable \"LOGDNA_KEY\"") + } + + if os.Getenv("INACTIVITY_TIMEOUT") == "" { + os.Setenv("INACTIVITY_TIMEOUT", "1m") + } + + config := adapter.Configuration{ + BackoffInterval: getDurationOpt("HTTP_CLIENT_BACKOFF", 2) * time.Millisecond, + FlushInterval: getDurationOpt("FLUSH_INTERVAL", 250) * time.Millisecond, + Hostname: os.Getenv("HOSTNAME"), + HTTPTimeout: getDurationOpt("HTTP_CLIENT_TIMEOUT", 30) * time.Second, + JitterInterval: getDurationOpt("HTTP_CLIENT_JITTER", 5) * time.Millisecond, + LogDNAKey: logdnaKey, + LogDNAURL: getStringOpt("LOGDNA_URL", "logs.logdna.com/logs/ingest"), + MaxBufferSize: getUintOpt("MAX_BUFFER_SIZE", 2) * 1024 * 1024, + RequestRetryCount: getUintOpt("MAX_REQUEST_RETRY", 5), + Tags: os.Getenv("TAGS"), + } + + return adapter.New(config), nil +} diff --git a/modules.go b/modules.go index acb6760..d9641b6 100644 --- a/modules.go +++ b/modules.go @@ -1,7 +1,7 @@ package main import ( + _ "github.com/angulito/logspout/logdna" _ "github.com/gliderlabs/logspout/httpstream" _ "github.com/gliderlabs/logspout/routesapi" - _ "github.com/logdna/logspout/logdna" ) From ca1c6668371783c8f325e47748a9ad6be9ac26fe Mon Sep 17 00:00:00 2001 From: Natalia Angulo Date: Tue, 28 Apr 2020 09:09:53 +0200 Subject: [PATCH 2/4] fix import --- logdna/logdna.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logdna/logdna.go b/logdna/logdna.go index b120b43..a574d5c 100644 --- a/logdna/logdna.go +++ b/logdna/logdna.go @@ -34,7 +34,7 @@ import ( "strings" "time" - "github.com/gliderlabs/logspout/adapters/logdna/adapter" + "github.com/angulito/logspout/logdna/adapter" "github.com/gliderlabs/logspout/router" "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" From 030dec86d5e36dc79d4d126b04773ddd17195397 Mon Sep 17 00:00:00 2001 From: Natalia Angulo Date: Tue, 28 Apr 2020 09:25:19 +0200 Subject: [PATCH 3/4] change importers in order to prepare PR to logdna --- CHANGELOG.md | 3 +++ logdna/logdna.go | 2 +- modules.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdcfebb..41dcdb8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ This file documents all notable changes in `LogDNA LogSpout Image`. The release numbering uses [semantic versioning](http://semver.org). +## v1.2.1 +* Add Prometheus metrics + ## v1.2.0 - Released on February 20, 2020 * Update License * Update Release Pipeline diff --git a/logdna/logdna.go b/logdna/logdna.go index a574d5c..a3c4587 100644 --- a/logdna/logdna.go +++ b/logdna/logdna.go @@ -34,9 +34,9 @@ import ( "strings" "time" - "github.com/angulito/logspout/logdna/adapter" "github.com/gliderlabs/logspout/router" "github.com/gorilla/mux" + "github.com/logdna/logspout/logdna/adapter" "github.com/prometheus/client_golang/prometheus/promhttp" ) diff --git a/modules.go b/modules.go index d9641b6..acb6760 100644 --- a/modules.go +++ b/modules.go @@ -1,7 +1,7 @@ package main import ( - _ "github.com/angulito/logspout/logdna" _ "github.com/gliderlabs/logspout/httpstream" _ "github.com/gliderlabs/logspout/routesapi" + _ "github.com/logdna/logspout/logdna" ) From 522988375e5eca7f08d1dce618574448c703586d Mon Sep 17 00:00:00 2001 From: Natalia Angulo Date: Tue, 28 Apr 2020 09:26:24 +0200 Subject: [PATCH 4/4] remove debug trace --- logdna/logdna.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/logdna/logdna.go b/logdna/logdna.go index a3c4587..a2d1739 100644 --- a/logdna/logdna.go +++ b/logdna/logdna.go @@ -26,7 +26,6 @@ package logdna import ( "errors" - "fmt" "log" "net/http" "os" @@ -80,8 +79,6 @@ func init() { router.HTTPHandlers.Register(metrics, "metrics") router.AdapterFactories.Register(NewLogDNAAdapter, "logdna") - fmt.Println(router.HTTPHandlers.All()) - filterLabels := make([]string, 0) if filterLabelsValue := os.Getenv("FILTER_LABELS"); filterLabelsValue != "" { filterLabels = strings.Split(filterLabelsValue, ",")