From 6cd8d99df7d57fb475a5afe2e7fecbbe7d61fc6e Mon Sep 17 00:00:00 2001 From: Peter Fokkinga Date: Thu, 23 Aug 2018 16:20:17 +0200 Subject: [PATCH 1/2] setting to exclude MDC keys from being sent to Elastic useful when a particular key (or keys) may contain a lot of text that is of no particular use in your Elastic index, or when you want to explicitly add the key as an object property keys are matched case sensitive, without wildcard support --- README.md | 2 + .../AbstractElasticsearchAppender.java | 15 +++---- .../ClassicElasticsearchPublisher.java | 39 +++++++++++++------ .../elasticsearch/config/Settings.java | 9 +++++ .../ElasticsearchAppenderTest.java | 3 ++ 5 files changed, 49 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 9cf953e..51b9ddd 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ In your `logback.xml`: 250 false false + stacktrace 100 @@ -108,6 +109,7 @@ Configuration Reference * `errorLoggerName` (optional): If set, any internal errors or problems will be logged to this logger * `rawJsonMessage` (optional, default false): If set to `true`, the log message is interpreted as pre-formatted raw JSON message. * `includeMdc` (optional, default false): If set to `true`, then all [MDC](http://www.slf4j.org/api/org/slf4j/MDC.html) values will be mapped to properties on the JSON payload. + * `excludedMdcKeys` (optional, default empty): comma separated (extra whitespace is fine) list of case sensitive MDC keys that should not be mapped automatically to properties; only useful when includeMdc is set to `true` * `maxMessageSize` (optional, default -1): If set to a number greater than 0, truncate messages larger than this length, then append "`..`" to denote that the message was truncated * `authentication` (optional): Add the ability to send authentication headers (see below) diff --git a/src/main/java/com/internetitem/logback/elasticsearch/AbstractElasticsearchAppender.java b/src/main/java/com/internetitem/logback/elasticsearch/AbstractElasticsearchAppender.java index b0e7cb1..49a736e 100644 --- a/src/main/java/com/internetitem/logback/elasticsearch/AbstractElasticsearchAppender.java +++ b/src/main/java/com/internetitem/logback/elasticsearch/AbstractElasticsearchAppender.java @@ -1,16 +1,13 @@ package com.internetitem.logback.elasticsearch; +import ch.qos.logback.core.UnsynchronizedAppenderBase; +import com.internetitem.logback.elasticsearch.config.*; +import com.internetitem.logback.elasticsearch.util.ErrorReporter; + import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; -import ch.qos.logback.core.UnsynchronizedAppenderBase; -import com.internetitem.logback.elasticsearch.config.Authentication; -import com.internetitem.logback.elasticsearch.config.ElasticsearchProperties; -import com.internetitem.logback.elasticsearch.config.HttpRequestHeaders; -import com.internetitem.logback.elasticsearch.config.Settings; -import com.internetitem.logback.elasticsearch.util.ErrorReporter; - public abstract class AbstractElasticsearchAppender extends UnsynchronizedAppenderBase { protected Settings settings; @@ -131,6 +128,10 @@ public void setIncludeMdc(boolean includeMdc) { settings.setIncludeMdc(includeMdc); } + public void setExcludedMdcKeys(String setExcludedMdcKeys) { + settings.setExcludedMdcKeys(setExcludedMdcKeys); + } + public void setAuthentication(Authentication auth) { settings.setAuthentication(auth); } diff --git a/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java b/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java index 8bde9c2..5f249bc 100644 --- a/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java +++ b/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java @@ -1,18 +1,14 @@ package com.internetitem.logback.elasticsearch; -import java.io.IOException; -import java.util.Map; - import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.Context; import com.fasterxml.jackson.core.JsonGenerator; -import com.internetitem.logback.elasticsearch.config.ElasticsearchProperties; -import com.internetitem.logback.elasticsearch.config.HttpRequestHeaders; -import com.internetitem.logback.elasticsearch.config.Property; -import com.internetitem.logback.elasticsearch.config.Settings; -import com.internetitem.logback.elasticsearch.util.AbstractPropertyAndEncoder; -import com.internetitem.logback.elasticsearch.util.ClassicPropertyAndEncoder; -import com.internetitem.logback.elasticsearch.util.ErrorReporter; +import com.internetitem.logback.elasticsearch.config.*; +import com.internetitem.logback.elasticsearch.util.*; + +import java.io.IOException; +import java.util.*; + public class ClassicElasticsearchPublisher extends AbstractElasticsearchPublisher { @@ -40,10 +36,29 @@ protected void serializeCommonFields(JsonGenerator gen, ILoggingEvent event) thr gen.writeObjectField("message", formattedMessage); } - if(settings.isIncludeMdc()) { + if (settings.isIncludeMdc()) { + List excludedKeys = getExcludedMdcKeys(); for (Map.Entry entry : event.getMDCPropertyMap().entrySet()) { - gen.writeObjectField(entry.getKey(), entry.getValue()); + if (!excludedKeys.contains(entry.getKey())) { + gen.writeObjectField(entry.getKey(), entry.getValue()); + } + } + } + } + + private List getExcludedMdcKeys() { + /* + * using a List instead of a Map because the assumption is that + * the number of excluded keys will be very small and not cause + * a performance issue + */ + List result = new ArrayList<>(); + if (settings.getExcludedMdcKeys() != null) { + String[] parts = settings.getExcludedMdcKeys().split(","); + for (String part : parts) { + result.add(part.trim()); } } + return result; } } diff --git a/src/main/java/com/internetitem/logback/elasticsearch/config/Settings.java b/src/main/java/com/internetitem/logback/elasticsearch/config/Settings.java index e349b47..dc12abf 100644 --- a/src/main/java/com/internetitem/logback/elasticsearch/config/Settings.java +++ b/src/main/java/com/internetitem/logback/elasticsearch/config/Settings.java @@ -19,6 +19,7 @@ public class Settings { private boolean errorsToStderr; private boolean includeCallerData; private boolean includeMdc; + private String excludedMdcKeys; private boolean rawJsonMessage; private int maxQueueSize = 100 * 1024 * 1024; private Authentication authentication; @@ -155,6 +156,14 @@ public void setIncludeMdc(boolean includeMdc) { this.includeMdc = includeMdc; } + public String getExcludedMdcKeys() { + return excludedMdcKeys; + } + + public void setExcludedMdcKeys(String excludedMdcKeys) { + this.excludedMdcKeys = excludedMdcKeys; + } + public int getMaxMessageSize() { return maxMessageSize; } diff --git a/src/test/java/com/internetitem/logback/elasticsearch/ElasticsearchAppenderTest.java b/src/test/java/com/internetitem/logback/elasticsearch/ElasticsearchAppenderTest.java index 86acab3..77a3ebe 100644 --- a/src/test/java/com/internetitem/logback/elasticsearch/ElasticsearchAppenderTest.java +++ b/src/test/java/com/internetitem/logback/elasticsearch/ElasticsearchAppenderTest.java @@ -176,6 +176,7 @@ public void should_delegate_setters_to_settings() throws MalformedURLException { boolean errorsToStderr = false; boolean rawJsonMessage = false; boolean includeMdc = true; + String excludedMdcKeys = "stacktrace,url"; String index = "app-logs"; String type = "appenderType"; int maxQueueSize = 10; @@ -202,6 +203,7 @@ public void should_delegate_setters_to_settings() throws MalformedURLException { appender.setConnectTimeout(connectTimeout); appender.setRawJsonMessage(rawJsonMessage); appender.setIncludeMdc(includeMdc); + appender.setExcludedMdcKeys(excludedMdcKeys); verify(settings, times(1)).setReadTimeout(readTimeout); verify(settings, times(1)).setSleepTime(aSleepTime); @@ -218,6 +220,7 @@ public void should_delegate_setters_to_settings() throws MalformedURLException { verify(settings, times(1)).setConnectTimeout(connectTimeout); verify(settings, times(1)).setRawJsonMessage(rawJsonMessage); verify(settings, times(1)).setIncludeMdc(includeMdc); + verify(settings, times(1)).setExcludedMdcKeys(excludedMdcKeys); } From 105d6e7846312814bd72baad555bcf4a05c8b4d3 Mon Sep 17 00:00:00 2001 From: Peter Fokkinga Date: Thu, 2 Jan 2020 10:59:06 +0100 Subject: [PATCH 2/2] performance improvement in ClassicElasticsearchPublisher as suggested by Knittl --- .../ClassicElasticsearchPublisher.java | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java b/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java index 5f249bc..9569bd8 100644 --- a/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java +++ b/src/main/java/com/internetitem/logback/elasticsearch/ClassicElasticsearchPublisher.java @@ -3,17 +3,43 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.core.Context; import com.fasterxml.jackson.core.JsonGenerator; -import com.internetitem.logback.elasticsearch.config.*; -import com.internetitem.logback.elasticsearch.util.*; +import com.internetitem.logback.elasticsearch.config.ElasticsearchProperties; +import com.internetitem.logback.elasticsearch.config.HttpRequestHeaders; +import com.internetitem.logback.elasticsearch.config.Property; +import com.internetitem.logback.elasticsearch.config.Settings; +import com.internetitem.logback.elasticsearch.util.AbstractPropertyAndEncoder; +import com.internetitem.logback.elasticsearch.util.ClassicPropertyAndEncoder; +import com.internetitem.logback.elasticsearch.util.ErrorReporter; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; public class ClassicElasticsearchPublisher extends AbstractElasticsearchPublisher { + private final List excludedMdcKeys; public ClassicElasticsearchPublisher(Context context, ErrorReporter errorReporter, Settings settings, ElasticsearchProperties properties, HttpRequestHeaders headers) throws IOException { super(context, errorReporter, settings, properties, headers); + + excludedMdcKeys = getExcludedMdcKeys(); + } + + private List getExcludedMdcKeys() { + /* + * using a List instead of a Map because the assumption is that + * the number of excluded keys will be very small and not cause + * a performance issue + */ + List result = new ArrayList<>(); + if (settings.getExcludedMdcKeys() != null) { + String[] parts = settings.getExcludedMdcKeys().split(","); + for (String part : parts) { + result.add(part.trim()); + } + } + return result; } @Override @@ -37,28 +63,11 @@ protected void serializeCommonFields(JsonGenerator gen, ILoggingEvent event) thr } if (settings.isIncludeMdc()) { - List excludedKeys = getExcludedMdcKeys(); for (Map.Entry entry : event.getMDCPropertyMap().entrySet()) { - if (!excludedKeys.contains(entry.getKey())) { + if (!excludedMdcKeys.contains(entry.getKey())) { gen.writeObjectField(entry.getKey(), entry.getValue()); } } } } - - private List getExcludedMdcKeys() { - /* - * using a List instead of a Map because the assumption is that - * the number of excluded keys will be very small and not cause - * a performance issue - */ - List result = new ArrayList<>(); - if (settings.getExcludedMdcKeys() != null) { - String[] parts = settings.getExcludedMdcKeys().split(","); - for (String part : parts) { - result.add(part.trim()); - } - } - return result; - } }