summary: Include APM Labels in forwarded application logs.

notice: feat: New Relic APM language agents now allow you to opt-in to adding your custom tags (labels) to agent-forwarded logs. With custom tags on logs, platform engineers can easily filter, search, and correlate log data for faster and more efficient troubleshooting, improved performance, and optimized resource utilization. To learn more about this feature see the [documentation](https://docs.newrelic.com/docs/logs/logs-context/APM-logs-custom-tags). (#2831)

feat: New Relic APM language agents now allow you to opt-in to adding your custom tags (labels) to agent-forwarded logs. With custom tags on logs, platform engineers can easily filter, search, and correlate log data for faster and more efficient troubleshooting, improved performance, and optimized resource utilization. To learn more about this feature see the [documentation](https://docs.newrelic.com/docs/logs/logs-context/APM-logs-custom-tags). (#2831)

src/Agent/NewRelic/Agent/Core/AgentHealth/AgentHealthReporter.cs

@@ -662,6 +662,7 @@ public void ReportLogForwardingConfiguredValues()
             ReportSupportabilityCountMetric(MetricNames.GetSupportabilityLogMetricsConfiguredName(_configuration.LogMetricsCollectorEnabled));
             ReportSupportabilityCountMetric(MetricNames.GetSupportabilityLogForwardingConfiguredName(_configuration.LogEventCollectorEnabled));
             ReportSupportabilityCountMetric(MetricNames.GetSupportabilityLogDecoratingConfiguredName(_configuration.LogDecoratorEnabled));
+            ReportSupportabilityCountMetric(MetricNames.GetSupportabilityLogLabelsConfiguredName(_configuration.LabelsEnabled));
         }
 
         #endregion

src/Agent/NewRelic/Agent/Core/Aggregators/LogEventAggregator.cs

@@ -13,6 +13,7 @@
 using NewRelic.Agent.Extensions.Collections;
 using NewRelic.Agent.Extensions.Logging;
 using NewRelic.Agent.Core.SharedInterfaces;
+using NewRelic.Agent.Core.Labels;
 
 namespace NewRelic.Agent.Core.Aggregators
 {
@@ -31,14 +32,17 @@ public class LogEventAggregator : AbstractAggregator<LogEventWireModel>, ILogEve
         private const double ReservoirReductionSizeMultiplier = 0.5;
 
         private readonly IAgentHealthReporter _agentHealthReporter;
+        private readonly ILabelsService _labelsService;
 
         private ConcurrentPriorityQueue<PrioritizedNode<LogEventWireModel>> _logEvents = new ConcurrentPriorityQueue<PrioritizedNode<LogEventWireModel>>(0);
         private int _logsDroppedCount;
 
-        public LogEventAggregator(IDataTransportService dataTransportService, IScheduler scheduler, IProcessStatic processStatic, IAgentHealthReporter agentHealthReporter)
+        public LogEventAggregator(IDataTransportService dataTransportService, IScheduler scheduler, IProcessStatic processStatic, IAgentHealthReporter agentHealthReporter,
+            ILabelsService labelsService)
             : base(dataTransportService, scheduler, processStatic)
         {
             _agentHealthReporter = agentHealthReporter;
+            _labelsService = labelsService;
             ResetCollections(_configuration.LogEventsMaxSamplesStored);
         }
 
@@ -98,6 +102,7 @@ protected void InternalHarvest(string transactionId = null)
                 _configuration.ApplicationNames.ElementAt(0),
                 _configuration.EntityGuid,
                 hostname,
+                _configuration.LabelsEnabled ? _labelsService.GetFilteredLabels(_configuration.LabelsExclude) : [],
                 aggregatedEvents);
 
             var responseStatus = DataTransportService.Send(modelsCollection, transactionId);

src/Agent/NewRelic/Agent/Core/Config/Configuration.cs

@@ -5472,6 +5472,8 @@ public partial class configurationApplicationLoggingForwarding
 
         private configurationApplicationLoggingForwardingContextData contextDataField;
 
+        private configurationApplicationLoggingForwardingLabels labelsField;
+
         private bool enabledField;
 
         private int maxSamplesStoredField;
@@ -5483,6 +5485,7 @@ public partial class configurationApplicationLoggingForwarding
         /// </summary>
         public configurationApplicationLoggingForwarding()
         {
+            this.labelsField = new configurationApplicationLoggingForwardingLabels();
             this.contextDataField = new configurationApplicationLoggingForwardingContextData();
             this.enabledField = true;
             this.maxSamplesStoredField = 10000;
@@ -5500,6 +5503,18 @@ public configurationApplicationLoggingForwardingContextData contextData
             }
         }
 
+        public configurationApplicationLoggingForwardingLabels labels
+        {
+            get
+            {
+                return this.labelsField;
+            }
+            set
+            {
+                this.labelsField = value;
+            }
+        }
+
         [System.Xml.Serialization.XmlAttributeAttribute()]
         [System.ComponentModel.DefaultValueAttribute(true)]
         public bool enabled
@@ -5628,6 +5643,65 @@ public virtual configurationApplicationLoggingForwardingContextData Clone()
         #endregion
     }
 
+    [System.CodeDom.Compiler.GeneratedCodeAttribute("Xsd2Code", "3.6.0.20097")]
+    [System.SerializableAttribute()]
+    [System.ComponentModel.DesignerCategoryAttribute("code")]
+    [System.Xml.Serialization.XmlTypeAttribute(AnonymousType=true, Namespace="urn:newrelic-config")]
+    public partial class configurationApplicationLoggingForwardingLabels
+    {
+
+        private bool enabledField;
+
+        private string excludeField;
+
+        /// <summary>
+        /// configurationApplicationLoggingForwardingLabels class constructor
+        /// </summary>
+        public configurationApplicationLoggingForwardingLabels()
+        {
+            this.enabledField = false;
+            this.excludeField = "";
+        }
+
+        [System.Xml.Serialization.XmlAttributeAttribute()]
+        [System.ComponentModel.DefaultValueAttribute(false)]
+        public bool enabled
+        {
+            get
+            {
+                return this.enabledField;
+            }
+            set
+            {
+                this.enabledField = value;
+            }
+        }
+
+        [System.Xml.Serialization.XmlAttributeAttribute()]
+        [System.ComponentModel.DefaultValueAttribute("")]
+        public string exclude
+        {
+            get
+            {
+                return this.excludeField;
+            }
+            set
+            {
+                this.excludeField = value;
+            }
+        }
+
+        #region Clone method
+        /// <summary>
+        /// Create a clone of this configurationApplicationLoggingForwardingLabels object
+        /// </summary>
+        public virtual configurationApplicationLoggingForwardingLabels Clone()
+        {
+            return ((configurationApplicationLoggingForwardingLabels)(this.MemberwiseClone()));
+        }
+        #endregion
+    }
+
     [System.CodeDom.Compiler.GeneratedCodeAttribute("Xsd2Code", "3.6.0.20097")]
     [System.SerializableAttribute()]
     [System.ComponentModel.DesignerCategoryAttribute("code")]

src/Agent/NewRelic/Agent/Core/Config/Configuration.xsd

@@ -1816,6 +1816,29 @@
                                                 </xs:attribute>
                                             </xs:complexType>
                                         </xs:element>
+                                        <xs:element name="labels" minOccurs="0" maxOccurs="1">
+                                            <xs:annotation>
+                                                <xs:documentation>
+                                                    Include configured labels with log records.
+                                                </xs:documentation>
+                                            </xs:annotation>
+                                            <xs:complexType>
+                                                <xs:attribute name="enabled" type="xs:boolean" default="false">
+                                                    <xs:annotation>
+                                                        <xs:documentation>
+                                                            Controls whether or not labels are included with log records. Defaults to false.
+                                                        </xs:documentation>
+                                                    </xs:annotation>
+                                                </xs:attribute>
+                                                <xs:attribute name="exclude" type="xs:string" default="">
+                                                    <xs:annotation>
+                                                        <xs:documentation>
+                                                            A comma-separated list of case-insensitive strings that define the labels that should NOT be added to log records.
+                                                        </xs:documentation>
+                                                    </xs:annotation>
+                                                </xs:attribute>
+                                            </xs:complexType>
+                                        </xs:element>
                                     </xs:sequence>
                                     <xs:attribute name="enabled" type="xs:boolean" default="true">
                                         <xs:annotation>

src/Agent/NewRelic/Agent/Core/Configuration/DefaultConfiguration.cs

@@ -2062,6 +2062,33 @@ public virtual HashSet<string> LogLevelDenyList
             }
         }
 
+        public virtual bool LabelsEnabled
+        {
+            get
+            {
+                return LogEventCollectorEnabled &&
+                    EnvironmentOverrides(_localConfiguration.applicationLogging.forwarding.labels.enabled, "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_ENABLED");
+            }
+        }
+
+        private HashSet<string> _labelsExclude;
+        public virtual IEnumerable<string> LabelsExclude
+        {
+            get
+            {
+                if (_labelsExclude == null)
+                {
+                    _labelsExclude = new HashSet<string>(
+                        EnvironmentOverrides(_localConfiguration.applicationLogging.forwarding.labels.exclude,
+                                "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_EXCLUDE")
+                            ?.Split(new[] { StringSeparators.CommaChar, ' ' }, StringSplitOptions.RemoveEmptyEntries)
+                        ?? Enumerable.Empty<string>());
+                }
+
+                return _labelsExclude;
+            }
+        }
+
         #endregion
 
         private IEnumerable<IDictionary<string, string>> _ignoredInstrumentation;

src/Agent/NewRelic/Agent/Core/Configuration/ReportedConfiguration.cs

@@ -638,6 +638,12 @@ public ReportedConfiguration(IConfiguration configuration)
         [JsonProperty("application_logging.forwarding.context_data.exclude")]
         public IEnumerable<string> ContextDataExclude => _configuration.ContextDataExclude;
 
+        [JsonProperty("application_logging.forwarding.labels.enabled")]
+        public bool LabelsEnabled => _configuration.LabelsEnabled;
+
+        [JsonProperty("application_logging.forwarding.labels.exclude")]
+        public IEnumerable<string> LabelsExclude => _configuration.LabelsExclude;
+
         [JsonProperty("metrics.harvest_cycle")]
         public TimeSpan MetricsHarvestCycle => _configuration.MetricsHarvestCycle;
 

src/Agent/NewRelic/Agent/Core/JsonConverters/LogEventWireModelCollectionJsonConverter.cs

@@ -1,4 +1,4 @@
-// Copyright 2020 New Relic, Inc. All rights reserved.
+// Copyright 2020 New Relic, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 
@@ -25,6 +25,7 @@ public class LogEventWireModelCollectionJsonConverter : JsonConverter<LogEventWi
         private const string ErrorMessage = "error.message";
         private const string ErrorClass = "error.class";
         private const string Context = "context";
+        private const string LabelPrefix = "tags.";
 
         public override LogEventWireModelCollection ReadJson(JsonReader reader, Type objectType, LogEventWireModelCollection existingValue, bool hasExistingValue, JsonSerializer serializer)
         {
@@ -50,6 +51,16 @@ private static void WriteJsonImpl(JsonWriter jsonWriter, LogEventWireModelCollec
             jsonWriter.WriteValue(value.EntityGuid);
             jsonWriter.WritePropertyName(Hostname);
             jsonWriter.WriteValue(value.Hostname);
+
+            if (value.Labels != null)
+            {
+                foreach (var label in value.Labels)
+                {
+                    jsonWriter.WritePropertyName(LabelPrefix + label.Type);
+                    jsonWriter.WriteValue(label.Value);
+                }
+            }
+
             jsonWriter.WriteEndObject();
             jsonWriter.WriteEndObject();
 

src/Agent/NewRelic/Agent/Core/Labels/ILabelsService.cs

@@ -9,5 +9,7 @@ namespace NewRelic.Agent.Core.Labels
     public interface ILabelsService : IDisposable
     {
         IEnumerable<Label> Labels { get; }
+
+        IEnumerable<Label> GetFilteredLabels(IEnumerable<string> labelsToExclude);
     }
 }

src/Agent/NewRelic/Agent/Core/Labels/LabelsService.cs

@@ -19,26 +19,34 @@ public class LabelsService : ILabelsService
 
         private readonly IConfigurationService _configurationService;
 
-        public IEnumerable<Label> Labels { get { return GetLabelsFromConfiguration(); } }
+        public IEnumerable<Label> Labels { get { return GetLabelsFromConfiguration([]); } }
+
+        public IEnumerable<Label> GetFilteredLabels(IEnumerable<string> labelsToExclude)
+        {
+            return GetLabelsFromConfiguration(labelsToExclude);
+        }
 
         public LabelsService(IConfigurationService configurationService)
         {
             _configurationService = configurationService;
         }
 
-        private IEnumerable<Label> GetLabelsFromConfiguration()
+        private IEnumerable<Label> GetLabelsFromConfiguration(IEnumerable<string> labelsToExclude)
         {
             var labelsString = _configurationService.Configuration.Labels;
             if (string.IsNullOrEmpty(labelsString))
                 return Enumerable.Empty<Label>();
 
+            labelsToExclude ??= [];
+
             try
             {
                 var labels = labelsString
                     .Trim()
                     .Trim(StringSeparators.SemiColon)
                     .Split(StringSeparators.SemiColon)
                     .Select(CreateLabelFromString)
+                    .Where(label => !labelsToExclude.Contains(label.Type, StringComparer.OrdinalIgnoreCase))
                     .GroupBy(label => label.Type)
                     .Select(labelGrouping => labelGrouping.Last())
                     .Take(MaxLabels)

src/Agent/NewRelic/Agent/Core/Metrics/MetricNames.cs

@@ -1111,11 +1111,13 @@ public static string GetLoggingMetricsDeniedName()
         private const string Metrics = "Metrics";
         private const string Forwarding = "Forwarding";
         private const string LocalDecorating = "LocalDecorating";
+        private const string Labels = "Labels";
         private const string DotNet = "DotNET";
 
         private const string SupportabilityLogMetricsConfigPs = SupportabilityLoggingEventsPs + Metrics + PathSeparator + DotNet + PathSeparator;
         private const string SupportabilityLogForwardingConfigPs = SupportabilityLoggingEventsPs + Forwarding + PathSeparator + DotNet + PathSeparator;
         private const string SupportabilityLogDecoratingConfigPs = SupportabilityLoggingEventsPs + LocalDecorating + PathSeparator + DotNet + PathSeparator;
+        private const string SupportabilityLogLabelsConfigPs = SupportabilityLoggingEventsPs + Labels + PathSeparator + DotNet + PathSeparator;
 
         public static string GetSupportabilityLogMetricsConfiguredName(bool enabled)
         {
@@ -1127,6 +1129,11 @@ public static string GetSupportabilityLogForwardingConfiguredName(bool enabled)
             return SupportabilityLogForwardingConfigPs + (enabled ? Enabled : Disabled);
         }
 
+        public static string GetSupportabilityLogLabelsConfiguredName(bool enabled)
+        {
+            return SupportabilityLogLabelsConfigPs + (enabled ? Enabled : Disabled);
+        }
+
         public static string GetSupportabilityLogDecoratingConfiguredName(bool enabled)
         {
             return SupportabilityLogDecoratingConfigPs + (enabled ? Enabled : Disabled);

src/Agent/NewRelic/Agent/Core/WireModels/LogEventWireModelCollection.cs

@@ -1,7 +1,8 @@
-// Copyright 2020 New Relic, Inc. All rights reserved.
+// Copyright 2020 New Relic, Inc. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 using NewRelic.Agent.Core.JsonConverters;
+using NewRelic.Agent.Core.Labels;
 using Newtonsoft.Json;
 using System.Collections.Generic;
 
@@ -13,14 +14,17 @@ public class LogEventWireModelCollection
         public string EntityName { get; }
         public string EntityGuid { get; }
         public string Hostname { get; }
+        public IEnumerable<Label> Labels { get; }
 
         public IList<LogEventWireModel> LoggingEvents { get; }
 
-        public LogEventWireModelCollection(string entityName, string entityGuid, string hostname, IList<LogEventWireModel> loggingEvents)
+        public LogEventWireModelCollection(string entityName, string entityGuid, string hostname,
+            IEnumerable<Label> labels, IList<LogEventWireModel> loggingEvents)
         {
             EntityName = entityName;
             EntityGuid = entityGuid;
             Hostname = hostname;
+            Labels = labels;
             LoggingEvents = loggingEvents;
         }
     }

src/Agent/NewRelic/Agent/Extensions/NewRelic.Agent.Extensions/Configuration/IConfiguration.cs

@@ -198,8 +198,10 @@ public interface IConfiguration
         bool LogDecoratorEnabled { get; }
         HashSet<string> LogLevelDenyList { get; }
         bool ContextDataEnabled { get; }
+        bool LabelsEnabled { get; }
         IEnumerable<string> ContextDataInclude { get; }
         IEnumerable<string> ContextDataExclude { get; }
+        IEnumerable<string> LabelsExclude { get; }
         bool AppDomainCachingDisabled { get; }
         bool ForceNewTransactionOnNewThread { get; }
         bool CodeLevelMetricsEnabled { get; }