From b8610d87bb55de1f4413460c05da529dab60c1c1 Mon Sep 17 00:00:00 2001 From: Jixiang Jin Date: Thu, 21 Sep 2023 16:21:44 +0800 Subject: [PATCH] Replace loggingMetricExporter with OtlpJsonLoggingMetricExporter. (#7373) * Replace loggingMetricExporter with OtlpJsonLoggingMetricExporter. * Fix bazel workspace * Move OtlpJsonLoggingMetricExporter to proxy and controller. * Fix Bazel imports. --- WORKSPACE | 1 + broker/BUILD.bazel | 1 + .../rocketmq/broker/metrics/BrokerMetricsManager.java | 9 +++++---- broker/src/main/resources/rmq.broker.logback.xml | 5 +++++ common/BUILD.bazel | 1 + common/pom.xml | 4 ++++ controller/BUILD.bazel | 1 + .../controller/metrics/ControllerMetricsManager.java | 9 +++++---- pom.xml | 5 +++++ proxy/BUILD.bazel | 1 + .../rocketmq/proxy/metrics/ProxyMetricsManager.java | 11 ++++++----- proxy/src/main/resources/rmq.proxy.logback.xml | 5 +++++ tieredstore/BUILD.bazel | 1 + 13 files changed, 41 insertions(+), 13 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 3126f2d1d5d..8640485ba24 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -92,6 +92,7 @@ maven_install( "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", "io.opentelemetry:opentelemetry-sdk:1.29.0", + "io.opentelemetry:opentelemetry-exporter-logging-otlp:1.29.0", "com.squareup.okio:okio-jvm:3.0.0", "io.opentelemetry:opentelemetry-api:1.29.0", "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel index 6adcdc7b99c..64cb1b34155 100644 --- a/broker/BUILD.bazel +++ b/broker/BUILD.bazel @@ -44,6 +44,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", "@maven//:io_opentelemetry_opentelemetry_exporter_logging", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java index 6af5afc1413..39af18b9faa 100644 --- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java @@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongHistogram; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongGauge; -import io.opentelemetry.exporter.logging.LoggingMetricExporter; +import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; @@ -36,6 +36,7 @@ import io.opentelemetry.sdk.metrics.View; import io.opentelemetry.sdk.metrics.ViewBuilder; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; import io.opentelemetry.sdk.resources.Resource; @@ -113,7 +114,7 @@ public class BrokerMetricsManager { private OtlpGrpcMetricExporter metricExporter; private PeriodicMetricReader periodicMetricReader; private PrometheusHttpServer prometheusHttpServer; - private LoggingMetricExporter loggingMetricExporter; + private MetricExporter loggingMetricExporter; private Meter brokerMeter; public static Supplier attributesBuilderSupplier = Attributes::builder; @@ -327,8 +328,8 @@ private void init() { if (metricsExporterType == MetricsExporterType.LOG) { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); - loggingMetricExporter = LoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); - java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); + java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) .setInterval(brokerConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) .build(); diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml index 3c51e59d4bc..32dc297360e 100644 --- a/broker/src/main/resources/rmq.broker.logback.xml +++ b/broker/src/main/resources/rmq.broker.logback.xml @@ -672,6 +672,11 @@ + + + + + diff --git a/common/BUILD.bazel b/common/BUILD.bazel index a95a19ccd42..e6701d0fcb5 100644 --- a/common/BUILD.bazel +++ b/common/BUILD.bazel @@ -35,6 +35,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:org_apache_commons_commons_lang3", "@maven//:org_lz4_lz4_java", "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", diff --git a/common/pom.xml b/common/pom.xml index 31eb0f087da..accc7f0a8f4 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -80,6 +80,10 @@ io.opentelemetry opentelemetry-sdk + + io.opentelemetry + opentelemetry-exporter-logging-otlp + io.grpc grpc-stub diff --git a/controller/BUILD.bazel b/controller/BUILD.bazel index 843d9dc7766..b2b743eb2d3 100644 --- a/controller/BUILD.bazel +++ b/controller/BUILD.bazel @@ -49,6 +49,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", "@maven//:io_opentelemetry_opentelemetry_exporter_logging", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:org_slf4j_jul_to_slf4j", ], ) diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java index 650740bcc6e..be9e77eeaeb 100644 --- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java @@ -26,7 +26,7 @@ import io.opentelemetry.api.metrics.LongUpDownCounter; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongGauge; -import io.opentelemetry.exporter.logging.LoggingMetricExporter; +import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; @@ -38,6 +38,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.metrics.View; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import io.opentelemetry.sdk.resources.Resource; import java.io.File; @@ -121,7 +122,7 @@ public class ControllerMetricsManager { private PrometheusHttpServer prometheusHttpServer; - private LoggingMetricExporter loggingMetricExporter; + private MetricExporter loggingMetricExporter; public static ControllerMetricsManager getInstance(ControllerManager controllerManager) { if (instance == null) { @@ -364,8 +365,8 @@ public void init() { if (type == MetricsExporterType.LOG) { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); - loggingMetricExporter = LoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); - java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); + java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) .setInterval(config.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) .build(); diff --git a/pom.xml b/pom.xml index 9f0b3eb96ba..4b382c6dae4 100644 --- a/pom.xml +++ b/pom.xml @@ -974,6 +974,11 @@ opentelemetry-sdk ${opentelemetry.version} + + io.opentelemetry + opentelemetry-exporter-logging-otlp + ${opentelemetry.version} + org.slf4j jul-to-slf4j diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel index b4f3c16e22d..cb7af925499 100644 --- a/proxy/BUILD.bazel +++ b/proxy/BUILD.bazel @@ -52,6 +52,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", "@maven//:io_opentelemetry_opentelemetry_exporter_logging", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java index f5050858f61..2b8dac5d8be 100644 --- a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java +++ b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java @@ -21,15 +21,16 @@ import io.opentelemetry.api.common.AttributesBuilder; import io.opentelemetry.api.metrics.Meter; import io.opentelemetry.api.metrics.ObservableLongGauge; +import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; -import io.opentelemetry.exporter.logging.LoggingMetricExporter; import io.opentelemetry.sdk.OpenTelemetrySdk; import io.opentelemetry.sdk.metrics.InstrumentType; import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +import io.opentelemetry.sdk.metrics.export.MetricExporter; import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; import io.opentelemetry.sdk.resources.Resource; import java.util.HashMap; @@ -42,9 +43,9 @@ import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.metrics.MetricsExporterType; import org.apache.rocketmq.common.utils.StartAndShutdown; -import org.apache.rocketmq.proxy.config.ProxyConfig; import org.apache.rocketmq.logging.org.slf4j.Logger; import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +import org.apache.rocketmq.proxy.config.ProxyConfig; import org.slf4j.bridge.SLF4JBridgeHandler; import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA; @@ -67,7 +68,7 @@ public class ProxyMetricsManager implements StartAndShutdown { private OtlpGrpcMetricExporter metricExporter; private PeriodicMetricReader periodicMetricReader; private PrometheusHttpServer prometheusHttpServer; - private LoggingMetricExporter loggingMetricExporter; + private MetricExporter loggingMetricExporter; public static ObservableLongGauge proxyUp = null; @@ -221,8 +222,8 @@ public void start() throws Exception { if (metricsExporterType == MetricsExporterType.LOG) { SLF4JBridgeHandler.removeHandlersForRootLogger(); SLF4JBridgeHandler.install(); - loggingMetricExporter = LoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); - java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); + loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); + java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) .setInterval(proxyConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) .build(); diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml index d38827f92d8..f968a45e631 100644 --- a/proxy/src/main/resources/rmq.proxy.logback.xml +++ b/proxy/src/main/resources/rmq.proxy.logback.xml @@ -418,6 +418,11 @@ + + + + + diff --git a/tieredstore/BUILD.bazel b/tieredstore/BUILD.bazel index 5b3885a4eae..dea2c561b20 100644 --- a/tieredstore/BUILD.bazel +++ b/tieredstore/BUILD.bazel @@ -36,6 +36,7 @@ java_library( "@maven//:io_opentelemetry_opentelemetry_sdk", "@maven//:io_opentelemetry_opentelemetry_sdk_common", "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", + "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", "@maven//:org_apache_commons_commons_lang3", "@maven//:org_apache_tomcat_annotations_api", "@maven//:com_alibaba_fastjson",