[8.x](backport #40411) {,x-pack/}metricbeat/module/prometheus/{collec…

…tor,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` (#40784) * {,x-pack/}metricbeat/module/prometheus/{collector,remote_write}: Add `metrics_count` to Prometheus module if `metrics_count` is `true` (#40411) Co-authored-by: Ishleen Kaur <102962586+ishleenk17@users.noreply.github.com> Co-authored-by: Aman <38116245+devamanv@users.noreply.github.com> (cherry picked from commit 3c9563c) * Update CHANGELOG.next.asciidoc --------- Co-authored-by: subham sarkar <subham.sarkar@elastic.co>
elastic · Sep 13, 2024 · 9136549 · 9136549
1 parent caccf51
commit 9136549
Show file tree

Hide file tree

Showing 36 changed files with 1,422 additions and 486 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -328,6 +328,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new metricset network for the vSphere module. {pull}40559[40559]
 - Add new metricset resourcepool for the vSphere module. {pull}40456[40456]
 - Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661]
+- Add `metrics_count` to Prometheus module if `metrics_count: true` is set. {pull}40411[40411]
 
 *Metricbeat*
 

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -58258,6 +58258,16 @@ Stats scraped from a Prometheus endpoint.
 
 
 
+*`metrics_count`*::
++
+--
+Number of metrics per Elasticsearch document.
+
+
+type: long
+
+--
+
 
 *`prometheus.labels.*`*::
 +

diff --git a/metricbeat/docs/modules/prometheus.asciidoc b/metricbeat/docs/modules/prometheus.asciidoc
@@ -50,6 +50,9 @@ metricbeat.modules:
   #username: "user"
   #password: "secret"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  #metrics_count: false
+
   # This can be used for service account based authorization:
   #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   #ssl.certificate_authorities:
@@ -62,6 +65,9 @@ metricbeat.modules:
 #  host: "localhost"
 #  port: "9201"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  #metrics_count: false
+
   # Secure settings for the server using TLS/SSL:
   #ssl.certificate: "/etc/pki/server/cert.pem"
   #ssl.key: "/etc/pki/server/cert.key"

diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
@@ -888,6 +888,9 @@ metricbeat.modules:
   #username: "user"
   #password: "secret"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  #metrics_count: false
+
   # This can be used for service account based authorization:
   #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   #ssl.certificate_authorities:
@@ -900,6 +903,9 @@ metricbeat.modules:
 #  host: "localhost"
 #  port: "9201"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  #metrics_count: false
+
   # Secure settings for the server using TLS/SSL:
   #ssl.certificate: "/etc/pki/server/cert.pem"
   #ssl.key: "/etc/pki/server/cert.key"

diff --git a/metricbeat/module/prometheus/_meta/config.epr.yml b/metricbeat/module/prometheus/_meta/config.epr.yml
@@ -10,6 +10,9 @@
   username: "user"
   password: "secret"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  #metrics_count: false
+
   # This can be used for service account based authorization:
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   ssl.certificate_authorities:
@@ -28,6 +31,9 @@
   host: "localhost"
   port: "9201"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  metrics_count: false
+
   # Secure settings for the server using TLS/SSL:
   ssl.enabled: false
   ssl.certificate: "/etc/pki/server/cert.pem"

diff --git a/metricbeat/module/prometheus/_meta/config.yml b/metricbeat/module/prometheus/_meta/config.yml
@@ -10,6 +10,9 @@
   #username: "user"
   #password: "secret"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  #metrics_count: false
+
   # This can be used for service account based authorization:
   #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
   #ssl.certificate_authorities:
@@ -22,6 +25,9 @@
 #  host: "localhost"
 #  port: "9201"
 
+  # Count number of metrics present in Elasticsearch document (default: false)
+  #metrics_count: false
+
   # Secure settings for the server using TLS/SSL:
   #ssl.certificate: "/etc/pki/server/cert.pem"
   #ssl.key: "/etc/pki/server/cert.key"

diff --git a/metricbeat/module/prometheus/_meta/fields.yml b/metricbeat/module/prometheus/_meta/fields.yml
@@ -6,6 +6,10 @@
   release: ga
   settings: ["ssl", "http"]
   fields:
+    - name: metrics_count
+      type: long
+      description: >
+        Number of metrics per Elasticsearch document.
     - name: prometheus
       type: group
       fields:

diff --git a/metricbeat/module/prometheus/collector/_meta/data.json b/metricbeat/module/prometheus/collector/_meta/data.json
@@ -5,20 +5,23 @@
         "duration": 115000,
         "module": "prometheus"
     },
+    "metrics_count": 2,
     "metricset": {
         "name": "collector",
         "period": 10000
     },
     "prometheus": {
         "labels": {
-            "job": "prometheus"
+            "job": "prometheus",
+            "listener_name": "http"
         },
         "metrics": {
-            "up": 1
+            "net_conntrack_listener_conn_accepted_total": 3,
+            "net_conntrack_listener_conn_closed_total": 0
         }
     },
     "service": {
         "address": "127.0.0.1:55555",
         "type": "prometheus"
     }
-}
+}
diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/config.yml b/metricbeat/module/prometheus/collector/_meta/testdata/config.yml
@@ -2,3 +2,5 @@ type: http
 url: "/metrics"
 suffix: plain
 remove_fields_from_comparison: ["prometheus.labels.instance"]
+module:
+  metrics_count: true
diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json b/metricbeat/module/prometheus/collector/_meta/testdata/docs.plain-expected.json
@@ -5,17 +5,20 @@
             "duration": 115000,
             "module": "prometheus"
         },
+        "metrics_count": 2,
         "metricset": {
             "name": "collector",
             "period": 10000
         },
         "prometheus": {
             "labels": {
-                "instance": "127.0.0.1:41103",
-                "job": "prometheus"
+                "instance": "127.0.0.1:61483",
+                "job": "prometheus",
+                "listener_name": "http"
             },
             "metrics": {
-                "up": 1
+                "net_conntrack_listener_conn_accepted_total": 3,
+                "net_conntrack_listener_conn_closed_total": 0
             }
         },
         "service": {
@@ -29,19 +32,18 @@
             "duration": 115000,
             "module": "prometheus"
         },
+        "metrics_count": 1,
         "metricset": {
             "name": "collector",
             "period": 10000
         },
         "prometheus": {
             "labels": {
-                "instance": "127.0.0.1:41103",
-                "job": "prometheus",
-                "listener_name": "http"
+                "instance": "127.0.0.1:61483",
+                "job": "prometheus"
             },
             "metrics": {
-                "net_conntrack_listener_conn_accepted_total": 3,
-                "net_conntrack_listener_conn_closed_total": 0
+                "up": 1
             }
         },
         "service": {

diff --git a/metricbeat/module/prometheus/collector/_meta/testdata/duplicate-metrics.plain-expected.json b/metricbeat/module/prometheus/collector/_meta/testdata/duplicate-metrics.plain-expected.json
@@ -5,18 +5,19 @@
             "duration": 115000,
             "module": "prometheus"
         },
+        "metrics_count": 1,
         "metricset": {
             "name": "collector",
             "period": 10000
         },
         "prometheus": {
             "labels": {
-                "instance": "127.0.0.1:44633",
+                "instance": "127.0.0.1:61485",
                 "job": "prometheus",
-                "name": "PS Scavenge"
+                "name": "PS MarkSweep"
             },
             "metrics": {
-                "base_gc_total_total": 34
+                "base_gc_total_total": 4
             }
         },
         "service": {
@@ -30,18 +31,18 @@
             "duration": 115000,
             "module": "prometheus"
         },
+        "metrics_count": 1,
         "metricset": {
             "name": "collector",
             "period": 10000
         },
         "prometheus": {
             "labels": {
-                "instance": "127.0.0.1:44633",
-                "job": "prometheus",
-                "name": "PS MarkSweep"
+                "instance": "127.0.0.1:61485",
+                "job": "prometheus"
             },
             "metrics": {
-                "base_gc_total_total": 4
+                "up": 1
             }
         },
         "service": {
@@ -55,17 +56,19 @@
             "duration": 115000,
             "module": "prometheus"
         },
+        "metrics_count": 1,
         "metricset": {
             "name": "collector",
             "period": 10000
         },
         "prometheus": {
             "labels": {
-                "instance": "127.0.0.1:44633",
-                "job": "prometheus"
+                "instance": "127.0.0.1:61485",
+                "job": "prometheus",
+                "name": "PS Scavenge"
             },
             "metrics": {
-                "up": 1
+                "base_gc_total_total": 34
             }
         },
         "service": {