[8.x](backport #40766) [Metricbeat][vSphere] New metrics support and …

…minor changes to existing metricsets (#40807) * [Metricbeat][vSphere] New metrics support and minor changes to existing metricsets (#40766) * initial commit * update field mappings * update virtual machine data.json * format data.json * resolve review comments * add changelog entry (cherry picked from commit 3f44bd1) # Conflicts: # metricbeat/module/vsphere/datastore/_meta/data.json # metricbeat/module/vsphere/datastore/datastore.go # metricbeat/module/vsphere/fields.go # metricbeat/module/vsphere/host/_meta/data.json # metricbeat/module/vsphere/host/host.go # metricbeat/module/vsphere/virtualmachine/_meta/data.json # metricbeat/module/vsphere/virtualmachine/data_test.go # metricbeat/module/vsphere/virtualmachine/virtualmachine.go * fix existing conflicts * remove extra changelog entries --------- Co-authored-by: Kush Rana <89848966+kush-elastic@users.noreply.github.com> Co-authored-by: Kush Rana <kush.rana@elastic.co>
elastic · Oct 1, 2024 · c3a001c · c3a001c
1 parent ffd7f25
commit c3a001c
Show file tree

Hide file tree

Showing 15 changed files with 379 additions and 298 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -339,6 +339,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add new metricset resourcepool for the vSphere module. {pull}40456[40456]
 - Add metrics for the vSphere Virtualmachine metricset. {pull}40485[40485]
 - Log the total time taken for GCP `ListTimeSeries` and `AggregatedList` requests {pull}40661[40661]
+- Add new metrics fot datastore and minor changes to overall vSphere metrics {pull}40766[40766]
 - Add new metrics for the vSphere Host metricset. {pull}40429[40429]
 - Add new metrics for the vSphere Datastore metricset. {pull}40441[40441]
 - Add new metricset cluster for the vSphere module. {pull}40536[40536]

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -67175,76 +67175,92 @@ format: percent
 
 --
 
-*`vsphere.datastore.fstype`*::
+*`vsphere.datastore.disk.capacity.bytes`*::
 +
 --
-Filesystem type.
+Configured size of the datastore.
 
 
-type: keyword
+type: long
+
+format: bytes
 
 --
 
-*`vsphere.datastore.host.count`*::
+*`vsphere.datastore.disk.capacity.usage.bytes`*::
 +
 --
-Number of hosts.
+The amount of storage capacity currently being consumed by datastore.
 
 
 type: long
 
+format: bytes
+
 --
 
-*`vsphere.datastore.host.names`*::
+*`vsphere.datastore.disk.provisioned.bytes`*::
 +
 --
-List of all the host names.
+Amount of storage set aside for use by a datastore.
+
+
+type: long
+
+format: bytes
+
+--
+
+*`vsphere.datastore.fstype`*::
++
+--
+Filesystem type.
 
 
 type: keyword
 
 --
 
-*`vsphere.datastore.iops`*::
+*`vsphere.datastore.host.count`*::
 +
 --
-Storage I/O Control aggregated Input/Output Operations Per Second.
+Number of hosts.
 
 
 type: long
 
 --
 
-*`vsphere.datastore.name`*::
+*`vsphere.datastore.host.names`*::
 +
 --
-Datastore name.
+List of all the host names.
 
 
 type: keyword
 
 --
 
-*`vsphere.datastore.read.bytes`*::
+*`vsphere.datastore.name`*::
 +
 --
-Rate of reading data from the datastore.
-
+Datastore name.
 
-type: long
 
-format: bytes
+type: keyword
 
 --
 
-*`vsphere.datastore.read.latency.total.ms`*::
+*`vsphere.datastore.read.bytes`*::
 +
 --
-Average amount of time for a read operation from the datastore in milliseconds.
+Rate of reading data from the datastore.
 
 
 type: long
 
+format: bytes
+
 --
 
 *`vsphere.datastore.status`*::
@@ -67289,16 +67305,6 @@ format: bytes
 
 --
 
-*`vsphere.datastore.write.latency.total.ms`*::
-+
---
-Average amount of time for a write operation from the datastore in milliseconds.
-
-
-type: long
-
---
-
 [float]
 === datastorecluster
 
@@ -68281,10 +68287,10 @@ type: long
 --
 
 
-*`vsphere.virtualmachine.snapshot.info`*::
+*`vsphere.virtualmachine.snapshot.info.*`*::
 +
 --
-Deatils of the snapshots of this virtualmachine.
+Details of the snapshots of this virtualmachine.
 
 type: object
 

diff --git a/metricbeat/module/vsphere/datastore/_meta/data.json b/metricbeat/module/vsphere/datastore/_meta/data.json
@@ -10,39 +10,55 @@
         "period": 10000
     },
     "service": {
-        "address": "127.0.0.1:55632",
+        "address": "127.0.0.1:33365",
         "type": "vsphere"
     },
     "vsphere": {
         "datastore": {
-            "capacity": {
-                "free": {
-                    "bytes": 10973641441280
-                },
-                "total": {
-                    "bytes": 10995116277760
-                },
-                "used": {
-                    "bytes": 21474836480,
-                    "pct": 0.001953125
-                }
-            },
-            "fstype": "OTHER",
             "host": {
                 "count": 1,
                 "names": [
-                    "localhost_localdomain"
+                    "DC3_H0"
                 ]
             },
-            "name": "LocalDS_0",
             "status": "green",
             "vm": {
-                "count": 2,
+                "count": 6,
                 "names": [
-                    "ha-host_VM0",
-                    "ha-host_VM1"
+                    "DC3_H0_VM0"
                 ]
-            }
+            },
+            "read": {
+                "bytes": 0
+            },
+            "write": {
+                "bytes": 337000
+            },
+            "disk": {
+                "capacity": {
+                    "usage": {
+                        "bytes": 520505786368
+                    },
+                    "bytes": 1610344300544
+                },
+                "provisioned": {
+                    "bytes": 520505786368
+                }
+            },
+            "capacity": {
+                "free": {
+                    "bytes": 37120094208
+                },
+                "total": {
+                    "bytes": 74686664704
+                },
+                "used": {
+                    "bytes": 37566570496,
+                    "pct": 0.502988996026061
+                }
+            },
+            "fstype": "local",
+            "name": "LocalDS_0"
         }
     }
 }
diff --git a/metricbeat/module/vsphere/datastore/_meta/fields.yml b/metricbeat/module/vsphere/datastore/_meta/fields.yml
@@ -24,6 +24,21 @@
       description: >
         Percentage of datastore capacity used.
       format: percent
+    - name: disk.capacity.bytes
+      type: long
+      description: >
+        Configured size of the datastore.
+      format: bytes
+    - name: disk.capacity.usage.bytes
+      type: long
+      description: >
+        The amount of storage capacity currently being consumed by datastore.
+      format: bytes
+    - name: disk.provisioned.bytes
+      type: long
+      description: >
+        Amount of storage set aside for use by a datastore.
+      format: bytes
     - name: fstype
       type: keyword
       description: >
@@ -36,10 +51,6 @@
       type: keyword
       description: >
         List of all the host names.
-    - name: iops
-      type: long
-      description: >
-        Storage I/O Control aggregated Input/Output Operations Per Second.
     - name: name
       type: keyword
       description: >
@@ -49,10 +60,6 @@
       description: >
         Rate of reading data from the datastore.
       format: bytes
-    - name: read.latency.total.ms
-      type: long
-      description: >
-        Average amount of time for a read operation from the datastore in milliseconds.
     - name: status
       type: keyword
       description: >
@@ -70,8 +77,4 @@
       description: >
         Rate of writing data to the datastore.
       format: bytes
-    - name: write.latency.total.ms
-      type: long
-      description: >
-        Average amount of time for a write operation from the datastore in milliseconds.
 
diff --git a/metricbeat/module/vsphere/datastore/data.go b/metricbeat/module/vsphere/datastore/data.go
@@ -65,16 +65,16 @@ func mapPerfMetricToEvent(event mapstr.M, perfMetricMap map[string]interface{})
 	if val, exist := perfMetricMap["datastore.read.average"]; exist {
 		event.Put("read.bytes", val.(int64)*bytesMultiplier)
 	}
-	if val, exist := perfMetricMap["datastore.totalReadLatency.average"]; exist {
-		event.Put("read.latency.total.ms", val)
-	}
 	if val, exist := perfMetricMap["datastore.write.average"]; exist {
 		event.Put("write.bytes", val.(int64)*bytesMultiplier)
 	}
-	if val, exist := perfMetricMap["datastore.totalWriteLatency.average"]; exist {
-		event.Put("write.latency.total.ms", val)
+	if val, exist := perfMetricMap["disk.capacity.latest"]; exist {
+		event.Put("disk.capacity.bytes", val.(int64)*bytesMultiplier)
+	}
+	if val, exist := perfMetricMap["disk.capacity.usage.average"]; exist {
+		event.Put("disk.capacity.usage.bytes", val.(int64)*bytesMultiplier)
 	}
-	if val, exist := perfMetricMap["datastore.datastoreIops.average"]; exist {
-		event.Put("iops", val)
+	if val, exist := perfMetricMap["disk.provisioned.latest"]; exist {
+		event.Put("disk.provisioned.bytes", val.(int64)*bytesMultiplier)
 	}
 }
diff --git a/metricbeat/module/vsphere/datastore/data_test.go b/metricbeat/module/vsphere/datastore/data_test.go
@@ -28,7 +28,6 @@ import (
 )
 
 func TestEventMapping(t *testing.T) {
-	var m *DataStoreMetricSet
 	var datastoreTest = mo.Datastore{
 		Summary: types.DatastoreSummary{
 			Name:      "datastore-test",
@@ -47,23 +46,22 @@ func TestEventMapping(t *testing.T) {
 
 	var metricDataTest = metricData{
 		perfMetrics: map[string]interface{}{
-			"datastore.read.average":              int64(100),
-			"datastore.write.average":             int64(200),
-			"datastore.datastoreIops.average":     int64(10),
-			"datastore.totalReadLatency.average":  int64(100),
-			"datastore.totalWriteLatency.average": int64(100),
+			"datastore.read.average":      int64(100),
+			"datastore.write.average":     int64(200),
+			"disk.capacity.latest":        int64(10000),
+			"disk.capacity.usage.average": int64(5000),
+			"disk.provisioned.latest":     int64(5000),
 		},
 		assetNames: assetNames{
 			outputHostNames: []string{"DC3_H0"},
 			outputVmNames:   []string{"DC3_H0_VM0"},
 		},
 	}
 
-	outputEvent := m.mapEvent(datastoreTest, &metricDataTest)
+	outputEvent := (&DataStoreMetricSet{}).mapEvent(datastoreTest, &metricDataTest)
 	testEvent := mapstr.M{
 		"fstype": "local",
 		"status": types.ManagedEntityStatus("green"),
-		"iops":   int64(10),
 		"name":   "datastore-test",
 		"host": mapstr.M{
 			"count": 1,
@@ -75,19 +73,20 @@ func TestEventMapping(t *testing.T) {
 		},
 		"read": mapstr.M{
 			"bytes": int64(102400),
-			"latency": mapstr.M{
-				"total": mapstr.M{
-					"ms": int64(100),
-				},
-			},
 		},
 		"write": mapstr.M{
 			"bytes": int64(204800),
-			"latency": mapstr.M{
-				"total": mapstr.M{
-					"ms": int64(100),
+		},
+		"disk": mapstr.M{
+			"capacity": mapstr.M{
+				"bytes": int64(10240000),
+				"usage": mapstr.M{
+					"bytes": int64(5120000),
 				},
 			},
+			"provisioned": mapstr.M{
+				"bytes": int64(5120000),
+			},
 		},
 		"capacity": mapstr.M{
 			"free": mapstr.M{
@@ -104,5 +103,4 @@ func TestEventMapping(t *testing.T) {
 	}
 
 	assert.Exactly(t, outputEvent, testEvent)
-
 }