diff --git a/tests/unit/test_vmware_exporter.py b/tests/unit/test_vmware_exporter.py index 94bc5c7..b395a65 100644 --- a/tests/unit/test_vmware_exporter.py +++ b/tests/unit/test_vmware_exporter.py @@ -10,6 +10,7 @@ from twisted.web.server import NOT_DONE_YET from vmware_exporter.vmware_exporter import main, HealthzResource, VmwareCollector, VMWareMetricsResource +from vmware_exporter.defer import BranchingDeferred EPOCH = datetime.datetime(1970, 1, 1, tzinfo=pytz.utc) @@ -23,11 +24,14 @@ def _check_properties(properties): return len(property_spec.pathSet) > 0 -@mock.patch('vmware_exporter.vmware_exporter.batch_fetch_properties') -@pytest_twisted.inlineCallbacks -def test_collect_vms(batch_fetch_properties): - content = mock.Mock() +def _succeed(result): + d = BranchingDeferred() + defer.succeed(result).chainDeferred(d) + return d + +@pytest_twisted.inlineCallbacks +def test_collect_vms(): boot_time = EPOCH + datetime.timedelta(seconds=60) snapshot_1 = mock.Mock() @@ -48,18 +52,6 @@ def test_collect_vms(batch_fetch_properties): disk.capacity = 100 disk.freeSpace = 50 - batch_fetch_properties.return_value = { - 'vm-1': { - 'name': 'vm-1', - 'runtime.host': vim.ManagedObject('host-1'), - 'runtime.powerState': 'poweredOn', - 'summary.config.numCpu': 1, - 'runtime.bootTime': boot_time, - 'snapshot': snapshot, - 'guest.disk': [disk], - } - } - collect_only = { 'vms': True, 'vmguests': True, @@ -73,23 +65,28 @@ def test_collect_vms(batch_fetch_properties): 'password', collect_only, ) + collector.content = _succeed(mock.Mock()) - inventory = { - 'host-1': { - 'name': 'host-1', - 'dc': 'dc', - 'cluster': 'cluster-1', - } - } + collector.__dict__['vm_labels'] = _succeed({ + 'vm-1': ['vm-1', 'host-1', 'dc', 'cluster-1'], + }) metrics = collector._create_metric_containers() - collector._labels = {} - - with mock.patch.object(collector, '_vmware_get_vm_perf_manager_metrics'): - yield collector._vmware_get_vms(content, metrics, inventory) - - assert _check_properties(batch_fetch_properties.call_args[0][2]) + with mock.patch.object(collector, 'batch_fetch_properties') as batch_fetch_properties: + batch_fetch_properties.return_value = _succeed({ + 'vm-1': { + 'name': 'vm-1', + 'runtime.host': vim.ManagedObject('host-1'), + 'runtime.powerState': 'poweredOn', + 'summary.config.numCpu': 1, + 'runtime.bootTime': boot_time, + 'snapshot': snapshot, + 'guest.disk': [disk], + } + }) + yield collector._vmware_get_vms(metrics) + assert _check_properties(batch_fetch_properties.call_args[0][1]) # General VM metrics assert metrics['vmware_vm_power_state'].samples[0][1] == { @@ -148,8 +145,6 @@ def test_collect_vms(batch_fetch_properties): @pytest_twisted.inlineCallbacks def test_collect_vm_perf(): - content = mock.Mock() - collect_only = { 'vms': True, 'vmguests': True, @@ -164,31 +159,8 @@ def test_collect_vm_perf(): collect_only, ) - inventory = { - 'host-1': { - 'name': 'host-1', - 'dc': 'dc', - 'cluster': 'cluster-1', - } - } - metrics = collector._create_metric_containers() - collector._labels = {'vm:1': ['vm-1', 'host-1', 'dc', 'cluster-1']} - - vms = { - 'vm-1': { - 'name': 'vm-1', - 'obj': vim.ManagedObject('vm-1'), - 'runtime.powerState': 'poweredOn', - }, - 'vm-2': { - 'name': 'vm-2', - 'obj': vim.ManagedObject('vm-2'), - 'runtime.powerState': 'poweredOff', - } - } - metric_1 = mock.Mock() metric_1.id.counterId = 9 metric_1.value = [9] @@ -201,22 +173,40 @@ def test_collect_vm_perf(): ent_1.value = [metric_1, metric_2] ent_1.entity = vim.ManagedObject('vm:1') + content = mock.Mock() content.perfManager.QueryStats.return_value = [ent_1] - - with mock.patch.object(collector, '_vmware_perf_metrics') as _vmware_perf_metrics: - _vmware_perf_metrics.return_value = { - 'cpu.ready.summation': 1, - 'cpu.usage.average': 2, - 'cpu.usagemhz.average': 3, - 'disk.usage.average': 4, - 'disk.read.average': 5, - 'disk.write.average': 6, - 'mem.usage.average': 7, - 'net.received.average': 8, - 'net.transmitted.average': 9, + collector.content = _succeed(content) + + collector.__dict__['counter_ids'] = _succeed({ + 'cpu.ready.summation': 1, + 'cpu.usage.average': 2, + 'cpu.usagemhz.average': 3, + 'disk.usage.average': 4, + 'disk.read.average': 5, + 'disk.write.average': 6, + 'mem.usage.average': 7, + 'net.received.average': 8, + 'net.transmitted.average': 9, + }) + + collector.__dict__['vm_labels'] = _succeed({ + 'vm:1': ['vm-1', 'host-1', 'dc', 'cluster-1'], + }) + + collector.__dict__['vm_inventory'] = _succeed({ + 'vm:1': { + 'name': 'vm-1', + 'obj': vim.ManagedObject('vm-1'), + 'runtime.powerState': 'poweredOn', + }, + 'vm:2': { + 'name': 'vm-2', + 'obj': vim.ManagedObject('vm-2'), + 'runtime.powerState': 'poweredOff', } + }) - yield collector._vmware_get_vm_perf_manager_metrics(content, vms, metrics, inventory) + yield collector._vmware_get_vm_perf_manager_metrics(metrics) # General VM metrics assert metrics['vmware_vm_net_transmitted_average'].samples[0][1] == { @@ -228,33 +218,10 @@ def test_collect_vm_perf(): assert metrics['vmware_vm_net_transmitted_average'].samples[0][2] == 9.0 -@mock.patch('vmware_exporter.vmware_exporter.batch_fetch_properties') -def test_collect_hosts(batch_fetch_properties): - content = mock.Mock() - +@pytest_twisted.inlineCallbacks +def test_collect_hosts(): boot_time = EPOCH + datetime.timedelta(seconds=60) - batch_fetch_properties.return_value = { - 'host-1': { - 'id': 'host:1', - 'name': 'host-1', - 'runtime.powerState': 'poweredOn', - 'runtime.bootTime': boot_time, - 'runtime.connectionState': 'connected', - 'runtime.inMaintenanceMode': True, - 'summary.quickStats.overallCpuUsage': 100, - 'summary.hardware.numCpuCores': 12, - 'summary.hardware.cpuMhz': 1000, - 'summary.quickStats.overallMemoryUsage': 1024, - 'summary.hardware.memorySize': 2048 * 1024 * 1024, - }, - 'host-2': { - 'id': 'host:2', - 'name': 'host-2', - 'runtime.powerState': 'poweredOff', - } - } - collect_only = { 'vms': True, 'vmguests': True, @@ -268,22 +235,38 @@ def test_collect_hosts(batch_fetch_properties): 'password', collect_only, ) + collector.content = _succeed(mock.Mock()) - inventory = { - 'host:1': { - 'dc': 'dc', - 'cluster': 'cluster', - }, - 'host:2': { - 'dc': 'dc', - 'cluster': 'cluster', - } - } + collector.__dict__['host_labels'] = _succeed({ + 'host:1': ['host-1', 'dc', 'cluster'], + 'host:2': ['host-1', 'dc', 'cluster'], + }) metrics = collector._create_metric_containers() - collector._vmware_get_hosts(content, metrics, inventory) - assert _check_properties(batch_fetch_properties.call_args[0][2]) + with mock.patch.object(collector, 'batch_fetch_properties') as batch_fetch_properties: + batch_fetch_properties.return_value = _succeed({ + 'host:1': { + 'id': 'host:1', + 'name': 'host-1', + 'runtime.powerState': 'poweredOn', + 'runtime.bootTime': boot_time, + 'runtime.connectionState': 'connected', + 'runtime.inMaintenanceMode': True, + 'summary.quickStats.overallCpuUsage': 100, + 'summary.hardware.numCpuCores': 12, + 'summary.hardware.cpuMhz': 1000, + 'summary.quickStats.overallMemoryUsage': 1024, + 'summary.hardware.memorySize': 2048 * 1024 * 1024, + }, + 'host:2': { + 'id': 'host:2', + 'name': 'host-2', + 'runtime.powerState': 'poweredOff', + } + }) + yield collector._vmware_get_hosts(metrics) + assert _check_properties(batch_fetch_properties.call_args[0][1]) assert metrics['vmware_host_memory_max'].samples[0][1] == { 'host_name': 'host-1', @@ -298,22 +281,8 @@ def test_collect_hosts(batch_fetch_properties): assert len(metrics['vmware_host_memory_max'].samples) == 1 -@mock.patch('vmware_exporter.vmware_exporter.batch_fetch_properties') -def test_collect_datastore(batch_fetch_properties): - content = mock.Mock() - - batch_fetch_properties.return_value = { - 'datastore-1': { - 'name': 'datastore-1', - 'summary.capacity': 0, - 'summary.freeSpace': 0, - 'host': ['host-1'], - 'vm': ['vm-1'], - 'summary.accessible': True, - 'summary.maintenanceMode': 'normal', - } - } - +@pytest_twisted.inlineCallbacks +def test_collect_datastore(): collect_only = { 'vms': True, 'vmguests': True, @@ -327,18 +296,29 @@ def test_collect_datastore(batch_fetch_properties): 'password', collect_only, ) + collector.content = _succeed(mock.Mock()) - inventory = { - 'datastore-1': { - 'dc': 'dc', - 'ds_cluster': 'ds_cluster', - } - } + collector.__dict__['datastore_labels'] = _succeed({ + 'datastore-1': ['datastore-1', 'dc', 'ds_cluster'], + }) metrics = collector._create_metric_containers() - collector._vmware_get_datastores(content, metrics, inventory) - assert _check_properties(batch_fetch_properties.call_args[0][2]) + with mock.patch.object(collector, 'batch_fetch_properties') as batch_fetch_properties: + batch_fetch_properties.return_value = _succeed({ + 'datastore-1': { + 'name': 'datastore-1', + 'summary.capacity': 0, + 'summary.freeSpace': 0, + 'host': ['host-1'], + 'vm': ['vm-1'], + 'summary.accessible': True, + 'summary.maintenanceMode': 'normal', + } + }) + + yield collector._vmware_get_datastores(metrics) + assert _check_properties(batch_fetch_properties.call_args[0][1]) assert metrics['vmware_datastore_capacity_size'].samples[0][1] == { 'ds_name': 'datastore-1', @@ -379,15 +359,16 @@ def test_collect(): collect_only, ignore_ssl=True, ) + collector.content = _succeed(mock.Mock()) with contextlib.ExitStack() as stack: - stack.enter_context(mock.patch.object(collector, '_vmware_connect')) - get_inventory = stack.enter_context(mock.patch.object(collector, '_vmware_get_inventory')) - get_inventory.return_value = ([], []) - stack.enter_context(mock.patch.object(collector, '_vmware_get_vms')).return_value = defer.succeed(None) - stack.enter_context(mock.patch.object(collector, '_vmware_get_datastores')) - stack.enter_context(mock.patch.object(collector, '_vmware_get_hosts')) - stack.enter_context(mock.patch.object(collector, '_vmware_disconnect')) + stack.enter_context(mock.patch.object(collector, '_vmware_get_vms')).return_value = _succeed(True) + stack.enter_context( + mock.patch.object(collector, '_vmware_get_vm_perf_manager_metrics') + ).return_value = _succeed(True) + stack.enter_context(mock.patch.object(collector, '_vmware_get_datastores')).return_value = _succeed(True) + stack.enter_context(mock.patch.object(collector, '_vmware_get_hosts')).return_value = _succeed(True) + stack.enter_context(mock.patch.object(collector, '_vmware_disconnect')).return_value = _succeed(None) metrics = yield collector.collect() assert metrics[0].name == 'vmware_vm_power_state' @@ -410,6 +391,7 @@ def test_collect_deferred_error_works(): collect_only, ignore_ssl=True, ) + collector.content = _succeed(mock.Mock()) @defer.inlineCallbacks def _fake_get_vms(*args, **kwargs): @@ -417,18 +399,19 @@ def _fake_get_vms(*args, **kwargs): raise RuntimeError('An error has occurred') with contextlib.ExitStack() as stack: - stack.enter_context(mock.patch.object(collector, '_vmware_connect')) - get_inventory = stack.enter_context(mock.patch.object(collector, '_vmware_get_inventory')) - get_inventory.return_value = ([], []) stack.enter_context(mock.patch.object(collector, '_vmware_get_vms')).side_effect = _fake_get_vms - stack.enter_context(mock.patch.object(collector, '_vmware_get_datastores')) - stack.enter_context(mock.patch.object(collector, '_vmware_get_hosts')) - stack.enter_context(mock.patch.object(collector, '_vmware_disconnect')) + stack.enter_context( + mock.patch.object(collector, '_vmware_get_vm_perf_manager_metrics') + ).return_value = _succeed(None) + stack.enter_context(mock.patch.object(collector, '_vmware_get_datastores')).return_value = _succeed(None) + stack.enter_context(mock.patch.object(collector, '_vmware_get_hosts')).return_value = _succeed(None) + stack.enter_context(mock.patch.object(collector, '_vmware_disconnect')).return_value = _succeed(None) with pytest.raises(defer.FirstError): yield collector.collect() +@pytest_twisted.inlineCallbacks def test_vmware_get_inventory(): content = mock.Mock() @@ -436,6 +419,7 @@ def test_vmware_get_inventory(): host_1 = mock.Mock() host_1._moId = 'host:1' host_1.name = 'host-1' + host_1.summary.config.name = 'host-1.' folder_1 = mock.Mock() folder_1.host = [host_1] @@ -458,9 +442,9 @@ def test_vmware_get_inventory(): datastore_2 = vim.Datastore('datastore:2') datastore_2.__dict__['name'] = 'datastore-2' - datastore_2_folder = mock.Mock() - datastore_2_folder.childEntity = [datastore_2] - datastore_2_folder.name = 'datastore2-folder' + datastore_2_folder = vim.StoragePod('storagepod:1') + datastore_2_folder.__dict__['childEntity'] = [datastore_2] + datastore_2_folder.__dict__['name'] = 'datastore2-folder' data_center_1 = mock.Mock() data_center_1.name = 'dc-1' @@ -483,40 +467,31 @@ def test_vmware_get_inventory(): collect_only, ignore_ssl=True, ) + collector.content = content with contextlib.ExitStack() as stack: # We have to disable the LazyObject magic on pyvmomi classes so that we can use them as fakes stack.enter_context(mock.patch.object(vim.ClusterComputeResource, 'name', None)) stack.enter_context(mock.patch.object(vim.ClusterComputeResource, 'host', None)) stack.enter_context(mock.patch.object(vim.Datastore, 'name', None)) + stack.enter_context(mock.patch.object(vim.StoragePod, 'childEntity', None)) + stack.enter_context(mock.patch.object(vim.StoragePod, 'name', None)) - host, ds = collector._vmware_get_inventory(content) + host = yield collector.host_labels + ds = yield collector.datastore_labels assert host == { - 'host:1': { - 'name': 'host-1', - 'dc': 'dc-1', - 'cluster': '', - }, - 'host:2': { - 'name': 'host-2', - 'dc': 'dc-1', - 'cluster': 'compute-cluster-1', - } + 'host:1': ['host-1', 'dc-1', ''], + 'host:2': ['host-2', 'dc-1', 'compute-cluster-1'], } assert ds == { - 'datastore-1': { - 'dc': 'dc-1', - 'ds_cluster': '', - }, - 'datastore-2': { - 'dc': 'dc-1', - 'ds_cluster': 'datastore2-folder', - } + 'datastore-1': ['datastore-1', 'dc-1', ''], + 'datastore-2': ['datastore-2', 'dc-1', 'datastore2-folder'], } +@pytest_twisted.inlineCallbacks def test_vmware_connect(): collect_only = { 'vms': True, @@ -534,7 +509,7 @@ def test_vmware_connect(): ) with mock.patch('vmware_exporter.vmware_exporter.connect') as connect: - collector._vmware_connect() + yield collector.connection call_kwargs = connect.SmartConnect.call_args[1] assert call_kwargs['host'] == '127.0.0.1' @@ -543,6 +518,7 @@ def test_vmware_connect(): assert call_kwargs['sslContext'] is not None +@pytest_twisted.inlineCallbacks def test_vmware_disconnect(): collect_only = { 'vms': True, @@ -560,14 +536,15 @@ def test_vmware_disconnect(): # Mock that we have a connection connection = object() - collector.vmware_connection = connection + collector.connection = connection with mock.patch('vmware_exporter.vmware_exporter.connect') as connect: - collector._vmware_disconnect() + yield collector._vmware_disconnect() connect.Disconnect.assert_called_with(connection) -def test_vmware_perf_metrics(): +@pytest_twisted.inlineCallbacks +def test_counter_ids(): counter = mock.Mock() counter.groupInfo.key = 'a' counter.nameInfo.key = 'b' @@ -590,9 +567,9 @@ def test_vmware_perf_metrics(): 'password', collect_only, ) + collector.content = content - result = collector._vmware_perf_metrics(content) - + result = yield collector.counter_ids assert result == {'a.b.c': 1} diff --git a/vmware_exporter/defer.py b/vmware_exporter/defer.py new file mode 100644 index 0000000..05ecffc --- /dev/null +++ b/vmware_exporter/defer.py @@ -0,0 +1,100 @@ +''' +Helpers for writing efficient twisted code, optimized for coroutine scheduling efficiency +''' + +from twisted.internet import defer +from twisted.python import failure + + +class BranchingDeferred(defer.Deferred): + + ''' + This is meant for code where you are doing something like this: + + content = yield self.get_connection_content() + results = yield defer.DeferredList([ + self.get_hosts(content), + self.get_datastores(content), + ]) + + This allows get_hosts and get_datastores to run in parallel, which is good. + But what if you don't want the whole of get_hosts to wait for + get_connection_content() to be complete? + + We have a bunch of places where it would be better for scheduling if we did this: + + content = self.get_connection_content() + results = yield defer.DeferredList([ + self.get_hosts(content), + self.get_datastores(content), + ]) + + Now we don't have to wait for content to be finished before get_hosts etc + starts running. It is up to get_hosts to block on the content deferred itself. + + (Thats a contrived example, the real win is allowing host_labels and + vm_inventory to run in parallel). + + Unfortunately you can't have parallel branches blocking on the same deferred + like this with a standard Twisted deferred. + + This is a deferred that enables the parallel branching use case. + ''' + + def __init__(self): + self.callbacks = [] + self.result = None + + def callback(self, result): + self.result = result + while self.callbacks: + self.callbacks.pop(0).callback(result) + + def errback(self, err): + self.result = err + while self.callbacks: + self.callbacks.pop(0).errback(err) + + def addCallbacks(self, *args, **kwargs): + if not self.result: + d = defer.Deferred() + d.addCallbacks(*args, **kwargs) + self.callbacks.append(d) + return + + if isinstance(self.result, failure.Failure): + defer.fail(self.result).addCallbacks(*args, **kwargs) + return + + defer.succeed(self.result).addCallbacks(*args, **kwargs) + + +class run_once_property(object): + + ''' + This is a property descriptor that caches the first result it retrieves. It + does this by setting keys in self.__dict__ on the parent class instance. + This is fast - python won't even bother running our descriptor next time + because attributes in self.__dict__ on a class instance trump descriptors + on the class. + + This is intended to be used with the Collector class which has a request + bound lifecycle (this isn't going to cache stuff forever and cause memory + leaks). + ''' + + def __init__(self, callable): + self.callable = callable + + def __get__(self, obj, cls): + if obj is None: + return self + result = obj.__dict__[self.callable.__name__] = BranchingDeferred() + self.callable(obj).chainDeferred(result) + return result + + +@defer.inlineCallbacks +def parallelize(*args): + results = yield defer.DeferredList(args, fireOnOneErrback=True) + return tuple(r[1] for r in results) diff --git a/vmware_exporter/vmware_exporter.py b/vmware_exporter/vmware_exporter.py index e179cbd..75a767d 100755 --- a/vmware_exporter/vmware_exporter.py +++ b/vmware_exporter/vmware_exporter.py @@ -6,7 +6,7 @@ """ from __future__ import print_function -from datetime import datetime +import datetime # Generic imports import argparse @@ -32,6 +32,7 @@ from prometheus_client import CollectorRegistry, generate_latest from .helpers import batch_fetch_properties +from .defer import parallelize, run_once_property class VmwareCollector(): @@ -164,28 +165,10 @@ def collect(self): """ collects metrics """ vsphere_host = self.host - host_inventory = {} - ds_inventory = {} - metrics = self._create_metric_containers() log("Start collecting metrics from {0}".format(vsphere_host)) - self.vmware_connection = yield threads.deferToThread(self._vmware_connect) - if not self.vmware_connection: - log(b"Cannot connect to vmware") - return - - content = yield threads.deferToThread(self.vmware_connection.RetrieveContent) - - # Generate inventory dict - log("Starting inventory collection") - host_inventory, ds_inventory = yield threads.deferToThread( - self._vmware_get_inventory, - content, - ) - log("Finished inventory collection") - self._labels = {} collect_only = self.collect_only @@ -194,39 +177,33 @@ def collect(self): # Collect vm / snahpshot / vmguest metrics if collect_only['vmguests'] is True or collect_only['vms'] is True or collect_only['snapshots'] is True: - tasks.append(self._vmware_get_vms(content, metrics, host_inventory)) + tasks.append(self._vmware_get_vms(metrics)) + + if collect_only['vms'] is True: + tasks.append(self._vmware_get_vm_perf_manager_metrics(metrics)) # Collect Datastore metrics if collect_only['datastores'] is True: - tasks.append(threads.deferToThread( - self._vmware_get_datastores, - content, - metrics, - ds_inventory, - )) + tasks.append(self._vmware_get_datastores(metrics,)) - # Collect Hosts metrics if collect_only['hosts'] is True: - tasks.append(threads.deferToThread( - self._vmware_get_hosts, - content, - metrics, - host_inventory, - )) + tasks.append(self._vmware_get_hosts(metrics)) + + yield parallelize(*tasks) - # Waits for these to finish - yield defer.DeferredList(tasks, fireOnOneErrback=True) + yield self._vmware_disconnect - yield threads.deferToThread(self._vmware_disconnect) log("Finished collecting metrics from {0}".format(vsphere_host)) return list(metrics.values()) def _to_epoch(self, my_date): """ convert to epoch time """ - return (my_date - datetime(1970, 1, 1, tzinfo=pytz.utc)).total_seconds() + return (my_date - datetime.datetime(1970, 1, 1, tzinfo=pytz.utc)).total_seconds() - def _vmware_connect(self): + @run_once_property + @defer.inlineCallbacks + def connection(self): """ Connect to Vcenter and get connection """ @@ -235,7 +212,8 @@ def _vmware_connect(self): context = ssl._create_unverified_context() try: - vmware_connect = connect.SmartConnect( + vmware_connect = yield threads.deferToThread( + connect.SmartConnect, host=self.host, user=self.username, pwd=self.password, @@ -247,18 +225,199 @@ def _vmware_connect(self): log("Caught vmodl fault: " + error.msg) return None - def _vmware_disconnect(self): - """ - Disconnect from Vcenter - """ - connect.Disconnect(self.vmware_connection) + @run_once_property + @defer.inlineCallbacks + def content(self): + log("Retrieving service instance content") + connection = yield self.connection + content = yield threads.deferToThread( + connection.RetrieveContent + ) + + log("Retrieved service instance content") + return content + + @defer.inlineCallbacks + def batch_fetch_properties(self, objtype, properties): + content = yield self.content + batch = yield threads.deferToThread( + batch_fetch_properties, + content, + objtype, + properties, + ) + return batch + + @run_once_property + @defer.inlineCallbacks + def datastore_inventory(self): + log("Fetching vim.Datastore inventory") + start = datetime.datetime.utcnow() + properties = [ + 'name', + 'summary.capacity', + 'summary.freeSpace', + 'summary.uncommitted', + 'summary.maintenanceMode', + 'summary.type', + 'summary.accessible', + 'host', + 'vm', + ] + + datastores = yield self.batch_fetch_properties( + vim.Datastore, + properties + ) + log("Fetched vim.Datastore inventory (%s)", datetime.datetime.utcnow() - start) + return datastores + + @run_once_property + @defer.inlineCallbacks + def host_system_inventory(self): + log("Fetching vim.HostSystem inventory") + start = datetime.datetime.utcnow() + properties = [ + 'name', + 'parent', + 'summary.hardware.numCpuCores', + 'summary.hardware.cpuMhz', + 'summary.hardware.memorySize', + 'runtime.powerState', + 'runtime.bootTime', + 'runtime.connectionState', + 'runtime.inMaintenanceMode', + 'summary.quickStats.overallCpuUsage', + 'summary.quickStats.overallMemoryUsage', + ] + + host_systems = yield self.batch_fetch_properties( + vim.HostSystem, + properties, + ) + log("Fetched vim.HostSystem inventory (%s)", datetime.datetime.utcnow() - start) + return host_systems + + @run_once_property + @defer.inlineCallbacks + def vm_inventory(self): + log("Fetching vim.VirtualMachine inventory") + start = datetime.datetime.utcnow() + properties = [ + 'name', + 'runtime.host', + 'parent', + ] + + if self.collect_only['vms'] is True: + properties.extend([ + 'runtime.powerState', + 'runtime.bootTime', + 'summary.config.numCpu', + ]) + + if self.collect_only['vmguests'] is True: + properties.append('guest.disk') + + if self.collect_only['snapshots'] is True: + properties.append('snapshot') + + virtual_machines = yield self.batch_fetch_properties( + vim.VirtualMachine, + properties, + ) + log("Fetched vim.VirtualMachine inventory (%s)", datetime.datetime.utcnow() - start) + return virtual_machines + + @run_once_property + @defer.inlineCallbacks + def datacenter_inventory(self): + content = yield self.content + # FIXME: It's unclear if this is operating on data already fetched in + # content or if this is doing stealth HTTP requests + # Right now we assume it does stealth lookups + datacenters = yield threads.deferToThread(lambda: content.rootFolder.childEntity) + return datacenters + + @run_once_property + @defer.inlineCallbacks + def datastore_labels(self): + + def _collect(dc, node): + inventory = {} + for folder in node.childEntity: # Iterate through datastore folders + if isinstance(folder, vim.Datastore): # Unclustered datastore + row = inventory[folder.name] = [ + folder.name, + dc.name, + ] + if isinstance(node, vim.StoragePod): + row.append(node.name) + else: + row.append('') + + else: # Folder is a Datastore Cluster + inventory.update(_collect(dc, folder)) + return inventory - def _vmware_perf_metrics(self, content): + labels = {} + dcs = yield self.datacenter_inventory + for dc in dcs: + result = yield threads.deferToThread(lambda: _collect(dc, dc.datastoreFolder)) + labels.update(result) + + return labels + + @run_once_property + @defer.inlineCallbacks + def host_labels(self): + + def _collect(dc, node): + host_inventory = {} + for folder in node.childEntity: + if hasattr(folder, 'host'): + for host in folder.host: # Iterate through Hosts in the Cluster + host_name = host.summary.config.name.rstrip('.') + host_inventory[host._moId] = [ + host_name, + dc.name, + folder.name if isinstance(folder, vim.ClusterComputeResource) else '' + ] + + if isinstance(folder, vim.Folder): + host_inventory.extend(_collect(dc, folder)) + + return host_inventory + + labels = {} + dcs = yield self.datacenter_inventory + for dc in dcs: + result = yield threads.deferToThread(lambda: _collect(dc, dc.hostFolder)) + labels.update(result) + + return labels + + @run_once_property + @defer.inlineCallbacks + def vm_labels(self): + virtual_machines, host_labels = yield parallelize(self.vm_inventory, self.host_labels) + + labels = {} + for moid, row in virtual_machines.items(): + host_moid = row['runtime.host']._moId + labels[moid] = [row['name']] + host_labels[host_moid] + + return labels + + @run_once_property + @defer.inlineCallbacks + def counter_ids(self): """ create a mapping from performance stats to their counterIDs counter_info: [performance stat => counterId] performance stat example: cpu.usagemhz.LATEST """ + content = yield self.content counter_info = {} for counter in content.perfManager.perfCounter: prefix = counter.groupInfo.key @@ -266,6 +425,18 @@ def _vmware_perf_metrics(self, content): counter_info[counter_full] = counter.key return counter_info + @defer.inlineCallbacks + def _vmware_disconnect(self): + """ + Disconnect from Vcenter + """ + connection = yield self.connection + yield threads.deferToThread( + connect.Disconnect, + connection, + ) + del self.connection + def _vmware_full_snapshots_list(self, snapshots): """ Get snapshots from a VM list, recursively @@ -279,29 +450,18 @@ def _vmware_full_snapshots_list(self, snapshots): snapshot.childSnapshotList) return snapshot_data - def _vmware_get_datastores(self, content, ds_metrics, inventory): + @defer.inlineCallbacks + def _vmware_get_datastores(self, ds_metrics): """ Get Datastore information """ log("Starting datastore metrics collection") + results, datastore_labels = yield parallelize(self.datastore_inventory, self.datastore_labels) - properties = [ - 'name', - 'summary.capacity', - 'summary.freeSpace', - 'summary.uncommitted', - 'summary.maintenanceMode', - 'summary.type', - 'summary.accessible', - 'host', - 'vm', - ] - - results = batch_fetch_properties(content, vim.Datastore, properties) for datastore_id, datastore in results.items(): name = datastore['name'] - labels = [name, inventory[name]['dc'], inventory[name]['ds_cluster']] + labels = datastore_labels[name] ds_capacity = float(datastore.get('summary.capacity', 0)) ds_freespace = float(datastore.get('summary.freeSpace', 0)) @@ -335,10 +495,10 @@ def _vmware_get_datastores(self, content, ds_metrics, inventory): log("Finished datastore metrics collection") @defer.inlineCallbacks - def _vmware_get_vm_perf_manager_metrics(self, content, virtual_machines, vm_metrics, inventory): + def _vmware_get_vm_perf_manager_metrics(self, vm_metrics): log('START: _vmware_get_vm_perf_manager_metrics') - counter_info = yield threads.deferToThread(self._vmware_perf_metrics, content) + virtual_machines, counter_info = yield parallelize(self.vm_inventory, self.counter_ids) # List of performance counter we want perf_list = [ @@ -383,64 +543,32 @@ def _vmware_get_vm_perf_manager_metrics(self, content, virtual_machines, vm_metr intervalId=20 )) - log('START: _vmware_get_vm_perf_manager_metrics: QUERY') - result = yield threads.deferToThread(content.perfManager.QueryStats, querySpec=specs) - log('FIN: _vmware_get_vm_perf_manager_metrics: QUERY') + content = yield self.content + + results, labels = yield parallelize( + threads.deferToThread(content.perfManager.QueryStats, querySpec=specs), + self.vm_labels, + ) - for ent in result: + for ent in results: for metric in ent.value: vm_metrics[metric_names[metric.id.counterId]].add_metric( - self._labels[ent.entity._moId], + labels[ent.entity._moId], float(sum(metric.value)), ) log('FIN: _vmware_get_vm_perf_manager_metrics') @defer.inlineCallbacks - def _vmware_get_vms(self, content, metrics, inventory): + def _vmware_get_vms(self, metrics): """ Get VM information """ log("Starting vm metrics collection") - properties = [ - 'name', - 'runtime.host', - ] - - if self.collect_only['vms'] is True: - properties.extend([ - 'runtime.powerState', - 'runtime.bootTime', - 'summary.config.numCpu', - ]) - - if self.collect_only['vmguests'] is True: - properties.append('guest.disk') - - if self.collect_only['snapshots'] is True: - properties.append('snapshot') - - virtual_machines = yield threads.deferToThread( - batch_fetch_properties, - content, - vim.VirtualMachine, - properties, - ) - - if self.collect_only['vms'] is True: - vm_perf_deferred = self._vmware_get_vm_perf_manager_metrics( - content, virtual_machines, metrics, inventory - ) + virtual_machines, vm_labels = yield parallelize(self.vm_inventory, self.vm_labels) for moid, row in virtual_machines.items(): - host_moid = row['runtime.host']._moId - - labels = self._labels[moid] = [ - row['name'], - inventory[host_moid]['name'], - inventory[host_moid]['dc'], - inventory[host_moid]['cluster'], - ] + labels = vm_labels[moid] if 'runtime.powerState' in row: power_state = 1 if row['runtime.powerState'] == 'poweredOn' else 0 @@ -478,32 +606,19 @@ def _vmware_get_vms(self, content, metrics, inventory): snapshot['timestamp_seconds'], ) - yield vm_perf_deferred log("Finished vm metrics collection") - def _vmware_get_hosts(self, content, host_metrics, inventory): + @defer.inlineCallbacks + def _vmware_get_hosts(self, host_metrics): """ Get Host (ESXi) information """ log("Starting host metrics collection") - properties = [ - 'name', - 'summary.hardware.numCpuCores', - 'summary.hardware.cpuMhz', - 'summary.hardware.memorySize', - 'runtime.powerState', - 'runtime.bootTime', - 'runtime.connectionState', - 'runtime.inMaintenanceMode', - 'summary.quickStats.overallCpuUsage', - 'summary.quickStats.overallMemoryUsage', - ] + results, host_labels = yield parallelize(self.host_system_inventory, self.host_labels) - results = batch_fetch_properties(content, vim.HostSystem, properties) for host_id, host in results.items(): - name = host['name'] - labels = [name, inventory[host['id']]['dc'], inventory[host['id']]['cluster']] + labels = host_labels[host_id] # Power state power_state = 1 if host['runtime.powerState'] == 'poweredOn' else 0 @@ -561,49 +676,7 @@ def _vmware_get_hosts(self, content, host_metrics, inventory): ) log("Finished host metrics collection") - - def _vmware_get_inventory(self, content): - """ - Get host and datastore inventory (datacenter, cluster) information - """ - host_inventory = {} - ds_inventory = {} - - children = content.rootFolder.childEntity - for child in children: # Iterate though DataCenters - dc = child - hostFolders = dc.hostFolder.childEntity - for folder in hostFolders: # Iterate through host folders - if isinstance(folder, vim.ClusterComputeResource): # Folder is a Cluster - hosts = folder.host - for host in hosts: # Iterate through Hosts in the Cluster - host_name = host.summary.config.name.rstrip('.') - row = host_inventory[host._moId] = {} - row['name'] = host_name - row['dc'] = dc.name - row['cluster'] = folder.name - else: # Unclustered host - for host in folder.host: - row = host_inventory[host._moId] = {} - host_name = host.name.rstrip('.') - row['name'] = host_name - row['dc'] = dc.name - row['cluster'] = '' - - dsFolders = dc.datastoreFolder.childEntity - for folder in dsFolders: # Iterate through datastore folders - if isinstance(folder, vim.Datastore): # Unclustered datastore - ds_inventory[folder.name] = {} - ds_inventory[folder.name]['dc'] = dc.name - ds_inventory[folder.name]['ds_cluster'] = '' - else: # Folder is a Datastore Cluster - datastores = folder.childEntity - for datastore in datastores: - ds_inventory[datastore.name] = {} - ds_inventory[datastore.name]['dc'] = dc.name - ds_inventory[datastore.name]['ds_cluster'] = folder.name - - return host_inventory, ds_inventory + return results class ListCollector(object): @@ -745,11 +818,11 @@ def render_GET(self, request): return 'Server is UP'.encode() -def log(data): +def log(data, *args): """ Log any message in a uniform format """ - print("[{0}] {1}".format(datetime.utcnow().replace(tzinfo=pytz.utc), data)) + print("[{0}] {1}".format(datetime.datetime.utcnow().replace(tzinfo=pytz.utc), data % args)) def main(argv=None):