From 7f65594557a5eff149406c8f861d32ea61c67588 Mon Sep 17 00:00:00 2001 From: Ben Ritcey Date: Fri, 16 Feb 2018 13:16:40 -0500 Subject: [PATCH 1/4] limit what data is collected via the config --- README.md | 14 ++++++ config.yml.sample | 9 ++++ vmware_exporter/vmware_exporter.py | 75 ++++++++++++++++++++++-------- 3 files changed, 78 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c8d52d7..e0c9785 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,20 @@ Get VMWare VCenter information: Alternatively, if you don't wish to install the package, run using `$ vmware_exporter/vmware_exporter.py` +### Limiting data collection + +Large installations may have trouble collecting all of the data in a timely fashion, +so you can limit which subsystems are collected by adding a +`collect_only` argument the config section, e.g. under `default`: + + collect_only: + # - vms + - datastores + - hosts + +would skip collecting information on the VMs. If there is no `collect_only` +argument, everything will be collected as normal. + ### Prometheus configuration You can use the following parameters in prometheus configuration file. The `params` section is used to manage multiple login/passwords. diff --git a/config.yml.sample b/config.yml.sample index dc66013..d589c5e 100644 --- a/config.yml.sample +++ b/config.yml.sample @@ -7,3 +7,12 @@ esx: vmware_user: 'root' vmware_password: 'password' ignore_ssl: True + +limited: + vmware_user: 'administrator@vsphere.local' + vmware_password: 'password' + ignore_ssl: True + collect_only: + # - vms + - datastores + - hosts diff --git a/vmware_exporter/vmware_exporter.py b/vmware_exporter/vmware_exporter.py index 2f6de0a..749011b 100755 --- a/vmware_exporter/vmware_exporter.py +++ b/vmware_exporter/vmware_exporter.py @@ -97,7 +97,8 @@ def collect(self, target=None, section='default'): if section not in self.config.keys(): print("{} is not a valid section, using default".format(section)) section='default' - metrics = { + metric_list ={} + metric_list['vms'] = { 'vmware_vm_power_state': GaugeMetricFamily( 'vmware_vm_power_state', 'VMWare VM Power state (On / Off)', @@ -117,7 +118,9 @@ def collect(self, target=None, section='default'): 'vmware_vm_num_cpu': GaugeMetricFamily( 'vmware_vm_num_cpu', 'VMWare Number of processors in the virtual machine', - labels=['vm_name']), + labels=['vm_name']) + } + metric_list['datastores'] = { 'vmware_datastore_capacity_size': GaugeMetricFamily( 'vmware_datastore_capacity_size', 'VMWare Datasore capacity in bytes', @@ -141,7 +144,9 @@ def collect(self, target=None, section='default'): 'vmware_datastore_vms': GaugeMetricFamily( 'vmware_datastore_vms', 'VMWare Virtual Machines number using this datastore', - labels=['ds_name']), + labels=['ds_name']) + } + metric_list['hosts'] = { 'vmware_host_power_state': GaugeMetricFamily( 'vmware_host_power_state', 'VMWare Host Power state (On / Off)', @@ -168,6 +173,12 @@ def collect(self, target=None, section='default'): labels=['host_name']), } + collect_subsystems = self._collect_subsystems(section, metric_list.keys()) + + metrics = {} + for s in collect_subsystems: + metrics.update(metric_list[s]) + print("[{0}] Start collecting vcenter metrics for {1}".format(datetime.utcnow().replace(tzinfo=pytz.utc), target)) self.si = self._vmware_connect(target, section) @@ -177,28 +188,31 @@ def collect(self, target=None, section='default'): content = self.si.RetrieveContent() - # Get performance metrics counter information - counter_info = self._vmware_perf_metrics(content) + if 'vms' in collect_subsystems: + # Get performance metrics counter information + counter_info = self._vmware_perf_metrics(content) - # Fill Snapshots (count and age) - vm_counts, vm_ages = self._vmware_get_snapshots(content) - for v in vm_counts: - metrics['vmware_vm_snapshots'].add_metric([v['vm_name']], - v['snapshot_count']) - for vm_age in vm_ages: - for v in vm_age: - metrics['vmware_vm_snapshot_timestamp_seconds'].add_metric([v['vm_name'], - v['vm_snapshot_name']], - v['vm_snapshot_timestamp_seconds']) + # Fill VM Informations + self._vmware_get_vms(content, metrics, counter_info) - # Fill Datastore - self._vmware_get_datastores(content, metrics) + # Fill Snapshots (count and age) + vm_counts, vm_ages = self._vmware_get_snapshots(content) + for v in vm_counts: + metrics['vmware_vm_snapshots'].add_metric([v['vm_name']], + v['snapshot_count']) + for vm_age in vm_ages: + for v in vm_age: + metrics['vmware_vm_snapshot_timestamp_seconds'].add_metric([v['vm_name'], + v['vm_snapshot_name']], + v['vm_snapshot_timestamp_seconds']) - # Fill VM Informations - self._vmware_get_vms(content, metrics, counter_info) + # Fill Datastore + if 'datastores' in collect_subsystems: + self._vmware_get_datastores(content, metrics) # Fill Hosts Informations - self._vmware_get_hosts(content, metrics) + if 'hosts' in collect_subsystems: + self._vmware_get_hosts(content, metrics) print("[{0}] Stop collecting vcenter metrics for {1}".format(datetime.utcnow().replace(tzinfo=pytz.utc), target)) @@ -209,6 +223,27 @@ def collect(self, target=None, section='default'): + def _collect_subsystems(self, section, valid_subsystems): + """ + Return the list of subsystems to collect - everything by default, a + subset if the config section has collect_only specified + """ + collect_subsystems = [] + + if not self.config[section].get('collect_only'): + collect_subsystems = valid_subsystems + else: + for subsystem in self.config[section].get('collect_only'): + if subsystem in valid_subsystems: + collect_subsystems.append(subsystem) + else: + print("invalid subsystem specified in collect_only: " + str(subsystem)) + + if not collect_subsystems: + print("no valid subystems specified in collect_only, collecting everything") + collect_subsystems = valid_subsystems + + return collect_subsystems def _to_unix_timestamp(self, my_date): return ((my_date - datetime(1970,1,1,tzinfo=pytz.utc)).total_seconds()) From 5b63746179baf17330df3ecaca7ece3fead3ff6b Mon Sep 17 00:00:00 2001 From: Teriand Date: Thu, 1 Mar 2018 12:52:48 +0300 Subject: [PATCH 2/4] Update vmware_exporter.py --- vmware_exporter/vmware_exporter.py | 79 ++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/vmware_exporter/vmware_exporter.py b/vmware_exporter/vmware_exporter.py index aa65e28..d89489e 100755 --- a/vmware_exporter/vmware_exporter.py +++ b/vmware_exporter/vmware_exporter.py @@ -97,7 +97,8 @@ def collect(self, target=None, section='default'): if section not in self.config.keys(): print("{} is not a valid section, using default".format(section)) section='default' - metrics = { + metric_list ={} + metric_list['vms'] = { 'vmware_vm_power_state': GaugeMetricFamily( 'vmware_vm_power_state', 'VMWare VM Power state (On / Off)', @@ -117,7 +118,9 @@ def collect(self, target=None, section='default'): 'vmware_vm_num_cpu': GaugeMetricFamily( 'vmware_vm_num_cpu', 'VMWare Number of processors in the virtual machine', - labels=['vm_name', 'host_name']), + labels=['vm_name', 'host_name']) + } + metric_list['datastores'] = { 'vmware_datastore_capacity_size': GaugeMetricFamily( 'vmware_datastore_capacity_size', 'VMWare Datasore capacity in bytes', @@ -141,7 +144,9 @@ def collect(self, target=None, section='default'): 'vmware_datastore_vms': GaugeMetricFamily( 'vmware_datastore_vms', 'VMWare Virtual Machines number using this datastore', - labels=['ds_name']), + labels=['ds_name']) + } + metric_list['hosts'] = { 'vmware_host_power_state': GaugeMetricFamily( 'vmware_host_power_state', 'VMWare Host Power state (On / Off)', @@ -167,6 +172,13 @@ def collect(self, target=None, section='default'): 'VMWare Host Memory Max availability in Mbytes', labels=['host_name']), } + collect_subsystems = self._collect_subsystems(section, metric_list.keys()) + + + metrics = {} + for s in collect_subsystems: + metrics.update(metric_list[s]) + print("[{0}] Start collecting vcenter metrics for {1}".format(datetime.utcnow().replace(tzinfo=pytz.utc), target)) @@ -177,28 +189,34 @@ def collect(self, target=None, section='default'): content = self.si.RetrieveContent() - # Get performance metrics counter information - counter_info = self._vmware_perf_metrics(content) + if 'vms' in collect_subsystems: + # Get performance metrics counter information + counter_info = self._vmware_perf_metrics(content) - # Fill Snapshots (count and age) - vm_counts, vm_ages = self._vmware_get_snapshots(content) - for v in vm_counts: - metrics['vmware_vm_snapshots'].add_metric([v['vm_name']], - v['snapshot_count']) - for vm_age in vm_ages: - for v in vm_age: - metrics['vmware_vm_snapshot_timestamp_seconds'].add_metric([v['vm_name'], - v['vm_snapshot_name']], - v['vm_snapshot_timestamp_seconds']) - # Fill Datastore - self._vmware_get_datastores(content, metrics) + # Fill VM Informations + self._vmware_get_vms(content, metrics, counter_info) - # Fill VM Informations - self._vmware_get_vms(content, metrics, counter_info) + # Fill Snapshots (count and age) + vm_counts, vm_ages = self._vmware_get_snapshots(content) + for v in vm_counts: + metrics['vmware_vm_snapshots'].add_metric([v['vm_name']], + v['snapshot_count']) + for vm_age in vm_ages: + for v in vm_age: + metrics['vmware_vm_snapshot_timestamp_seconds'].add_metric([v['vm_name'], + v['vm_snapshot_name']], + v['vm_snapshot_timestamp_seconds']) + + + # Fill Datastore + if 'datastores' in collect_subsystems: + self._vmware_get_datastores(content, metrics) # Fill Hosts Informations - self._vmware_get_hosts(content, metrics) + if 'hosts' in collect_subsystems: + self._vmware_get_hosts(content, metrics) + print("[{0}] Stop collecting vcenter metrics for {1}".format(datetime.utcnow().replace(tzinfo=pytz.utc), target)) @@ -207,6 +225,27 @@ def collect(self, target=None, section='default'): for metricname, metric in metrics.items(): yield metric + def _collect_subsystems(self, section, valid_subsystems): + """ + Return the list of subsystems to collect - everything by default, a + subset if the config section has collect_only specified + """ + collect_subsystems = [] + + if not self.config[section].get('collect_only'): + collect_subsystems = valid_subsystems + else: + for subsystem in self.config[section].get('collect_only'): + if subsystem in valid_subsystems: + collect_subsystems.append(subsystem) + else: + print("invalid subsystem specified in collect_only: " + str(subsystem)) + + if not collect_subsystems: + print("no valid subystems specified in collect_only, collecting everything") + collect_subsystems = valid_subsystems + + return collect_subsystems From a4531bda8b349dbe69833a46eb8bae20d10071eb Mon Sep 17 00:00:00 2001 From: Teriand Date: Thu, 1 Mar 2018 13:01:53 +0300 Subject: [PATCH 3/4] Update vmware_exporter.py --- vmware_exporter/vmware_exporter.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/vmware_exporter/vmware_exporter.py b/vmware_exporter/vmware_exporter.py index d89489e..cbb8cd2 100755 --- a/vmware_exporter/vmware_exporter.py +++ b/vmware_exporter/vmware_exporter.py @@ -198,15 +198,15 @@ def collect(self, target=None, section='default'): self._vmware_get_vms(content, metrics, counter_info) # Fill Snapshots (count and age) - vm_counts, vm_ages = self._vmware_get_snapshots(content) - for v in vm_counts: - metrics['vmware_vm_snapshots'].add_metric([v['vm_name']], - v['snapshot_count']) - for vm_age in vm_ages: - for v in vm_age: - metrics['vmware_vm_snapshot_timestamp_seconds'].add_metric([v['vm_name'], - v['vm_snapshot_name']], - v['vm_snapshot_timestamp_seconds']) + vm_counts, vm_ages = self._vmware_get_snapshots(content) + for v in vm_counts: + metrics['vmware_vm_snapshots'].add_metric([v['vm_name']], + v['snapshot_count']) + for vm_age in vm_ages: + for v in vm_age: + metrics['vmware_vm_snapshot_timestamp_seconds'].add_metric([v['vm_name'], + v['vm_snapshot_name']], + v['vm_snapshot_timestamp_seconds']) # Fill Datastore @@ -225,7 +225,7 @@ def collect(self, target=None, section='default'): for metricname, metric in metrics.items(): yield metric - def _collect_subsystems(self, section, valid_subsystems): + def _collect_subsystems(self, section, valid_subsystems): """ Return the list of subsystems to collect - everything by default, a subset if the config section has collect_only specified From d43e38fcf76fd90fdc6209daa8ac383048fd5135 Mon Sep 17 00:00:00 2001 From: Ben Ritcey Date: Fri, 16 Mar 2018 10:56:26 -0400 Subject: [PATCH 4/4] remove duplicate code (bad merge?) --- vmware_exporter/vmware_exporter.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/vmware_exporter/vmware_exporter.py b/vmware_exporter/vmware_exporter.py index 2b09ec8..c0ebcdd 100755 --- a/vmware_exporter/vmware_exporter.py +++ b/vmware_exporter/vmware_exporter.py @@ -175,13 +175,6 @@ def collect(self, target=None, section='default'): collect_subsystems = self._collect_subsystems(section, metric_list.keys()) - metrics = {} - for s in collect_subsystems: - metrics.update(metric_list[s]) - - - collect_subsystems = self._collect_subsystems(section, metric_list.keys()) - metrics = {} for s in collect_subsystems: metrics.update(metric_list[s]) @@ -231,29 +224,6 @@ def collect(self, target=None, section='default'): for metricname, metric in metrics.items(): yield metric - def _collect_subsystems(self, section, valid_subsystems): - """ - Return the list of subsystems to collect - everything by default, a - subset if the config section has collect_only specified - """ - collect_subsystems = [] - - if not self.config[section].get('collect_only'): - collect_subsystems = valid_subsystems - else: - for subsystem in self.config[section].get('collect_only'): - if subsystem in valid_subsystems: - collect_subsystems.append(subsystem) - else: - print("invalid subsystem specified in collect_only: " + str(subsystem)) - - if not collect_subsystems: - print("no valid subystems specified in collect_only, collecting everything") - collect_subsystems = valid_subsystems - - return collect_subsystems - - def _collect_subsystems(self, section, valid_subsystems): """ Return the list of subsystems to collect - everything by default, a