Modify OpenStack plugins to harmonize metrics
This change updates the OpenStack plugins to avoid special cases in the
Heka collectd decoder.
Change-Id: I0d04262924a455e144d624ecd605874a3a17a96b
Depends-On: I7a56e3b2390b83bb6a2c72f1c33ab47498d4202c
diff --git a/collectd/files/plugin/check_openstack_api.py b/collectd/files/plugin/check_openstack_api.py
index f775208..82c4ae8 100644
--- a/collectd/files/plugin/check_openstack_api.py
+++ b/collectd/files/plugin/check_openstack_api.py
@@ -20,7 +20,7 @@
from urlparse import urlparse
-PLUGIN_NAME = 'check_openstack_api'
+PLUGIN_NAME = 'openstack_check_api'
INTERVAL = openstack.INTERVAL
@@ -104,13 +104,16 @@
if item['status'] != self.UNKNOWN:
# skip if status is UNKNOWN
yield {
- 'plugin_instance': item['service'],
'values': item['status'],
- 'meta': {'region': item['region']},
+ 'meta': {
+ 'region': item['region'],
+ 'service': item['service'],
+ 'discard_hostname': True,
+ },
}
-plugin = APICheckPlugin(collectd, PLUGIN_NAME)
+plugin = APICheckPlugin(collectd, PLUGIN_NAME, disable_check_metric=True)
def config_callback(conf):
diff --git a/collectd/files/plugin/hypervisor_stats.py b/collectd/files/plugin/hypervisor_stats.py
index 7c0d26d..5fc3bdb 100644
--- a/collectd/files/plugin/hypervisor_stats.py
+++ b/collectd/files/plugin/hypervisor_stats.py
@@ -21,7 +21,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'hypervisor_stats'
+PLUGIN_NAME = 'openstack_nova'
INTERVAL = openstack.INTERVAL
@@ -30,12 +30,18 @@
VALUE_MAP = {
'current_workload': 'running_tasks',
'running_vms': 'running_instances',
- 'local_gb_used': 'used_disk_GB',
- 'free_disk_gb': 'free_disk_GB',
- 'memory_mb_used': 'used_ram_MB',
- 'free_ram_mb': 'free_ram_MB',
+ 'local_gb_used': 'used_disk',
+ 'free_disk_gb': 'free_disk',
+ 'memory_mb_used': 'used_ram',
+ 'free_ram_mb': 'free_ram',
'vcpus_used': 'used_vcpus',
}
+ UNIT_MAP = {
+ 'local_gb_used': 'GB',
+ 'free_disk_gb': 'GB',
+ 'memory_mb_used': 'MB',
+ 'free_ram_mb': 'MB',
+ }
def __init__(self, *args, **kwargs):
super(HypervisorStatsPlugin, self).__init__(*args, **kwargs)
@@ -50,6 +56,12 @@
if 'cpu_ratio' not in self.extra_config:
self.logger.warning('CpuAllocationRatio parameter not set')
+ @classmethod
+ def initialize_metrics(self):
+ metrics = {v: 0 for v in self.VALUE_MAP.values()}
+ metrics['free_vcpus'] = 0
+ return metrics
+
def itermetrics(self):
nova_aggregates = {}
r = self.get('nova', 'os-aggregates')
@@ -61,29 +73,28 @@
nova_aggregates[agg['name']] = {
'id': agg['id'],
'hosts': [h.split('.')[0] for h in agg['hosts']],
- 'metrics': {'free_vcpus': 0},
+ 'metrics': self.initialize_metrics()
}
- nova_aggregates[agg['name']]['metrics'].update(
- {v: 0 for v in self.VALUE_MAP.values()}
- )
r = self.get('nova', 'os-hypervisors/detail')
if not r:
self.logger.warning("Could not get hypervisor statistics")
return
- total_stats = {v: 0 for v in self.VALUE_MAP.values()}
- total_stats['free_vcpus'] = 0
+ total_stats = self.initialize_metrics()
hypervisor_stats = r.json().get('hypervisors', [])
for stats in hypervisor_stats:
# remove domain name and keep only the hostname portion
host = stats['hypervisor_hostname'].split('.')[0]
for k, v in self.VALUE_MAP.iteritems():
m_val = stats.get(k, 0)
+ meta = {'hostname': host}
+ if k in self.UNIT_MAP:
+ meta['unit'] = self.UNIT_MAP[k]
yield {
- 'type_instance': v,
+ 'plugin_instance': v,
'values': m_val,
- 'meta': {'host': host},
+ 'meta': meta
}
total_stats[v] += m_val
for agg in nova_aggregates.keys():
@@ -96,9 +107,9 @@
free = (int(self.extra_config['cpu_ratio'] *
m_vcpus)) - m_vcpus_used
yield {
- 'type_instance': 'free_vcpus',
+ 'plugin_instance': 'free_vcpus',
'values': free,
- 'meta': {'host': host},
+ 'meta': {'hostname': host},
}
total_stats['free_vcpus'] += free
for agg in nova_aggregates.keys():
@@ -109,25 +120,22 @@
m_vcpus_used)
nova_aggregates[agg]['metrics']['free_vcpus'] += free
- # Dispatch the aggregate metrics
+ # Dispatch metrics for every aggregate
for agg in nova_aggregates.keys():
agg_id = nova_aggregates[agg]['id']
agg_total_free_ram = (
- nova_aggregates[agg]['metrics']['free_ram_MB'] +
- nova_aggregates[agg]['metrics']['used_ram_MB']
+ nova_aggregates[agg]['metrics']['free_ram'] +
+ nova_aggregates[agg]['metrics']['used_ram']
)
- # Only emit metric when value is > 0
- # If this is not the case, (for instance when no host
- # in aggregate), this requires the corresponding alarms to
- # have a 'skip' no_data_policy, so as not to be triggered
+ # Only emit metric when the value is > 0 to avoid division by zero
if agg_total_free_ram > 0:
nova_aggregates[agg]['metrics']['free_ram_percent'] = round(
- (100.0 * nova_aggregates[agg]['metrics']['free_ram_MB']) /
+ (100.0 * nova_aggregates[agg]['metrics']['free_ram']) /
agg_total_free_ram,
2)
for k, v in nova_aggregates[agg]['metrics'].iteritems():
yield {
- 'type_instance': 'aggregate_{}'.format(k),
+ 'plugin_instance': 'aggregate_{}'.format(k),
'values': v,
'meta': {
'aggregate': agg,
diff --git a/collectd/files/plugin/openstack_cinder.py b/collectd/files/plugin/openstack_cinder.py
index 96a92c6..fe667a6 100644
--- a/collectd/files/plugin/openstack_cinder.py
+++ b/collectd/files/plugin/openstack_cinder.py
@@ -21,7 +21,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'cinder'
+PLUGIN_NAME = 'openstack_cinder'
INTERVAL = openstack.INTERVAL
@@ -55,18 +55,18 @@
for s, nb in status.iteritems():
yield {
'plugin_instance': 'volumes',
- 'type_instance': s,
- 'values': nb
+ 'values': nb,
+ 'meta': {'state': s, 'discard_hostname': True}
}
sizes = self.count_objects_group_by(volumes_details,
group_by_func=groupby,
count_func=count_size_bytes)
- for n, size in sizes.iteritems():
+ for s, size in sizes.iteritems():
yield {
'plugin_instance': 'volumes_size',
- 'type_instance': n,
- 'values': size
+ 'values': size,
+ 'meta': {'state': s, 'discard_hostname': True}
}
snaps_details = self.get_objects('cinderv2', 'snapshots',
@@ -76,8 +76,8 @@
for s, nb in status_snaps.iteritems():
yield {
'plugin_instance': 'snapshots',
- 'type_instance': s,
- 'values': nb
+ 'values': nb,
+ 'meta': {'state': s, 'discard_hostname': True}
}
sizes = self.count_objects_group_by(snaps_details,
@@ -86,8 +86,8 @@
for n, size in sizes.iteritems():
yield {
'plugin_instance': 'snapshots_size',
- 'type_instance': n,
- 'values': size
+ 'values': size,
+ 'meta': {'state': s, 'discard_hostname': True}
}
diff --git a/collectd/files/plugin/openstack_cinder_services.py b/collectd/files/plugin/openstack_cinder_services.py
index b37dad2..6488da1 100644
--- a/collectd/files/plugin/openstack_cinder_services.py
+++ b/collectd/files/plugin/openstack_cinder_services.py
@@ -24,7 +24,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'cinder'
+PLUGIN_NAME = 'openstack_cinder'
INTERVAL = openstack.INTERVAL
@@ -55,9 +55,9 @@
aggregated_workers[service][state] += 1
yield {
- 'plugin_instance': 'cinder_service',
+ 'plugin_instance': 'service',
'values': self.states[state],
- 'meta': {'host': host, 'service': service, 'state': state},
+ 'meta': {'hostname': host, 'service': service, 'state': state},
}
for service in aggregated_workers:
@@ -66,14 +66,16 @@
for state in self.states:
prct = (100.0 * aggregated_workers[service][state]) / totalw
yield {
- 'plugin_instance': 'cinder_services_percent',
+ 'plugin_instance': 'services_percent',
'values': prct,
- 'meta': {'state': state, 'service': service}
+ 'meta': {'state': state, 'service': service,
+ 'discard_hostname': True}
}
yield {
- 'plugin_instance': 'cinder_services',
+ 'plugin_instance': 'services',
'values': aggregated_workers[service][state],
- 'meta': {'state': state, 'service': service},
+ 'meta': {'state': state, 'service': service,
+ 'discard_hostname': True},
}
diff --git a/collectd/files/plugin/openstack_glance.py b/collectd/files/plugin/openstack_glance.py
index da9c469..90bc9f8 100644
--- a/collectd/files/plugin/openstack_glance.py
+++ b/collectd/files/plugin/openstack_glance.py
@@ -21,7 +21,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'glance'
+PLUGIN_NAME = 'openstack_glance'
INTERVAL = openstack.INTERVAL
@@ -57,11 +57,12 @@
status = self.count_objects_group_by(images_details,
group_by_func=groupby)
for s, nb in status.iteritems():
- (name, visibility, status) = s.split('.')
+ (name, visibility, state) = s.split('.')
yield {
- 'type_instance': name,
+ 'plugin_instance': name,
'values': nb,
- 'meta': {'visibility': visibility, 'status': status}
+ 'meta': {'visibility': visibility, 'state': state,
+ 'discard_hostname': True}
}
# sizes
@@ -79,11 +80,12 @@
group_by_func=groupby_size,
count_func=count_size_bytes)
for s, nb in sizes.iteritems():
- (name, visibility, status) = s.split('.')
+ (name, visibility, state) = s.split('.')
yield {
- 'type_instance': name,
+ 'plugin_instance': name,
'values': nb,
- 'meta': {'visibility': visibility, 'status': status},
+ 'meta': {'visibility': visibility, 'state': state,
+ 'discard_hostname': True},
}
plugin = GlanceStatsPlugin(collectd, PLUGIN_NAME, disable_check_metric=True)
diff --git a/collectd/files/plugin/openstack_keystone.py b/collectd/files/plugin/openstack_keystone.py
index 4ad4611..0a0d975 100644
--- a/collectd/files/plugin/openstack_keystone.py
+++ b/collectd/files/plugin/openstack_keystone.py
@@ -21,7 +21,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'keystone'
+PLUGIN_NAME = 'openstack_keystone'
INTERVAL = openstack.INTERVAL
@@ -52,9 +52,9 @@
group_by_func=groupby)
for s, nb in status.iteritems():
yield {
- 'type_instance': 'tenants',
+ 'plugin_instance': 'tenants',
'values': nb,
- 'meta': {'state': s},
+ 'meta': {'state': s, 'discard_hostname': True},
}
# users
@@ -67,9 +67,9 @@
group_by_func=groupby)
for s, nb in status.iteritems():
yield {
- 'type_instance': 'users',
+ 'plugin_instance': 'users',
'values': nb,
- 'meta': {'state': s},
+ 'meta': {'state': s, 'discard_hostname': True},
}
# roles
@@ -79,8 +79,9 @@
return
roles = r.json().get('roles', [])
yield {
- 'type_instance': 'roles',
+ 'plugin_instance': 'roles',
'values': len(roles),
+ 'meta': {'discard_hostname': True},
}
plugin = KeystoneStatsPlugin(collectd, PLUGIN_NAME,
diff --git a/collectd/files/plugin/openstack_neutron.py b/collectd/files/plugin/openstack_neutron.py
index f1b72af..1d147c7 100644
--- a/collectd/files/plugin/openstack_neutron.py
+++ b/collectd/files/plugin/openstack_neutron.py
@@ -21,7 +21,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'neutron'
+PLUGIN_NAME = 'openstack_neutron'
INTERVAL = openstack.INTERVAL
@@ -43,11 +43,8 @@
def itermetrics(self):
- def groupby_network(x):
- return "networks.%s" % x.get('status', 'unknown').lower()
-
- def groupby_router(x):
- return "routers.%s" % x.get('status', 'unknown').lower()
+ def groupby_status(x):
+ return x.get('status', 'unknown').lower()
def groupby_port(x):
owner = x.get('device_owner', 'none')
@@ -59,28 +56,34 @@
else:
owner = 'none'
status = x.get('status', 'unknown').lower()
- return "ports.%s.%s" % (owner, status)
+ return "%s.%s" % (owner, status)
def groupby_floating(x):
- if x.get('port_id', None):
- status = 'associated'
- else:
- status = 'free'
- return "floatingips.%s" % status
+ return 'associated' if x.get('port_id', None) else None
# Networks
networks = self.get_objects('neutron', 'networks', api_version='v2.0',
params={'fields': ['id', 'status']})
status = self.count_objects_group_by(networks,
- group_by_func=groupby_network)
+ group_by_func=groupby_status)
for s, nb in status.iteritems():
- yield {'type_instance': s, 'values': nb}
- yield {'type_instance': 'networks', 'values': len(networks)}
+ yield {
+ 'plugin_instance': 'networks',
+ 'values': nb,
+ 'meta': {'state': s, 'discard_hostname': True}
+ }
+ yield {
+ 'plugin_instance': 'networks',
+ 'type_instance': 'total',
+ 'values': len(status),
+ 'meta': {'discard_hostname': True},
+ }
# Subnets
subnets = self.get_objects('neutron', 'subnets', api_version='v2.0',
- params={'fields': ['id', 'status']})
- yield {'type_instance': 'subnets', 'values': len(subnets)}
+ params={'fields': ['id']})
+ yield {'plugin_instance': 'subnets', 'values': len(subnets),
+ 'meta': {'discard_hostname': True}}
# Ports
ports = self.get_objects('neutron', 'ports', api_version='v2.0',
@@ -89,17 +92,37 @@
status = self.count_objects_group_by(ports,
group_by_func=groupby_port)
for s, nb in status.iteritems():
- yield {'type_instance': s, 'values': nb}
- yield {'type_instance': 'ports', 'values': len(ports)}
+ (owner, state) = s.split('.')
+ yield {
+ 'plugin_instance': 'ports',
+ 'values': nb,
+ 'meta': {'state': state, 'owner': owner,
+ 'discard_hostname': True}
+ }
+ yield {
+ 'plugin_instance': 'ports',
+ 'type_instance': 'total',
+ 'values': len(ports),
+ 'meta': {'discard_hostname': True},
+ }
# Routers
routers = self.get_objects('neutron', 'routers', api_version='v2.0',
params={'fields': ['id', 'status']})
status = self.count_objects_group_by(routers,
- group_by_func=groupby_router)
+ group_by_func=groupby_status)
for s, nb in status.iteritems():
- yield {'type_instance': s, 'values': nb}
- yield {'type_instance': 'routers', 'values': len(routers)}
+ yield {
+ 'plugin_instance': 'routers',
+ 'values': nb,
+ 'meta': {'state': s, 'discard_hostname': True}
+ }
+ yield {
+ 'plugin_instance': 'routers',
+ 'type_instance': 'total',
+ 'values': len(routers),
+ 'meta': {'discard_hostname': True},
+ }
# Floating IP addresses
floatingips = self.get_objects('neutron', 'floatingips',
@@ -109,8 +132,17 @@
status = self.count_objects_group_by(floatingips,
group_by_func=groupby_floating)
for s, nb in status.iteritems():
- yield {'type_instance': s, 'values': nb}
- yield {'type_instance': 'floatingips', 'values': len(routers)}
+ yield {
+ 'plugin_instance': 'floatingips',
+ 'values': nb,
+ 'meta': {'state': s, 'discard_hostname': True}
+ }
+ yield {
+ 'plugin_instance': 'floatingips',
+ 'type_instance': 'total',
+ 'values': len(floatingips),
+ 'meta': {'discard_hostname': True},
+ }
plugin = NeutronStatsPlugin(collectd, PLUGIN_NAME, disable_check_metric=True)
diff --git a/collectd/files/plugin/openstack_neutron_agents.py b/collectd/files/plugin/openstack_neutron_agents.py
index 0f060a3..e1b3848 100644
--- a/collectd/files/plugin/openstack_neutron_agents.py
+++ b/collectd/files/plugin/openstack_neutron_agents.py
@@ -24,7 +24,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'neutron'
+PLUGIN_NAME = 'openstack_neutron'
INTERVAL = openstack.INTERVAL
@@ -58,9 +58,9 @@
aggregated_agents[service][state] += 1
yield {
- 'type_instance': 'neutron_agent',
+ 'plugin_instance': 'agent',
'values': self.states[state],
- 'meta': {'host': host, 'service': service, 'state': state}
+ 'meta': {'hostname': host, 'service': service, 'state': state}
}
for service in aggregated_agents:
@@ -69,14 +69,16 @@
for state in self.states:
prct = (100.0 * aggregated_agents[service][state]) / totala
yield {
- 'type_instance': 'neutron_agents_percent',
+ 'plugin_instance': 'agents_percent',
'values': prct,
- 'meta': {'service': service, 'state': state},
+ 'meta': {'service': service, 'state': state,
+ 'discard_hostname': True},
}
yield {
- 'type_instance': 'neutron_agents',
+ 'plugin_instance': 'agents',
'values': aggregated_agents[service][state],
- 'meta': {'service': service, 'state': state},
+ 'meta': {'service': service, 'state': state,
+ 'discard_hostname': True},
}
diff --git a/collectd/files/plugin/openstack_nova.py b/collectd/files/plugin/openstack_nova.py
index ea2bd77..ecfee1e 100644
--- a/collectd/files/plugin/openstack_nova.py
+++ b/collectd/files/plugin/openstack_nova.py
@@ -23,7 +23,7 @@
from itertools import groupby
-PLUGIN_NAME = 'nova'
+PLUGIN_NAME = 'openstack_nova'
INTERVAL = openstack.INTERVAL
@@ -63,7 +63,7 @@
yield {
'plugin_instance': 'instances',
'values': len(list(g)),
- 'type_instance': status,
+ 'meta': {'state': status, 'discard_hostname': True},
}
diff --git a/collectd/files/plugin/openstack_nova_services.py b/collectd/files/plugin/openstack_nova_services.py
index 51b0df4..e6860e6 100644
--- a/collectd/files/plugin/openstack_nova_services.py
+++ b/collectd/files/plugin/openstack_nova_services.py
@@ -25,7 +25,7 @@
import collectd_openstack as openstack
-PLUGIN_NAME = 'nova'
+PLUGIN_NAME = 'openstack_nova'
INTERVAL = openstack.INTERVAL
@@ -55,9 +55,9 @@
aggregated_workers[service][state] += 1
yield {
- 'plugin_instance': 'nova_service',
+ 'plugin_instance': 'service',
'values': self.states[state],
- 'meta': {'host': host, 'service': service, 'state': state}
+ 'meta': {'hostname': host, 'service': service, 'state': state}
}
for service in set(aggregated_workers.keys()).union(
@@ -71,14 +71,16 @@
prct = (100.0 * aggregated_workers[service][state]) / total
yield {
- 'plugin_instance': 'nova_services_percent',
+ 'plugin_instance': 'services_percent',
'values': prct,
- 'meta': {'state': state, 'service': service},
+ 'meta': {'state': state, 'service': service,
+ 'discard_hostname': True},
}
yield {
- 'plugin_instance': 'nova_services',
+ 'plugin_instance': 'services',
'values': aggregated_workers[service][state],
- 'meta': {'state': state, 'service': service},
+ 'meta': {'state': state, 'service': service,
+ 'discard_hostname': True},
}