Support pagination for OpenStack services
This concerns:
- Nova server list
- Neutron
- Glance
- Cinder
Change-Id: I6981d05776aa2cb4b20a163493a0b337f45500d2
Co-Authored-By: Swann Croiset <scroiset@mirantis.com>
diff --git a/collectd/files/plugin/collectd_openstack.py b/collectd/files/plugin/collectd_openstack.py
index 7108d04..a4ec4de 100644
--- a/collectd/files/plugin/collectd_openstack.py
+++ b/collectd/files/plugin/collectd_openstack.py
@@ -115,7 +115,8 @@
self.logger.debug("Got token '%s'" % self.token)
return self.token
- def make_request(self, verb, url, data=None, token_required=True):
+ def make_request(self, verb, url, data=None, token_required=True,
+ params=None):
kwargs = {
'url': url,
'timeout': self.timeout,
@@ -131,6 +132,9 @@
if data is not None:
kwargs['data'] = data
+ if params is not None:
+ kwargs['params'] = params
+
func = getattr(self.session, verb.lower())
try:
@@ -159,6 +163,7 @@
self.max_retries = 2
self.os_client = None
self.extra_config = {}
+ self.pagination_limit = None
def _build_url(self, service, resource):
s = (self.get_service(service) or {})
@@ -241,12 +246,12 @@
yield data
- def get(self, service, resource):
+ def get(self, service, resource, params=None):
url = self._build_url(service, resource)
if not url:
return
self.logger.info("GET '%s'" % url)
- return self.os_client.make_request('get', url)
+ return self.os_client.make_request('get', url, params=params)
@property
def service_catalog(self):
@@ -271,46 +276,72 @@
tenant_name = node.values[0]
elif node.key == 'KeystoneUrl':
keystone_url = node.values[0]
+ elif node.key == 'PaginationLimit':
+ self.pagination_limit = int(node.values[0])
self.os_client = OSClient(username, password, tenant_name,
keystone_url, self.timeout, self.logger,
self.max_retries)
def get_objects(self, project, object_name, api_version='',
- params='all_tenants=1'):
+ params=None, detail=False):
""" Return a list of OpenStack objects
- See get_objects_details()
- """
- return self._get_objects(project, object_name, api_version, params,
- False)
-
- def get_objects_details(self, project, object_name, api_version='',
- params='all_tenants=1'):
- """ Return a list of details about OpenStack objects
-
The API version is not always included in the URL endpoint
registered in Keystone (eg Glance). In this case, use the
api_version parameter to specify which version should be used.
- """
- return self._get_objects(project, object_name, api_version, params,
- True)
- def _get_objects(self, project, object_name, api_version, params, detail):
+ """
+ if params is None:
+ params = {}
+
if api_version:
resource = '%s/%s' % (api_version, object_name)
else:
resource = '%s' % (object_name)
+
if detail:
- resource = '%s/detail' % (resource)
- if params:
- resource = '%s?%s' % (resource, params)
- # TODO(scroiset): use pagination to handle large collection
- r = self.get(project, resource)
- if not r or object_name not in r.json():
- self.logger.warning('Could not find %s %s' % (project,
- object_name))
- return []
- return r.json().get(object_name)
+ resource = '{}/detail'.format(resource)
+
+ url = self._build_url(project, resource)
+ if not url:
+ return
+
+ opts = {}
+ if self.pagination_limit:
+ opts['limit'] = self.pagination_limit
+
+ opts.update(params)
+ objs = []
+
+ while True:
+ r = self.os_client.make_request('get', url, params=opts)
+ if not r or object_name not in r.json():
+ self.logger.warning('Could not find %s %s' % (project,
+ object_name))
+ return objs
+
+ resp = r.json()
+ bulk_objs = resp.get(object_name)
+
+ if not bulk_objs:
+ break
+
+ objs.extend(bulk_objs)
+
+ links = resp.get('{}_links'.format(object_name))
+ if links is None or self.pagination_limit is None:
+ # Either the pagination is not supported or there is no more
+ # data
+ break
+
+ # if there is no 'next' link in the response, all data has been
+ # read.
+ if len([i for i in links if i.get('rel') == 'next']) == 0:
+ break
+
+ opts['marker'] = bulk_objs[-1]['id']
+
+ return objs
def count_objects_group_by(self,
list_object,
diff --git a/collectd/files/plugin/openstack_cinder.py b/collectd/files/plugin/openstack_cinder.py
index 373c081..8211f71 100644
--- a/collectd/files/plugin/openstack_cinder.py
+++ b/collectd/files/plugin/openstack_cinder.py
@@ -33,10 +33,13 @@
super(CinderStatsPlugin, self).__init__(*args, **kwargs)
self.plugin = PLUGIN_NAME
self.interval = INTERVAL
+ self.pagination_limit = 500
def itermetrics(self):
- volumes_details = self.get_objects_details('cinder', 'volumes')
+ volumes_details = self.get_objects('cinder', 'volumes',
+ params={'all_tenants': 1},
+ detail=True)
def groupby(d):
return d.get('status', 'unknown').lower()
@@ -63,7 +66,8 @@
'values': size
}
- snaps_details = self.get_objects_details('cinder', 'snapshots')
+ snaps_details = self.get_objects('cinder', 'snapshots',
+ params={'all_tenants': 1})
status_snaps = self.count_objects_group_by(snaps_details,
group_by_func=groupby)
for s, nb in status_snaps.iteritems():
diff --git a/collectd/files/plugin/openstack_glance.py b/collectd/files/plugin/openstack_glance.py
index a6b451f..bcb9712 100644
--- a/collectd/files/plugin/openstack_glance.py
+++ b/collectd/files/plugin/openstack_glance.py
@@ -33,6 +33,7 @@
super(GlanceStatsPlugin, self).__init__(*args, **kwargs)
self.plugin = PLUGIN_NAME
self.interval = INTERVAL
+ self.pagination_limit = 25
def itermetrics(self):
@@ -48,7 +49,9 @@
images_details = self.get_objects_details('glance', 'images',
api_version='v1',
- params='is_public=None')
+ params={},
+ detail=True)
+
status = self.count_objects_group_by(images_details,
group_by_func=groupby)
for s, nb in status.iteritems():
diff --git a/collectd/files/plugin/openstack_neutron.py b/collectd/files/plugin/openstack_neutron.py
index 798f091..e677ec1 100644
--- a/collectd/files/plugin/openstack_neutron.py
+++ b/collectd/files/plugin/openstack_neutron.py
@@ -36,6 +36,7 @@
super(NeutronStatsPlugin, self).__init__(*args, **kwargs)
self.plugin = PLUGIN_NAME
self.interval = INTERVAL
+ self.pagination_limit = 100
def itermetrics(self):
diff --git a/collectd/files/plugin/openstack_nova.py b/collectd/files/plugin/openstack_nova.py
index 455e5d1..8cdd57d 100644
--- a/collectd/files/plugin/openstack_nova.py
+++ b/collectd/files/plugin/openstack_nova.py
@@ -31,9 +31,12 @@
super(NovaInstanceStatsPlugin, self).__init__(*args, **kwargs)
self.plugin = PLUGIN_NAME
self.interval = INTERVAL
+ self.pagination_limit = 500
def itermetrics(self):
- servers_details = self.get_objects_details('nova', 'servers')
+ servers_details = self.get_objects('nova', 'servers',
+ params={'all_tenants': 1},
+ detail=True)
def groupby(d):
return d.get('status', 'unknown').lower()