Merge "Service Clients registration interface for plugins"
diff --git a/releasenotes/notes/plugin-service-client-registration-00b19a2dd4935ba0.yaml b/releasenotes/notes/plugin-service-client-registration-00b19a2dd4935ba0.yaml
new file mode 100644
index 0000000..64f729a
--- /dev/null
+++ b/releasenotes/notes/plugin-service-client-registration-00b19a2dd4935ba0.yaml
@@ -0,0 +1,12 @@
+---
+features:
+ - A new optional interface `TempestPlugin.get_service_clients`
+ is available to plugins. It allows them to declare
+ any service client they implement. For now this is used by
+ tempest only, for auto-registration of service clients
+ in the new class `ServiceClients`.
+ - A new singleton class `clients.ClientsRegistry` is
+ available. It holds the service clients registration data
+ from all plugins. It is used by `ServiceClients` for
+ auto-registration of the service clients implemented
+ in plugins.
diff --git a/tempest/lib/exceptions.py b/tempest/lib/exceptions.py
index 5ca78f9..de2d713 100644
--- a/tempest/lib/exceptions.py
+++ b/tempest/lib/exceptions.py
@@ -229,3 +229,13 @@
class UnknownServiceClient(TempestException):
message = "Service clients named %(services)s are not known"
+
+
+class ServiceClientRegistrationException(TempestException):
+ message = ("Error registering module %(name)s in path %(module_path)s, "
+ "with service %(service_version)s and clients "
+ "%(client_names)s: %(detailed_error)s")
+
+
+class PluginRegistrationException(TempestException):
+ message = "Error registering plugin %(name)s: %(detailed_error)s"
diff --git a/tempest/lib/services/clients.py b/tempest/lib/services/clients.py
new file mode 100644
index 0000000..8054e62
--- /dev/null
+++ b/tempest/lib/services/clients.py
@@ -0,0 +1,36 @@
+# Copyright (c) 2016 Hewlett-Packard Enterprise Development Company, L.P.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from tempest.lib.common.utils import misc
+from tempest.lib import exceptions
+
+
+@misc.singleton
+class ClientsRegistry(object):
+ """Registry of all service clients available from plugins"""
+
+ def __init__(self):
+ self._service_clients = {}
+
+ def register_service_client(self, plugin_name, service_client_data):
+ if plugin_name in self._service_clients:
+ detailed_error = 'Clients for plugin %s already registered'
+ raise exceptions.PluginRegistrationException(
+ name=plugin_name,
+ detailed_error=detailed_error % plugin_name)
+ self._service_clients[plugin_name] = service_client_data
+
+ def get_service_clients(self):
+ return self._service_clients
diff --git a/tempest/service_clients.py b/tempest/service_clients.py
index 252ebf4..d0daa2b 100644
--- a/tempest/service_clients.py
+++ b/tempest/service_clients.py
@@ -17,9 +17,13 @@
import copy
import importlib
import inspect
+import logging
from tempest.lib import auth
from tempest.lib import exceptions
+from tempest.lib.services import clients
+
+LOG = logging.getLogger(__name__)
def tempest_modules():
@@ -33,9 +37,48 @@
def available_modules():
- """List of service client modules available in Tempest and plugins"""
- # TODO(andreaf) For now this returns only tempest_modules
- return tempest_modules()
+ """List of service client modules available in Tempest and plugins
+
+ The list of available modules can be used for automatic configuration.
+
+ :raise PluginRegistrationException: if a plugin exposes a service_version
+ already defined by Tempest or another plugin.
+
+ Examples:
+
+ >>> from tempest import config
+ >>> params = {}
+ >>> for service_version in available_modules():
+ >>> service = service_version.split('.')[0]
+ >>> params[service] = config.service_client_config(service)
+ >>> service_clients = ServiceClients(creds, identity_uri,
+ >>> client_parameters=params)
+ """
+ extra_service_versions = set([])
+ plugin_services = clients.ClientsRegistry().get_service_clients()
+ for plugin_name in plugin_services:
+ plug_service_versions = set([x['service_version'] for x in
+ plugin_services[plugin_name]])
+ # If a plugin exposes a duplicate service_version raise an exception
+ if plug_service_versions:
+ if not plug_service_versions.isdisjoint(extra_service_versions):
+ detailed_error = (
+ 'Plugin %s is trying to register a service %s already '
+ 'claimed by another one' % (plugin_name,
+ extra_service_versions &
+ plug_service_versions))
+ raise exceptions.PluginRegistrationException(
+ name=plugin_name, detailed_error=detailed_error)
+ if not plug_service_versions.isdisjoint(tempest_modules()):
+ detailed_error = (
+ 'Plugin %s is trying to register a service %s already '
+ 'claimed by a Tempest one' % (plugin_name,
+ tempest_modules() &
+ plug_service_versions))
+ raise exceptions.PluginRegistrationException(
+ name=plugin_name, detailed_error=detailed_error)
+ extra_service_versions |= plug_service_versions
+ return tempest_modules() | extra_service_versions
class ClientsFactory(object):
@@ -49,8 +92,6 @@
ClientsFactory can be used directly, or consumed via the `ServiceClients`
class, which manages the authorization part.
"""
- # TODO(andreaf) This version includes ClientsFactory but it does not
- # use it yet in ServiceClients
def __init__(self, module_path, client_names, auth_provider, **kwargs):
"""Initialises the client factory
@@ -206,6 +247,7 @@
>>> client_parameters['service_y'] = params_service_y
"""
+ self._registered_services = set([])
self.credentials = credentials
self.identity_uri = identity_uri
if not identity_uri:
@@ -247,6 +289,84 @@
raise exceptions.UnknownServiceClient(
services=list(client_parameters.keys()))
+ # Register service clients from plugins
+ clients_registry = clients.ClientsRegistry()
+ plugin_service_clients = clients_registry.get_service_clients()
+ for plugin in plugin_service_clients:
+ service_clients = plugin_service_clients[plugin]
+ # Each plugin returns a list of service client parameters
+ for service_client in service_clients:
+ # NOTE(andreaf) If a plugin cannot register, stop the
+ # registration process, log some details to help
+ # troubleshooting, and re-raise
+ try:
+ self.register_service_client_module(**service_client)
+ except Exception:
+ LOG.exception(
+ 'Failed to register service client from plugin %s '
+ 'with parameters %s' % (plugin, service_client))
+ raise
+
+ def register_service_client_module(self, name, service_version,
+ module_path, client_names, **kwargs):
+ """Register a service client module
+
+ Initiates a client factory for the specified module, using this
+ class auth_provider, and accessible via a `name` attribute in the
+ service client.
+
+ :param name: Name used to access the client
+ :param service_version: Name of the service complete with version.
+ Used to track registered services. When a plugin implements it,
+ it can be used by other plugins to obtain their configuration.
+ :param module_path: Path to module that includes all service clients.
+ All service client classes must be exposed by a single module.
+ If they are separated in different modules, defining __all__
+ in the root module can help, similar to what is done by service
+ clients in tempest.
+ :param client_names: List or set of names of service client classes.
+ :param kwargs: Extra optional parameters to be passed to all clients.
+ ServiceClient provides defaults for region, dscv, ca_certs and
+ trace_requests.
+ :raise ServiceClientRegistrationException: if the provided name is
+ already in use or if service_version is already registered.
+ :raise ImportError: if module_path cannot be imported.
+ """
+ if hasattr(self, name):
+ using_name = getattr(self, name)
+ detailed_error = 'Module name already in use: %s' % using_name
+ raise exceptions.ServiceClientRegistrationException(
+ name=name, service_version=service_version,
+ module_path=module_path, client_names=client_names,
+ detailed_error=detailed_error)
+ if service_version in self.registered_services:
+ detailed_error = 'Service %s already registered.' % service_version
+ raise exceptions.ServiceClientRegistrationException(
+ name=name, service_version=service_version,
+ module_path=module_path, client_names=client_names,
+ detailed_error=detailed_error)
+ params = dict(region=self.region,
+ disable_ssl_certificate_validation=self.dscv,
+ ca_certs=self.ca_certs,
+ trace_requests=self.trace_requests)
+ params.update(kwargs)
+ # Instantiate the client factory
+ _factory = ClientsFactory(module_path=module_path,
+ client_names=client_names,
+ auth_provider=self.auth_provider,
+ **params)
+ # Adds the client factory to the service_client
+ setattr(self, name, _factory)
+ # Add the name of the new service in self.SERVICES for discovery
+ self._registered_services.add(service_version)
+
+ @property
+ def registered_services(self):
+ # TODO(andreaf) For now add all Tempest services. to the list of
+ # registered service
+ _default_services = tempest_modules()
+ return self._registered_services | _default_services
+
def _setup_parameters(self, parameters):
"""Setup default values for client parameters
diff --git a/tempest/test_discover/plugins.py b/tempest/test_discover/plugins.py
index d604b28..cfb0c7f 100644
--- a/tempest/test_discover/plugins.py
+++ b/tempest/test_discover/plugins.py
@@ -19,6 +19,7 @@
import stevedore
from tempest.lib.common.utils import misc
+from tempest.lib.services import clients
LOG = logging.getLogger(__name__)
@@ -62,6 +63,54 @@
"""
return []
+ def get_service_clients(self):
+ """Get a list of the service clients for registration
+
+ If the plugin implements service clients for one or more APIs, it
+ may return their details by this method for automatic registration
+ in any ServiceClients object instantiated by tests.
+ The default implementation returns an empty list.
+
+ :return list of dictionaries. Each element of the list represents
+ the service client for an API. Each dict must define all
+ parameters required for the invocation of
+ `service_clients.ServiceClients.register_service_client_module`.
+ :rtype: list
+
+ Example:
+
+ >>> # Example implementation with one service client
+ >>> myservice_config = config.service_client_config('myservice')
+ >>> params = {
+ >>> 'name': 'myservice',
+ >>> 'service_version': 'myservice',
+ >>> 'module_path': 'myservice_tempest_tests.services',
+ >>> 'client_names': ['API1Client', 'API2Client'],
+ >>> }
+ >>> params.update(myservice_config)
+ >>> return [params]
+
+ >>> # Example implementation with two service clients
+ >>> foo1_config = config.service_client_config('foo')
+ >>> params_foo1 = {
+ >>> 'name': 'foo_v1',
+ >>> 'service_version': 'foo.v1',
+ >>> 'module_path': 'bar_tempest_tests.services.foo.v1',
+ >>> 'client_names': ['API1Client', 'API2Client'],
+ >>> }
+ >>> params_foo1.update(foo_config)
+ >>> foo2_config = config.service_client_config('foo')
+ >>> params_foo2 = {
+ >>> 'name': 'foo_v2',
+ >>> 'service_version': 'foo.v2',
+ >>> 'module_path': 'bar_tempest_tests.services.foo.v2',
+ >>> 'client_names': ['API1Client', 'API2Client'],
+ >>> }
+ >>> params_foo2.update(foo2_config)
+ >>> return [params_foo1, params_foo2]
+ """
+ return []
+
@misc.singleton
class TempestTestPluginManager(object):
@@ -75,6 +124,7 @@
'tempest.test_plugins', invoke_on_load=True,
propagate_map_exceptions=True,
on_load_failure_callback=self.failure_hook)
+ self._register_service_clients()
@staticmethod
def failure_hook(_, ep, err):
@@ -102,3 +152,13 @@
if opt_list:
plugin_options.extend(opt_list)
return plugin_options
+
+ def _register_service_clients(self):
+ registry = clients.ClientsRegistry()
+ for plug in self.ext_plugins:
+ try:
+ registry.register_service_client(
+ plug.name, plug.obj.get_service_clients())
+ except Exception:
+ LOG.exception('Plugin %s raised an exception trying to run '
+ 'get_service_clients' % plug.name)
diff --git a/tempest/tests/fake_tempest_plugin.py b/tempest/tests/fake_tempest_plugin.py
index f718d0b..56aae1e 100644
--- a/tempest/tests/fake_tempest_plugin.py
+++ b/tempest/tests/fake_tempest_plugin.py
@@ -18,6 +18,7 @@
class FakePlugin(plugins.TempestPlugin):
expected_load_test = ["my/test/path", "/home/dir"]
+ expected_service_clients = [{'foo': 'bar'}]
def load_tests(self):
return self.expected_load_test
@@ -28,6 +29,9 @@
def get_opt_lists(self):
return []
+ def get_service_clients(self):
+ return self.expected_service_clients
+
class FakeStevedoreObj(object):
obj = FakePlugin()
@@ -38,3 +42,26 @@
def __init__(self, name='Test1'):
self._name = name
+
+
+class FakePluginNoServiceClients(plugins.TempestPlugin):
+
+ def load_tests(self):
+ return []
+
+ def register_opts(self, conf):
+ return
+
+ def get_opt_lists(self):
+ return []
+
+
+class FakeStevedoreObjNoServiceClients(object):
+ obj = FakePluginNoServiceClients()
+
+ @property
+ def name(self):
+ return self._name
+
+ def __init__(self, name='Test2'):
+ self._name = name
diff --git a/tempest/tests/test_service_clients.py b/tempest/tests/test_service_clients.py
index 26cc93f..befed68 100644
--- a/tempest/tests/test_service_clients.py
+++ b/tempest/tests/test_service_clients.py
@@ -263,3 +263,82 @@
for _key in _params.keys():
self.assertEqual(expected_params[_key],
_params[_key])
+
+ def test_register_service_client_module(self):
+ factory_mock = self.useFixture(fixtures.MockPatch(
+ 'tempest.service_clients.ClientsFactory')).mock
+ expected_params = {'fake_param1': 'fake_value1',
+ 'fake_param2': 'fake_value2'}
+ _manager = self._get_manager(init_region='fake_region_default')
+ _manager.register_service_client_module(
+ name='fake_module',
+ service_version='fake_service',
+ module_path='fake.path.to.module',
+ client_names=[],
+ **expected_params)
+ self.assertThat(_manager, has_attribute('fake_module'))
+ # Assert called once, without check for exact parameters
+ self.assertTrue(factory_mock.called)
+ self.assertEqual(1, factory_mock.call_count)
+ # Assert expected params are in with their values
+ actual_kwargs = factory_mock.call_args[1]
+ self.assertIn('region', actual_kwargs)
+ self.assertEqual('fake_region_default', actual_kwargs['region'])
+ for param in expected_params:
+ self.assertIn(param, actual_kwargs)
+ self.assertEqual(expected_params[param], actual_kwargs[param])
+ # Assert the new service is registered
+ self.assertIn('fake_service', _manager._registered_services)
+
+ def test_register_service_client_module_override_default(self):
+ factory_mock = self.useFixture(fixtures.MockPatch(
+ 'tempest.service_clients.ClientsFactory')).mock
+ new_region = 'new_region'
+ expected_params = {'fake_param1': 'fake_value1',
+ 'fake_param2': 'fake_value2',
+ 'region': new_region}
+ _manager = self._get_manager(init_region='fake_region_default')
+ _manager.register_service_client_module(
+ name='fake_module',
+ service_version='fake_service',
+ module_path='fake.path.to.module',
+ client_names=[],
+ **expected_params)
+ self.assertThat(_manager, has_attribute('fake_module'))
+ # Assert called once, without check for exact parameters
+ self.assertTrue(factory_mock.called)
+ self.assertEqual(1, factory_mock.call_count)
+ # Assert expected params are in with their values
+ actual_kwargs = factory_mock.call_args[1]
+ self.assertIn('region', actual_kwargs)
+ self.assertEqual(new_region, actual_kwargs['region'])
+ for param in expected_params:
+ self.assertIn(param, actual_kwargs)
+ self.assertEqual(expected_params[param], actual_kwargs[param])
+ # Assert the new service is registered
+ self.assertIn('fake_service', _manager._registered_services)
+
+ def test_register_service_client_module_duplicate_name(self):
+ self.useFixture(fixtures.MockPatch(
+ 'tempest.service_clients.ClientsFactory'))
+ _manager = self._get_manager()
+ name_owner = 'this_is_a_string'
+ setattr(_manager, 'fake_module', name_owner)
+ expected_error = '.*' + name_owner
+ with testtools.ExpectedException(
+ exceptions.ServiceClientRegistrationException, expected_error):
+ _manager.register_service_client_module(
+ name='fake_module', module_path='fake.path.to.module',
+ service_version='fake_service', client_names=[])
+
+ def test_register_service_client_module_duplicate_service(self):
+ self.useFixture(fixtures.MockPatch(
+ 'tempest.service_clients.ClientsFactory'))
+ _manager = self._get_manager()
+ duplicate_service = 'fake_service1'
+ expected_error = '.*' + duplicate_service
+ with testtools.ExpectedException(
+ exceptions.ServiceClientRegistrationException, expected_error):
+ _manager.register_service_client_module(
+ name='fake_module', module_path='fake.path.to.module',
+ service_version=duplicate_service, client_names=[])
diff --git a/tempest/tests/test_tempest_plugin.py b/tempest/tests/test_tempest_plugin.py
index c07e98c..dd50125 100644
--- a/tempest/tests/test_tempest_plugin.py
+++ b/tempest/tests/test_tempest_plugin.py
@@ -13,6 +13,7 @@
# License for the specific language governing permissions and limitations
# under the License.
+from tempest.lib.services import clients
from tempest.test_discover import plugins
from tempest.tests import base
from tempest.tests import fake_tempest_plugin as fake_plugin
@@ -42,3 +43,39 @@
result['fake01'])
self.assertEqual(fake_plugin.FakePlugin.expected_load_test,
result['fake02'])
+
+ def test__register_service_clients_with_one_plugin(self):
+ registry = clients.ClientsRegistry()
+ manager = plugins.TempestTestPluginManager()
+ fake_obj = fake_plugin.FakeStevedoreObj()
+ manager.ext_plugins = [fake_obj]
+ manager._register_service_clients()
+ expected_result = fake_plugin.FakePlugin.expected_service_clients
+ registered_clients = registry.get_service_clients()
+ self.assertIn(fake_obj.name, registered_clients)
+ self.assertEqual(expected_result, registered_clients[fake_obj.name])
+
+ def test__get_service_clients_with_two_plugins(self):
+ registry = clients.ClientsRegistry()
+ manager = plugins.TempestTestPluginManager()
+ obj1 = fake_plugin.FakeStevedoreObj('fake01')
+ obj2 = fake_plugin.FakeStevedoreObj('fake02')
+ manager.ext_plugins = [obj1, obj2]
+ manager._register_service_clients()
+ expected_result = fake_plugin.FakePlugin.expected_service_clients
+ registered_clients = registry.get_service_clients()
+ self.assertIn('fake01', registered_clients)
+ self.assertIn('fake02', registered_clients)
+ self.assertEqual(expected_result, registered_clients['fake01'])
+ self.assertEqual(expected_result, registered_clients['fake02'])
+
+ def test__register_service_clients_one_plugin_no_service_clients(self):
+ registry = clients.ClientsRegistry()
+ manager = plugins.TempestTestPluginManager()
+ fake_obj = fake_plugin.FakeStevedoreObjNoServiceClients()
+ manager.ext_plugins = [fake_obj]
+ manager._register_service_clients()
+ expected_result = []
+ registered_clients = registry.get_service_clients()
+ self.assertIn(fake_obj.name, registered_clients)
+ self.assertEqual(expected_result, registered_clients[fake_obj.name])