tempest: rework gabbi setup
The current approach is a bit hacky and create one
tempest scenario per gabbi HTTP call.
The side effect is that tests continue to run even the previous
one have fail.
This changes the approach by running gabbi to have one scenario
per yaml file from tempest point of view.
This will make easier to debug the scenario in case of failure.
Change-Id: I594288322d9ac5d3d128d601cba1d2291a632e20
diff --git a/ceilometer/tests/integration/hooks/post_test_hook.sh b/ceilometer/tests/integration/hooks/post_test_hook.sh
index 051b4d8..1e35fd4 100755
--- a/ceilometer/tests/integration/hooks/post_test_hook.sh
+++ b/ceilometer/tests/integration/hooks/post_test_hook.sh
@@ -77,17 +77,13 @@
}
-# If we're running in the gate find our keystone endpoint to give to
-# gabbi tests and do a chown. Otherwise the existing environment
-# should provide URL and TOKEN.
-if [ -d $BASE/new/devstack ]; then
- export CEILOMETER_DIR="$BASE/new/ceilometer"
- STACK_USER=stack
- sudo chown -R $STACK_USER:stack $CEILOMETER_DIR
- source $BASE/new/devstack/openrc admin admin
- # Go to the ceilometer dir
- cd $CEILOMETER_DIR
-fi
+export CEILOMETER_DIR="$BASE/new/ceilometer"
+STACK_USER=stack
+sudo chown -R $STACK_USER:stack $CEILOMETER_DIR
+# NOTE(sileht): on swift job permissions are wrong, I don't known why
+sudo chown -R tempest:stack $BASE/new/tempest
+sudo chown -R tempest:stack $BASE/data/tempest
+source $BASE/new/devstack/openrc admin admin
openstack catalog list
export AODH_SERVICE_URL=$(openstack catalog show alarming -c endpoints -f value | awk '/public/{print $2}')
@@ -99,26 +95,11 @@
export ADMIN_TOKEN=$(openstack token issue -c id -f value)
export OS_AUTH_TYPE=password
-# Run tests with gabbi
-echo "Running telemetry integration test suite"
-set +e
-sudo -E -H -u ${STACK_USER:-${USER}} tox -eintegration
+# Run tests with tempest
+cd $BASE/new/tempest
+sudo -H -u tempest OS_TEST_TIMEOUT=$TEMPEST_OS_TEST_TIMEOUT tox -eall-plugin -- ceilometer.tests.tempest.scenario.test_telemetry_integration --concurrency=$TEMPEST_CONCURRENCY
EXIT_CODE=$?
-
-if [ -d $BASE/new/devstack ]; then
- export_subunit_data "integration"
- generate_reports_and_maybe_exit $EXIT_CODE
-
- # NOTE(sileht): on swift job permissions are wrong, I don't known why
- sudo chown -R tempest:stack $BASE/new/tempest
- sudo chown -R tempest:stack $BASE/data/tempest
-
- # Run tests with tempest
- cd $BASE/new/tempest
- sudo -H -u tempest OS_TEST_TIMEOUT=$TEMPEST_OS_TEST_TIMEOUT tox -eall-plugin -- ceilometer.tests.tempest.scenario.test_autoscaling --concurrency=$TEMPEST_CONCURRENCY
- EXIT_CODE=$?
- export_subunit_data "all-plugin"
- generate_reports_and_maybe_exit $EXIT_CODE
-fi
+export_subunit_data "all-plugin"
+generate_reports_and_maybe_exit $EXIT_CODE
exit $EXIT_CODE
diff --git a/ceilometer/tests/tempest/scenario/test_autoscaling.py b/ceilometer/tests/tempest/scenario/test_autoscaling.py
deleted file mode 100644
index 4932ed3..0000000
--- a/ceilometer/tests/tempest/scenario/test_autoscaling.py
+++ /dev/null
@@ -1,116 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import os
-import unittest
-
-from gabbi import driver
-from tempest import config
-
-from ceilometer.tests.tempest.scenario import manager
-
-
-class TestAutoscalingGabbi(manager.ScenarioTest):
- credentials = ['admin', 'primary']
-
- @classmethod
- def skip_checks(cls):
- super(TestAutoscalingGabbi, cls).skip_checks()
- for name in ["aodh_plugin", "gnocchi", "nova", "heat", "panko",
- "ceilometer", "glance"]:
- cls._check_service(name)
-
- @classmethod
- def _check_service(cls, name):
- if not getattr(config.CONF.service_available, name, False):
- raise cls.skipException("%s support is required" %
- name.capitalize())
-
- @classmethod
- def resource_setup(cls):
- super(TestAutoscalingGabbi, cls).resource_setup()
- test_dir = os.path.join(os.path.dirname(__file__), '..', '..',
- 'integration', 'gabbi', 'gabbits-live')
- cls.tests = driver.build_tests(
- test_dir, unittest.TestLoader(),
- host='localhost', port='13245',
- test_loader_name='tempest.scenario.telemetry-autoscaling.test')
-
- auth = cls.os_admin.auth_provider.get_auth()
- os.environ["ADMIN_TOKEN"] = auth[0]
- os.environ["AODH_SERVICE_URL"] = cls._get_endpoint_for(
- auth, "alarming_plugin")
- os.environ["GNOCCHI_SERVICE_URL"] = cls._get_endpoint_for(
- auth, "metric")
- os.environ["PANKO_SERVICE_URL"] = cls._get_endpoint_for(
- auth, "event")
- os.environ["HEAT_SERVICE_URL"] = cls._get_endpoint_for(
- auth, "orchestration")
- os.environ["NOVA_SERVICE_URL"] = cls._get_endpoint_for(auth, "compute")
- os.environ["GLANCE_SERVICE_URL"] = cls._get_endpoint_for(auth, "image")
-
- @staticmethod
- def clear_credentials():
- # FIXME(sileht): We don't want the token to be invalided, but
- # for some obcurs reason, clear_credentials is called before/during run
- # So, make the one used by tearDropClass a dump, and call it manually
- # in run()
- pass
-
- def run(self, result=None):
- self.setUp()
- os.environ["GLANCE_IMAGE_NAME"] = self.glance_image_create()
- try:
- self.tests.run(result)
- finally:
- super(TestAutoscalingGabbi, self).clear_credentials()
- self.tearDown()
-
- @staticmethod
- def _get_endpoint_for(auth, service):
- opt_section = getattr(config.CONF, service)
- endpoint_type = opt_section.endpoint_type
- is_keystone_v3 = 'catalog' in auth[1]
-
- if is_keystone_v3:
- if endpoint_type.endswith("URL"):
- endpoint_type = endpoint_type[:-3]
- catalog = auth[1]['catalog']
- endpoints = [e['endpoints'] for e in catalog
- if e['type'] == opt_section.catalog_type]
- if not endpoints:
- raise Exception("%s endpoint not found" %
- opt_section.catalog_type)
- endpoints = [e['url'] for e in endpoints[0]
- if e['interface'] == endpoint_type]
- if not endpoints:
- raise Exception("%s interface not found for endpoint %s" %
- (endpoint_type,
- opt_section.catalog_type))
- return endpoints[0]
-
- else:
- if not endpoint_type.endswith("URL"):
- endpoint_type += "URL"
- catalog = auth[1]['serviceCatalog']
- endpoints = [e for e in catalog
- if e['type'] == opt_section.catalog_type]
- if not endpoints:
- raise Exception("%s endpoint not found" %
- opt_section.catalog_type)
- return endpoints[0]['endpoints'][0][endpoint_type]
-
- @staticmethod
- def test_fake():
- # NOTE(sileht): A fake test is needed to have the class loaded
- # by the test runner
- pass
diff --git a/ceilometer/tests/tempest/scenario/test_telemetry_integration.py b/ceilometer/tests/tempest/scenario/test_telemetry_integration.py
new file mode 100644
index 0000000..afede75
--- /dev/null
+++ b/ceilometer/tests/tempest/scenario/test_telemetry_integration.py
@@ -0,0 +1,134 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import unittest
+
+from gabbi import runner
+from gabbi import suitemaker
+from gabbi import utils
+from tempest import config
+
+from ceilometer.tests.tempest.scenario import manager
+
+TEST_DIR = os.path.join(os.path.dirname(__file__), '..', '..',
+ 'integration', 'gabbi', 'gabbits-live')
+
+
+class TestTelemetryIntegration(manager.ScenarioTest):
+ credentials = ['admin', 'primary']
+
+ @classmethod
+ def skip_checks(cls):
+ super(TestTelemetryIntegration, cls).skip_checks()
+ for name in ["aodh_plugin", "gnocchi", "nova", "heat", "panko",
+ "ceilometer", "glance"]:
+ cls._check_service(name)
+
+ @classmethod
+ def _check_service(cls, name):
+ if not getattr(config.CONF.service_available, name, False):
+ raise cls.skipException("%s support is required" %
+ name.capitalize())
+
+ @staticmethod
+ def _get_endpoint(auth, service):
+ opt_section = getattr(config.CONF, service)
+ endpoint_type = opt_section.endpoint_type
+ is_keystone_v3 = 'catalog' in auth[1]
+
+ if is_keystone_v3:
+ if endpoint_type.endswith("URL"):
+ endpoint_type = endpoint_type[:-3]
+ catalog = auth[1]['catalog']
+ endpoints = [e['endpoints'] for e in catalog
+ if e['type'] == opt_section.catalog_type]
+ if not endpoints:
+ raise Exception("%s endpoint not found" %
+ opt_section.catalog_type)
+ endpoints = [e['url'] for e in endpoints[0]
+ if e['interface'] == endpoint_type]
+ if not endpoints:
+ raise Exception("%s interface not found for endpoint %s" %
+ (endpoint_type,
+ opt_section.catalog_type))
+ return endpoints[0]
+
+ else:
+ if not endpoint_type.endswith("URL"):
+ endpoint_type += "URL"
+ catalog = auth[1]['serviceCatalog']
+ endpoints = [e for e in catalog
+ if e['type'] == opt_section.catalog_type]
+ if not endpoints:
+ raise Exception("%s endpoint not found" %
+ opt_section.catalog_type)
+ return endpoints[0]['endpoints'][0][endpoint_type]
+
+ def _do_test(self, filename):
+ auth = self.os_admin.auth_provider.get_auth()
+
+ os.environ.update({
+ "ADMIN_TOKEN": auth[0],
+ "AODH_SERVICE_URL": self._get_endpoint(auth, "alarming_plugin"),
+ "GNOCCHI_SERVICE_URL": self._get_endpoint(auth, "metric"),
+ "PANKO_SERVICE_URL": self._get_endpoint(auth, "event"),
+ "HEAT_SERVICE_URL": self._get_endpoint(auth, "orchestration"),
+ "NOVA_SERVICE_URL": self._get_endpoint(auth, "compute"),
+ "GLANCE_SERVICE_URL": self._get_endpoint(auth, "image"),
+ "GLANCE_IMAGE_NAME": self.glance_image_create(),
+ })
+
+ with file(os.path.join(TEST_DIR, filename)) as f:
+ test_suite = suitemaker.test_suite_from_dict(
+ loader=unittest.defaultTestLoader,
+ test_base_name="gabbi",
+ suite_dict=utils.load_yaml(f),
+ test_directory=TEST_DIR,
+ host=None, port=None,
+ fixture_module=None,
+ intercept=None,
+ handlers=runner.initialize_handlers([]),
+ test_loader_name="tempest")
+
+ # NOTE(sileht): We hide stdout/stderr and reraise the failure
+ # manually, tempest will print it itself.
+ with open(os.devnull, 'w') as stream:
+ result = unittest.TextTestRunner(
+ stream=stream, verbosity=0, failfast=True,
+ ).run(test_suite)
+
+ if not result.wasSuccessful():
+ failures = (result.errors + result.failures +
+ result.unexpectedSuccesses)
+ if failures:
+ test, bt = failures[0]
+ name = test.test_data.get('name', test.id())
+ msg = 'From test "%s" :\n%s' % (name, bt)
+ self.fail(msg)
+
+ self.assertTrue(result.wasSuccessful())
+
+
+def test_maker(name, filename):
+ def test(self):
+ self._do_test(filename)
+ test.__name__ = name
+ return test
+
+# Create one scenario per yaml file
+for filename in os.listdir(TEST_DIR):
+ if not filename.endswith('.yaml'):
+ continue
+ name = "test_%s" % filename[:-5].lower().replace("-", "_")
+ setattr(TestTelemetryIntegration, name,
+ test_maker(name, filename))