Merge "Updating HACKING with some test writing recommendations"
diff --git a/etc/tempest.conf.sample b/etc/tempest.conf.sample
index f5e51cd..a73e8a0 100644
--- a/etc/tempest.conf.sample
+++ b/etc/tempest.conf.sample
@@ -209,8 +209,6 @@
# for each tenant to have their own router.
public_router_id = {$PUBLIC_ROUTER_ID}
-# Whether or not neutron is expected to be available
-neutron_available = false
[volume]
# This section contains the configuration options used when executing tests
@@ -308,9 +306,6 @@
# tests spawn full VMs, which could be slow if the test is already in a VM.
build_timeout = 300
-# Whether or not Heat is expected to be available
-heat_available = false
-
# Instance type for tests. Needs to be big enough for a
# full OS plus the test workload
instance_type = m1.micro
@@ -348,3 +343,17 @@
enabled = True
# directory where python client binaries are located
cli_dir = /usr/local/bin
+
+[service_available]
+# Whether or not cinder is expected to be available
+cinder = True
+# Whether or not neutron is expected to be available
+neutron = false
+# Whether or not glance is expected to be available
+glance = True
+# Whether or not swift is expected to be available
+swift = True
+# Whether or not nova is expected to be available
+nova = True
+# Whether or not Heat is expected to be available
+heat = false
diff --git a/run_tests.sh b/run_tests.sh
index d5081c7..a645b22 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -118,7 +118,7 @@
if [ $with_testr -eq 1 ]; then
testr_init
${wrapper} find . -type f -name "*.pyc" -delete
- ${wrapper} testr run --parallel $noseargs
+ ${wrapper} testr run --parallel --subunit $noseargs | ${wrapper} subunit-2to1 | ${wrapper} tools/colorizer.py
else
${wrapper} $NOSETESTS
fi
diff --git a/tempest/api/compute/admin/test_fixed_ips.py b/tempest/api/compute/admin/test_fixed_ips.py
index 2eaf3b0..8b96370 100644
--- a/tempest/api/compute/admin/test_fixed_ips.py
+++ b/tempest/api/compute/admin/test_fixed_ips.py
@@ -56,7 +56,7 @@
CONF = config.TempestConfig()
- @testtools.skipIf(CONF.network.neutron_available, "This feature is not" +
+ @testtools.skipIf(CONF.service_available.neutron, "This feature is not" +
"implemented by Neutron. See bug: #1194569")
@attr(type='gate')
def test_list_fixed_ip_details(self):
diff --git a/tempest/api/compute/base.py b/tempest/api/compute/base.py
index abc5899..095be7c 100644
--- a/tempest/api/compute/base.py
+++ b/tempest/api/compute/base.py
@@ -36,6 +36,9 @@
@classmethod
def setUpClass(cls):
+ if not cls.config.service_available.nova:
+ skip_msg = ("%s skipped as nova is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
cls.isolated_creds = []
if cls.config.compute.allow_tenant_isolation:
diff --git a/tempest/api/compute/images/test_image_metadata.py b/tempest/api/compute/images/test_image_metadata.py
index 7b8e1cc..52239cd 100644
--- a/tempest/api/compute/images/test_image_metadata.py
+++ b/tempest/api/compute/images/test_image_metadata.py
@@ -27,6 +27,10 @@
@classmethod
def setUpClass(cls):
super(ImagesMetadataTestJSON, cls).setUpClass()
+ if not cls.config.service_available.glance:
+ skip_msg = ("%s skipped as glance is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
+
cls.servers_client = cls.servers_client
cls.client = cls.images_client
diff --git a/tempest/api/compute/images/test_images.py b/tempest/api/compute/images/test_images.py
index f9b4346..95ea820 100644
--- a/tempest/api/compute/images/test_images.py
+++ b/tempest/api/compute/images/test_images.py
@@ -30,6 +30,9 @@
@classmethod
def setUpClass(cls):
super(ImagesTestJSON, cls).setUpClass()
+ if not cls.config.service_available.glance:
+ skip_msg = ("%s skipped as glance is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
cls.client = cls.images_client
cls.servers_client = cls.servers_client
diff --git a/tempest/api/compute/images/test_images_oneserver.py b/tempest/api/compute/images/test_images_oneserver.py
index 7740cfc..64f1854 100644
--- a/tempest/api/compute/images/test_images_oneserver.py
+++ b/tempest/api/compute/images/test_images_oneserver.py
@@ -40,6 +40,9 @@
def setUpClass(cls):
super(ImagesOneServerTestJSON, cls).setUpClass()
cls.client = cls.images_client
+ if not cls.config.service_available.glance:
+ skip_msg = ("%s skipped as glance is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
try:
resp, cls.server = cls.create_server(wait_until='ACTIVE')
diff --git a/tempest/api/compute/images/test_list_image_filters.py b/tempest/api/compute/images/test_list_image_filters.py
index 5c6b630..b27d710 100644
--- a/tempest/api/compute/images/test_list_image_filters.py
+++ b/tempest/api/compute/images/test_list_image_filters.py
@@ -31,6 +31,9 @@
@classmethod
def setUpClass(cls):
super(ListImageFiltersTestJSON, cls).setUpClass()
+ if not cls.config.service_available.glance:
+ skip_msg = ("%s skipped as glance is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
cls.client = cls.images_client
cls.image_ids = []
diff --git a/tempest/api/compute/images/test_list_images.py b/tempest/api/compute/images/test_list_images.py
index fddad14..c7e23b1 100644
--- a/tempest/api/compute/images/test_list_images.py
+++ b/tempest/api/compute/images/test_list_images.py
@@ -25,6 +25,9 @@
@classmethod
def setUpClass(cls):
super(ListImagesTestJSON, cls).setUpClass()
+ if not cls.config.service_available.glance:
+ skip_msg = ("%s skipped as glance is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
cls.client = cls.images_client
@attr(type='smoke')
diff --git a/tempest/api/compute/security_groups/test_security_groups.py b/tempest/api/compute/security_groups/test_security_groups.py
index e105121..ab100a3 100644
--- a/tempest/api/compute/security_groups/test_security_groups.py
+++ b/tempest/api/compute/security_groups/test_security_groups.py
@@ -158,7 +158,7 @@
self.client.create_security_group, s_name,
s_description)
- @testtools.skipIf(config.TempestConfig().network.neutron_available,
+ @testtools.skipIf(config.TempestConfig().service_available.neutron,
"Neutron allows duplicate names for security groups")
@attr(type=['negative', 'gate'])
def test_security_group_create_with_duplicate_name(self):
diff --git a/tempest/api/compute/servers/test_attach_interfaces.py b/tempest/api/compute/servers/test_attach_interfaces.py
index de095c5..9f66a6c 100644
--- a/tempest/api/compute/servers/test_attach_interfaces.py
+++ b/tempest/api/compute/servers/test_attach_interfaces.py
@@ -24,7 +24,7 @@
@classmethod
def setUpClass(cls):
- if not cls.config.network.neutron_available:
+ if not cls.config.service_available.neutron:
raise cls.skipException("Neutron is required")
super(AttachInterfacesTestJSON, cls).setUpClass()
cls.client = cls.os.interfaces_client
diff --git a/tempest/api/compute/servers/test_virtual_interfaces.py b/tempest/api/compute/servers/test_virtual_interfaces.py
index 35f0fc0..2a5be8c 100644
--- a/tempest/api/compute/servers/test_virtual_interfaces.py
+++ b/tempest/api/compute/servers/test_virtual_interfaces.py
@@ -37,7 +37,7 @@
resp, server = cls.create_server(wait_until='ACTIVE')
cls.server_id = server['id']
- @testtools.skipIf(CONF.network.neutron_available, "This feature is not " +
+ @testtools.skipIf(CONF.service_available.neutron, "This feature is not " +
"implemented by Neutron. See bug: #1183436")
@attr(type='gate')
def test_list_virtual_interfaces(self):
diff --git a/tempest/api/compute/volumes/test_attach_volume.py b/tempest/api/compute/volumes/test_attach_volume.py
index b507e03..6571491 100644
--- a/tempest/api/compute/volumes/test_attach_volume.py
+++ b/tempest/api/compute/volumes/test_attach_volume.py
@@ -37,6 +37,9 @@
def setUpClass(cls):
super(AttachVolumeTestJSON, cls).setUpClass()
cls.device = 'vdb'
+ if not cls.config.service_available.cinder:
+ skip_msg = ("%s skipped as Cinder is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
def _detach(self, server_id, volume_id):
self.servers_client.detach_volume(server_id, volume_id)
diff --git a/tempest/api/compute/volumes/test_volumes_get.py b/tempest/api/compute/volumes/test_volumes_get.py
index 1acc57d..363cd6a 100644
--- a/tempest/api/compute/volumes/test_volumes_get.py
+++ b/tempest/api/compute/volumes/test_volumes_get.py
@@ -28,6 +28,9 @@
def setUpClass(cls):
super(VolumesGetTestJSON, cls).setUpClass()
cls.client = cls.volumes_extensions_client
+ if not cls.config.service_available.cinder:
+ skip_msg = ("%s skipped as Cinder is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
@attr(type='smoke')
def test_volume_create_get_delete(self):
diff --git a/tempest/api/compute/volumes/test_volumes_list.py b/tempest/api/compute/volumes/test_volumes_list.py
index d52349e..02cc4e1 100644
--- a/tempest/api/compute/volumes/test_volumes_list.py
+++ b/tempest/api/compute/volumes/test_volumes_list.py
@@ -36,6 +36,9 @@
def setUpClass(cls):
super(VolumesTestJSON, cls).setUpClass()
cls.client = cls.volumes_extensions_client
+ if not cls.config.service_available.cinder:
+ skip_msg = ("%s skipped as Cinder is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
# Create 3 Volumes
cls.volume_list = []
cls.volume_id_list = []
diff --git a/tempest/api/compute/volumes/test_volumes_negative.py b/tempest/api/compute/volumes/test_volumes_negative.py
index de214fc..f1ef5a4 100644
--- a/tempest/api/compute/volumes/test_volumes_negative.py
+++ b/tempest/api/compute/volumes/test_volumes_negative.py
@@ -28,6 +28,9 @@
def setUpClass(cls):
super(VolumesNegativeTest, cls).setUpClass()
cls.client = cls.volumes_extensions_client
+ if not cls.config.service_available.cinder:
+ skip_msg = ("%s skipped as Cinder is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
@attr(type='gate')
def test_volume_get_nonexistant_volume_id(self):
diff --git a/tempest/api/image/base.py b/tempest/api/image/base.py
index e62d84b..e27ec13 100644
--- a/tempest/api/image/base.py
+++ b/tempest/api/image/base.py
@@ -30,6 +30,9 @@
def setUpClass(cls):
cls.os = clients.Manager()
cls.created_images = []
+ if not cls.config.service_available.glance:
+ skip_msg = ("%s skipped as glance is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
@classmethod
def tearDownClass(cls):
diff --git a/tempest/api/network/base.py b/tempest/api/network/base.py
index 3b7f9dd..142ad7d 100644
--- a/tempest/api/network/base.py
+++ b/tempest/api/network/base.py
@@ -44,7 +44,7 @@
def setUpClass(cls):
os = clients.Manager()
cls.network_cfg = os.config.network
- if not cls.network_cfg.neutron_available:
+ if not cls.config.service_available.neutron:
raise cls.skipException("Neutron support is required")
cls.client = os.network_client
cls.networks = []
diff --git a/tempest/api/object_storage/base.py b/tempest/api/object_storage/base.py
index bf013ec..5a1fb5a 100644
--- a/tempest/api/object_storage/base.py
+++ b/tempest/api/object_storage/base.py
@@ -26,6 +26,9 @@
@classmethod
def setUpClass(cls):
+ if not cls.config.service_available.swift:
+ skip_msg = ("%s skipped as swift is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
cls.os = clients.Manager()
cls.object_client = cls.os.object_client
cls.container_client = cls.os.container_client
@@ -42,12 +45,6 @@
cls.data = DataGenerator(cls.identity_admin_client)
- try:
- cls.account_client.list_account_containers()
- except exceptions.EndpointNotFound:
- skip_msg = "No OpenStack Object Storage API endpoint"
- raise cls.skipException(skip_msg)
-
@classmethod
def delete_containers(cls, containers, container_client=None,
object_client=None):
diff --git a/tempest/api/orchestration/base.py b/tempest/api/orchestration/base.py
index ffa534a..a0b248c 100644
--- a/tempest/api/orchestration/base.py
+++ b/tempest/api/orchestration/base.py
@@ -31,7 +31,7 @@
os = clients.OrchestrationManager()
cls.orchestration_cfg = os.config.orchestration
- if not cls.orchestration_cfg.heat_available:
+ if not os.config.service_available.heat:
raise cls.skipException("Heat support is required")
cls.build_timeout = cls.orchestration_cfg.build_timeout
cls.build_interval = cls.orchestration_cfg.build_interval
diff --git a/tempest/api/volume/base.py b/tempest/api/volume/base.py
index fc510cb..5770d28 100644
--- a/tempest/api/volume/base.py
+++ b/tempest/api/volume/base.py
@@ -20,7 +20,6 @@
from tempest import clients
from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
-from tempest import exceptions
import tempest.test
LOG = logging.getLogger(__name__)
@@ -34,6 +33,10 @@
def setUpClass(cls):
cls.isolated_creds = []
+ if not cls.config.service_available.cinder:
+ skip_msg = ("%s skipped as Cinder is not available" % cls.__name__)
+ raise cls.skipException(skip_msg)
+
if cls.config.compute.allow_tenant_isolation:
creds = cls._get_isolated_creds()
username, tenant_name, password = creds
@@ -55,17 +58,11 @@
cls.snapshots = []
cls.volumes = []
- skip_msg = ("%s skipped as Cinder endpoint is not available" %
- cls.__name__)
- try:
- cls.volumes_client.keystone_auth(cls.os.username,
- cls.os.password,
- cls.os.auth_url,
- cls.volumes_client.service,
- cls.os.tenant_name)
- except exceptions.EndpointNotFound:
- cls.clear_isolated_creds()
- raise cls.skipException(skip_msg)
+ cls.volumes_client.keystone_auth(cls.os.username,
+ cls.os.password,
+ cls.os.auth_url,
+ cls.volumes_client.service,
+ cls.os.tenant_name)
@classmethod
def _get_identity_admin_client(cls):
diff --git a/tempest/cli/simple_read_only/test_compute.py b/tempest/cli/simple_read_only/test_compute.py
index af91968..5dadbeb 100644
--- a/tempest/cli/simple_read_only/test_compute.py
+++ b/tempest/cli/simple_read_only/test_compute.py
@@ -22,7 +22,6 @@
import tempest.cli
from tempest.common import log as logging
-from tempest import config
CONF = cfg.CONF
@@ -69,7 +68,7 @@
def test_admin_credentials(self):
self.nova('credentials')
- @testtools.skipIf(config.TempestConfig().network.neutron_available,
+ @testtools.skipIf(CONF.service_available.neutron,
"Neutron does not provide this feature")
def test_admin_dns_domains(self):
self.nova('dns-domains')
diff --git a/tempest/config.py b/tempest/config.py
index 6e6488b..d9de205 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -302,9 +302,6 @@
default="",
help="Id of the public router that provides external "
"connectivity"),
- cfg.BoolOpt('neutron_available',
- default=False,
- help="Whether or not neutron is expected to be available"),
]
@@ -394,9 +391,6 @@
cfg.IntOpt('build_timeout',
default=300,
help="Timeout in seconds to wait for a stack to build."),
- cfg.BoolOpt('heat_available',
- default=False,
- help="Whether or not Heat is expected to be available"),
cfg.StrOpt('instance_type',
default='m1.micro',
help="Instance type for tests. Needs to be big enough for a "
@@ -538,6 +532,37 @@
conf.register_opt(opt, group='scenario')
+service_available_group = cfg.OptGroup(name="service_available",
+ title="Available OpenStack Services")
+
+ServiceAvailableGroup = [
+ cfg.BoolOpt('cinder',
+ default=True,
+ help="Whether or not cinder is expected to be available"),
+ cfg.BoolOpt('neutron',
+ default=False,
+ help="Whether or not neutron is expected to be available"),
+ cfg.BoolOpt('glance',
+ default=True,
+ help="Whether or not glance is expected to be available"),
+ cfg.BoolOpt('swift',
+ default=True,
+ help="Whether or not swift is expected to be available"),
+ cfg.BoolOpt('nova',
+ default=True,
+ help="Whether or not nova is expected to be available"),
+ cfg.BoolOpt('heat',
+ default=False,
+ help="Whether or not Heat is expected to be available"),
+]
+
+
+def register_service_available_opts(conf):
+ conf.register_group(scenario_group)
+ for opt in ServiceAvailableGroup:
+ conf.register_opt(opt, group='service_available')
+
+
@singleton
class TempestConfig:
"""Provides OpenStack configuration information."""
@@ -588,6 +613,7 @@
register_compute_admin_opts(cfg.CONF)
register_stress_opts(cfg.CONF)
register_scenario_opts(cfg.CONF)
+ register_service_available_opts(cfg.CONF)
self.compute = cfg.CONF.compute
self.whitebox = cfg.CONF.whitebox
self.identity = cfg.CONF.identity
@@ -600,6 +626,7 @@
self.compute_admin = cfg.CONF['compute-admin']
self.stress = cfg.CONF.stress
self.scenario = cfg.CONF.scenario
+ self.service_available = cfg.CONF.service_available
if not self.compute_admin.username:
self.compute_admin.username = self.identity.admin_username
self.compute_admin.password = self.identity.admin_password
diff --git a/tempest/scenario/manager.py b/tempest/scenario/manager.py
index f968411..fcd5d0e 100644
--- a/tempest/scenario/manager.py
+++ b/tempest/scenario/manager.py
@@ -223,7 +223,7 @@
@classmethod
def check_preconditions(cls):
- if (cls.config.network.neutron_available):
+ if (cls.config.service_available.neutron):
cls.enabled = True
#verify that neutron_available is telling the truth
try:
diff --git a/tempest/stress/driver.py b/tempest/stress/driver.py
index 785da7d..b04a93a 100644
--- a/tempest/stress/driver.py
+++ b/tempest/stress/driver.py
@@ -98,7 +98,7 @@
return getattr(importlib.import_module(module_part), obj_name)
-def stress_openstack(tests, duration):
+def stress_openstack(tests, duration, max_runs=None):
"""
Workload driver. Executes an action function against a nova-cluster.
@@ -116,7 +116,7 @@
manager = admin_manager
else:
manager = clients.Manager()
- for _ in xrange(test.get('threads', 1)):
+ for p_number in xrange(test.get('threads', 1)):
if test.get('use_isolated_tenants', False):
username = rand_name("stress_user")
tenant_name = rand_name("stress_tenant")
@@ -132,24 +132,46 @@
tenant_name=tenant_name)
test_obj = get_action_object(test['action'])
- test_run = test_obj(manager, logger)
+ test_run = test_obj(manager, logger, max_runs)
kwargs = test.get('kwargs', {})
test_run.setUp(**dict(kwargs.iteritems()))
logger.debug("calling Target Object %s" %
test_run.__class__.__name__)
- p = multiprocessing.Process(target=test_run.execute,
- args=())
- processes.append(p)
+ mp_manager = multiprocessing.Manager()
+ shared_statistic = mp_manager.dict()
+ shared_statistic['runs'] = 0
+ shared_statistic['fails'] = 0
+
+ p = multiprocessing.Process(target=test_run.execute,
+ args=(shared_statistic,))
+
+ process = {'process': p,
+ 'p_number': p_number,
+ 'action': test['action'],
+ 'statistic': shared_statistic}
+
+ processes.append(process)
p.start()
end_time = time.time() + duration
had_errors = False
while True:
- remaining = end_time - time.time()
- if remaining <= 0:
- break
+ if max_runs is None:
+ remaining = end_time - time.time()
+ if remaining <= 0:
+ break
+ else:
+ remaining = log_check_interval
+ all_proc_term = True
+ for process in processes:
+ if process['process'].is_alive():
+ all_proc_term = False
+ break
+ if all_proc_term:
+ break
+
time.sleep(min(remaining, log_check_interval))
if not logfiles:
continue
@@ -158,9 +180,28 @@
had_errors = True
break
- for p in processes:
- p.terminate()
- p.join()
+ for process in processes:
+ if process['process'].is_alive():
+ process['process'].terminate()
+ process['process'].join()
+
+ sum_fails = 0
+ sum_runs = 0
+
+ logger.info("Statistics (per process):")
+ for process in processes:
+ if process['statistic']['fails'] > 0:
+ had_errors = True
+ sum_runs += process['statistic']['runs']
+ sum_fails += process['statistic']['fails']
+ logger.info(" Process %d (%s): Run %d actions (%d failed)" %
+ (process['p_number'],
+ process['action'],
+ process['statistic']['runs'],
+ process['statistic']['fails']))
+ logger.info("Summary:")
+ logger.info("Run %d actions (%d failed)" %
+ (sum_runs, sum_fails))
if not had_errors:
logger.info("cleaning up")
diff --git a/tempest/stress/run_stress.py b/tempest/stress/run_stress.py
index 109f334..9ec1527 100755
--- a/tempest/stress/run_stress.py
+++ b/tempest/stress/run_stress.py
@@ -26,9 +26,9 @@
tests = json.load(open(ns.tests, 'r'))
if ns.serial:
for test in tests:
- driver.stress_openstack([test], ns.duration)
+ driver.stress_openstack([test], ns.duration, ns.number)
else:
- driver.stress_openstack(tests, ns.duration)
+ driver.stress_openstack(tests, ns.duration, ns.number)
parser = argparse.ArgumentParser(description='Run stress tests. ')
@@ -36,5 +36,7 @@
help="Duration of test in secs.")
parser.add_argument('-s', '--serial', action='store_true',
help="Trigger running tests serially.")
+parser.add_argument('-n', '--number', type=int,
+ help="How often an action is executed for each process.")
parser.add_argument('tests', help="Name of the file with test description.")
main(parser.parse_args())
diff --git a/tempest/stress/stressaction.py b/tempest/stress/stressaction.py
index f45ef17..77ddd1c 100644
--- a/tempest/stress/stressaction.py
+++ b/tempest/stress/stressaction.py
@@ -20,10 +20,10 @@
class StressAction(object):
- def __init__(self, manager, logger):
+ def __init__(self, manager, logger, max_runs=None):
self.manager = manager
self.logger = logger
- self.runs = 0
+ self.max_runs = max_runs
def _shutdown_handler(self, signal, frame):
self.tearDown()
@@ -45,7 +45,7 @@
"""
self.logger.debug("tearDown")
- def execute(self):
+ def execute(self, shared_statistic):
"""This is the main execution entry point called
by the driver. We register a signal handler to
allow us to gracefull tearDown, and then exit.
@@ -53,9 +53,16 @@
"""
signal.signal(signal.SIGHUP, self._shutdown_handler)
signal.signal(signal.SIGTERM, self._shutdown_handler)
- while True:
- self.run()
- self.runs = self.runs + 1
+
+ while self.max_runs is None or (shared_statistic['runs'] <
+ self.max_runs):
+ try:
+ self.run()
+ except Exception:
+ shared_statistic['fails'] += 1
+ self.logger.exception("Failure in run")
+ finally:
+ shared_statistic['runs'] += 1
def run(self):
"""This method is where the stress test code runs."""
diff --git a/test-requirements.txt b/test-requirements.txt
index 2185997..693daff 100644
--- a/test-requirements.txt
+++ b/test-requirements.txt
@@ -2,7 +2,6 @@
pep8==1.4.5
pyflakes==0.7.2
flake8==2.0
-hacking>=0.5.3,<0.6
+hacking>=0.5.6,<0.7
# needed for doc build
sphinx>=1.1.2
-
diff --git a/tools/colorizer.py b/tools/colorizer.py
new file mode 100755
index 0000000..76a3bd3
--- /dev/null
+++ b/tools/colorizer.py
@@ -0,0 +1,333 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2013, Nebula, Inc.
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Colorizer Code is borrowed from Twisted:
+# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""Display a subunit stream through a colorized unittest test runner."""
+
+import heapq
+import subunit
+import sys
+import unittest
+
+import testtools
+
+
+class _AnsiColorizer(object):
+ """
+ A colorizer is an object that loosely wraps around a stream, allowing
+ callers to write text to the stream in a particular color.
+
+ Colorizer classes must implement C{supported()} and C{write(text, color)}.
+ """
+ _colors = dict(black=30, red=31, green=32, yellow=33,
+ blue=34, magenta=35, cyan=36, white=37)
+
+ def __init__(self, stream):
+ self.stream = stream
+
+ def supported(cls, stream=sys.stdout):
+ """
+ A class method that returns True if the current platform supports
+ coloring terminal output using this method. Returns False otherwise.
+ """
+ if not stream.isatty():
+ return False # auto color only on TTYs
+ try:
+ import curses
+ except ImportError:
+ return False
+ else:
+ try:
+ try:
+ return curses.tigetnum("colors") > 2
+ except curses.error:
+ curses.setupterm()
+ return curses.tigetnum("colors") > 2
+ except Exception:
+ # guess false in case of error
+ return False
+ supported = classmethod(supported)
+
+ def write(self, text, color):
+ """
+ Write the given text to the stream in the given color.
+
+ @param text: Text to be written to the stream.
+
+ @param color: A string label for a color. e.g. 'red', 'white'.
+ """
+ color = self._colors[color]
+ self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text))
+
+
+class _Win32Colorizer(object):
+ """
+ See _AnsiColorizer docstring.
+ """
+ def __init__(self, stream):
+ import win32console
+ red, green, blue, bold = (win32console.FOREGROUND_RED,
+ win32console.FOREGROUND_GREEN,
+ win32console.FOREGROUND_BLUE,
+ win32console.FOREGROUND_INTENSITY)
+ self.stream = stream
+ self.screenBuffer = win32console.GetStdHandle(
+ win32console.STD_OUT_HANDLE)
+ self._colors = {'normal': red | green | blue,
+ 'red': red | bold,
+ 'green': green | bold,
+ 'blue': blue | bold,
+ 'yellow': red | green | bold,
+ 'magenta': red | blue | bold,
+ 'cyan': green | blue | bold,
+ 'white': red | green | blue | bold}
+
+ def supported(cls, stream=sys.stdout):
+ try:
+ import win32console
+ screenBuffer = win32console.GetStdHandle(
+ win32console.STD_OUT_HANDLE)
+ except ImportError:
+ return False
+ import pywintypes
+ try:
+ screenBuffer.SetConsoleTextAttribute(
+ win32console.FOREGROUND_RED |
+ win32console.FOREGROUND_GREEN |
+ win32console.FOREGROUND_BLUE)
+ except pywintypes.error:
+ return False
+ else:
+ return True
+ supported = classmethod(supported)
+
+ def write(self, text, color):
+ color = self._colors[color]
+ self.screenBuffer.SetConsoleTextAttribute(color)
+ self.stream.write(text)
+ self.screenBuffer.SetConsoleTextAttribute(self._colors['normal'])
+
+
+class _NullColorizer(object):
+ """
+ See _AnsiColorizer docstring.
+ """
+ def __init__(self, stream):
+ self.stream = stream
+
+ def supported(cls, stream=sys.stdout):
+ return True
+ supported = classmethod(supported)
+
+ def write(self, text, color):
+ self.stream.write(text)
+
+
+def get_elapsed_time_color(elapsed_time):
+ if elapsed_time > 1.0:
+ return 'red'
+ elif elapsed_time > 0.25:
+ return 'yellow'
+ else:
+ return 'green'
+
+
+class NovaTestResult(testtools.TestResult):
+ def __init__(self, stream, descriptions, verbosity):
+ super(NovaTestResult, self).__init__()
+ self.stream = stream
+ self.showAll = verbosity > 1
+ self.num_slow_tests = 10
+ self.slow_tests = [] # this is a fixed-sized heap
+ self.colorizer = None
+ # NOTE(vish): reset stdout for the terminal check
+ stdout = sys.stdout
+ sys.stdout = sys.__stdout__
+ for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
+ if colorizer.supported():
+ self.colorizer = colorizer(self.stream)
+ break
+ sys.stdout = stdout
+ self.start_time = None
+ self.last_time = {}
+ self.results = {}
+ self.last_written = None
+
+ def _writeElapsedTime(self, elapsed):
+ color = get_elapsed_time_color(elapsed)
+ self.colorizer.write(" %.2f" % elapsed, color)
+
+ def _addResult(self, test, *args):
+ try:
+ name = test.id()
+ except AttributeError:
+ name = 'Unknown.unknown'
+ test_class, test_name = name.rsplit('.', 1)
+
+ elapsed = (self._now() - self.start_time).total_seconds()
+ item = (elapsed, test_class, test_name)
+ if len(self.slow_tests) >= self.num_slow_tests:
+ heapq.heappushpop(self.slow_tests, item)
+ else:
+ heapq.heappush(self.slow_tests, item)
+
+ self.results.setdefault(test_class, [])
+ self.results[test_class].append((test_name, elapsed) + args)
+ self.last_time[test_class] = self._now()
+ self.writeTests()
+
+ def _writeResult(self, test_name, elapsed, long_result, color,
+ short_result, success):
+ if self.showAll:
+ self.stream.write(' %s' % str(test_name).ljust(66))
+ self.colorizer.write(long_result, color)
+ if success:
+ self._writeElapsedTime(elapsed)
+ self.stream.writeln()
+ else:
+ self.colorizer.write(short_result, color)
+
+ def addSuccess(self, test):
+ super(NovaTestResult, self).addSuccess(test)
+ self._addResult(test, 'OK', 'green', '.', True)
+
+ def addFailure(self, test, err):
+ if test.id() == 'process-returncode':
+ return
+ super(NovaTestResult, self).addFailure(test, err)
+ self._addResult(test, 'FAIL', 'red', 'F', False)
+
+ def addError(self, test, err):
+ super(NovaTestResult, self).addFailure(test, err)
+ self._addResult(test, 'ERROR', 'red', 'E', False)
+
+ def addSkip(self, test, reason=None, details=None):
+ super(NovaTestResult, self).addSkip(test, reason, details)
+ self._addResult(test, 'SKIP', 'blue', 'S', True)
+
+ def startTest(self, test):
+ self.start_time = self._now()
+ super(NovaTestResult, self).startTest(test)
+
+ def writeTestCase(self, cls):
+ if not self.results.get(cls):
+ return
+ if cls != self.last_written:
+ self.colorizer.write(cls, 'white')
+ self.stream.writeln()
+ for result in self.results[cls]:
+ self._writeResult(*result)
+ del self.results[cls]
+ self.stream.flush()
+ self.last_written = cls
+
+ def writeTests(self):
+ time = self.last_time.get(self.last_written, self._now())
+ if not self.last_written or (self._now() - time).total_seconds() > 2.0:
+ diff = 3.0
+ while diff > 2.0:
+ classes = self.results.keys()
+ oldest = min(classes, key=lambda x: self.last_time[x])
+ diff = (self._now() - self.last_time[oldest]).total_seconds()
+ self.writeTestCase(oldest)
+ else:
+ self.writeTestCase(self.last_written)
+
+ def done(self):
+ self.stopTestRun()
+
+ def stopTestRun(self):
+ for cls in list(self.results.iterkeys()):
+ self.writeTestCase(cls)
+ self.stream.writeln()
+ self.writeSlowTests()
+
+ def writeSlowTests(self):
+ # Pare out 'fast' tests
+ slow_tests = [item for item in self.slow_tests
+ if get_elapsed_time_color(item[0]) != 'green']
+ if slow_tests:
+ slow_total_time = sum(item[0] for item in slow_tests)
+ slow = ("Slowest %i tests took %.2f secs:"
+ % (len(slow_tests), slow_total_time))
+ self.colorizer.write(slow, 'yellow')
+ self.stream.writeln()
+ last_cls = None
+ # sort by name
+ for elapsed, cls, name in sorted(slow_tests,
+ key=lambda x: x[1] + x[2]):
+ if cls != last_cls:
+ self.colorizer.write(cls, 'white')
+ self.stream.writeln()
+ last_cls = cls
+ self.stream.write(' %s' % str(name).ljust(68))
+ self._writeElapsedTime(elapsed)
+ self.stream.writeln()
+
+ def printErrors(self):
+ if self.showAll:
+ self.stream.writeln()
+ self.printErrorList('ERROR', self.errors)
+ self.printErrorList('FAIL', self.failures)
+
+ def printErrorList(self, flavor, errors):
+ for test, err in errors:
+ self.colorizer.write("=" * 70, 'red')
+ self.stream.writeln()
+ self.colorizer.write(flavor, 'red')
+ self.stream.writeln(": %s" % test.id())
+ self.colorizer.write("-" * 70, 'red')
+ self.stream.writeln()
+ self.stream.writeln("%s" % err)
+
+
+test = subunit.ProtocolTestCase(sys.stdin, passthrough=None)
+
+if sys.version_info[0:2] <= (2, 6):
+ runner = unittest.TextTestRunner(verbosity=2)
+else:
+ runner = unittest.TextTestRunner(verbosity=2, resultclass=NovaTestResult)
+
+if runner.run(test).wasSuccessful():
+ exit_code = 0
+else:
+ exit_code = 1
+sys.exit(exit_code)
diff --git a/tools/pretty_tox.sh b/tools/pretty_tox.sh
new file mode 100755
index 0000000..a5a6076
--- /dev/null
+++ b/tools/pretty_tox.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+
+TESTRARGS=$1
+python setup.py testr --slowest --testr-args="--subunit $TESTRARGS" | subunit2pyunit
diff --git a/tox.ini b/tox.ini
index 04b845a..eb1ef4b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,7 +30,7 @@
sitepackages = True
setenv = VIRTUAL_ENV={envdir}
commands =
- python setup.py testr --slowest --testr-args='tempest.api tempest.scenario tempest.thirdparty tempest.cli'
+ sh tools/pretty_tox.sh 'tempest.api tempest.scenario tempest.thirdparty tempest.cli'
[testenv:smoke]
sitepackages = True
@@ -78,6 +78,7 @@
local-check-factory = tempest.hacking.checks.factory
[flake8]
+# E125 is a won't fix until https://github.com/jcrocholl/pep8/issues/126 is resolved. For further detail see https://review.openstack.org/#/c/36788/
ignore = E125,H302,H404
show-source = True
exclude = .git,.venv,.tox,dist,doc,openstack,*egg