Merge "Remove Whitebox tests"
diff --git a/HACKING.rst b/HACKING.rst
index 03e7dc3..5153fe1 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -20,6 +20,7 @@
- [T102] Cannot import OpenStack python clients in tempest/api tests
- [T103] tempest/tests is deprecated
+- [T104] Scenario tests require a services decorator
Test Data/Configuration
-----------------------
@@ -96,6 +97,24 @@
credentials management, testresources and so on. These facilities, MUST be able
to work even if just one ``test_method`` selected for execution.
+Service Tagging
+---------------
+Service tagging is used to specify which services are exercised by a particular
+test method. You specify the services with the tempest.test.services decorator.
+For example:
+
+@services('compute', 'image')
+
+Valid service tag names are the same as the list of directories in tempest.api
+that have tests.
+
+For scenario tests having a service tag is required. For the api tests service
+tags are only needed if the test method makes an api call (either directly or
+indirectly through another service) that differs from the parent directory
+name. For example, any test that make an api call to a service other than nova
+in tempest.api.compute would require a service tag for those services, however
+they do not need to be tagged as compute.
+
Guidelines
----------
- Do not submit changesets with only testcases which are skipped as
diff --git a/tempest/hacking/checks.py b/tempest/hacking/checks.py
index 8cfd548..aa97211 100644
--- a/tempest/hacking/checks.py
+++ b/tempest/hacking/checks.py
@@ -22,6 +22,8 @@
SKIP_DECORATOR_RE = re.compile(r'\s*@testtools.skip\((.*)\)')
SKIP_STR_RE = re.compile(r'.*Bug #\d+.*')
PYTHON_CLIENT_RE = re.compile('import (%s)client' % '|'.join(PYTHON_CLIENTS))
+TEST_DEFINITION = re.compile(r'^\s*def test.*')
+SCENARIO_DECORATOR = re.compile(r'\s*@.*services\(')
def skip_bugs(physical_line):
@@ -53,6 +55,21 @@
" in tempest/api/* tests"))
+def scenario_tests_need_service_tags(physical_line, filename,
+ previous_logical):
+ """Check that scenario tests have service tags
+
+ T104: Scenario tests require a services decorator
+ """
+
+ if 'tempest/scenario' in filename:
+ if TEST_DEFINITION.match(physical_line):
+ if not SCENARIO_DECORATOR.match(previous_logical):
+ return (physical_line.find('def'),
+ "T104: Scenario tests require a service decorator")
+
+
def factory(register):
register(skip_bugs)
register(import_no_clients_in_api)
+ register(scenario_tests_need_service_tags)
diff --git a/tempest/scenario/orchestration/test_autoscaling.py b/tempest/scenario/orchestration/test_autoscaling.py
index b31a0a7..88f2ebd 100644
--- a/tempest/scenario/orchestration/test_autoscaling.py
+++ b/tempest/scenario/orchestration/test_autoscaling.py
@@ -17,6 +17,7 @@
from tempest.scenario import manager
from tempest.test import attr
from tempest.test import call_until_true
+from tempest.test import services
class AutoScalingTest(manager.OrchestrationScenarioTest):
@@ -59,6 +60,7 @@
self.set_resource('stack', self.stack)
@attr(type='slow')
+ @services('orchestration', 'compute')
def test_scale_up_then_down(self):
self.assign_keypair()
diff --git a/tempest/scenario/test_dashboard_basic_ops.py b/tempest/scenario/test_dashboard_basic_ops.py
index 9a45572..1081a3e 100644
--- a/tempest/scenario/test_dashboard_basic_ops.py
+++ b/tempest/scenario/test_dashboard_basic_ops.py
@@ -20,6 +20,7 @@
from lxml import html
from tempest.scenario import manager
+from tempest.test import services
class TestDashboardBasicOps(manager.OfficialClientTest):
@@ -66,6 +67,7 @@
response = self.opener.open(self.config.dashboard.dashboard_url)
self.assertIn('Overview', response.read())
+ @services('dashboard')
def test_basic_scenario(self):
self.check_login_page()
self.user_login()
diff --git a/tempest/scenario/test_large_ops.py b/tempest/scenario/test_large_ops.py
index 39b1e10..33b7adc 100644
--- a/tempest/scenario/test_large_ops.py
+++ b/tempest/scenario/test_large_ops.py
@@ -18,6 +18,7 @@
from tempest.common.utils.data_utils import rand_name
from tempest.openstack.common import log as logging
from tempest.scenario import manager
+from tempest.test import services
LOG = logging.getLogger(__name__)
@@ -96,6 +97,7 @@
self.addCleanup(delete, self.servers)
self._wait_for_server_status('ACTIVE')
+ @services('compute', 'image')
def test_large_ops_scenario(self):
if self.config.scenario.large_ops_number < 1:
return
diff --git a/tempest/scenario/test_minimum_basic.py b/tempest/scenario/test_minimum_basic.py
index 5cddde2..ce4d1bd 100644
--- a/tempest/scenario/test_minimum_basic.py
+++ b/tempest/scenario/test_minimum_basic.py
@@ -18,6 +18,7 @@
from tempest.common.utils.data_utils import rand_name
from tempest.openstack.common import log as logging
from tempest.scenario import manager
+from tempest.test import services
LOG = logging.getLogger(__name__)
@@ -145,6 +146,7 @@
volume = self.volume_client.volumes.get(self.volume.id)
self.assertEqual('available', volume.status)
+ @services('compute', 'volume', 'image', 'network')
def test_minimum_basic_scenario(self):
self.glance_image_create()
self.nova_keypair_add()
diff --git a/tempest/scenario/test_network_basic_ops.py b/tempest/scenario/test_network_basic_ops.py
index 930ffae..662e919 100644
--- a/tempest/scenario/test_network_basic_ops.py
+++ b/tempest/scenario/test_network_basic_ops.py
@@ -22,6 +22,7 @@
from tempest.openstack.common import log as logging
from tempest.scenario import manager
from tempest.test import attr
+from tempest.test import services
LOG = logging.getLogger(__name__)
@@ -251,6 +252,7 @@
self._check_vm_connectivity(ip_address, ssh_login, private_key)
@attr(type='smoke')
+ @services('compute', 'network')
def test_network_basic_ops(self):
self._create_keypairs()
self._create_security_groups()
diff --git a/tempest/scenario/test_network_quotas.py b/tempest/scenario/test_network_quotas.py
index 8259feb..3268066 100644
--- a/tempest/scenario/test_network_quotas.py
+++ b/tempest/scenario/test_network_quotas.py
@@ -18,6 +18,7 @@
from neutronclient.common import exceptions as exc
from tempest.scenario.manager import NetworkScenarioTest
+from tempest.test import services
MAX_REASONABLE_ITERATIONS = 51 # more than enough. Default for port is 50.
@@ -42,6 +43,7 @@
cls.subnets = []
cls.ports = []
+ @services('network')
def test_create_network_until_quota_hit(self):
hit_limit = False
for n in xrange(MAX_REASONABLE_ITERATIONS):
@@ -56,6 +58,7 @@
break
self.assertTrue(hit_limit, "Failed: Did not hit quota limit !")
+ @services('network')
def test_create_subnet_until_quota_hit(self):
if not self.networks:
self.networks.append(
@@ -74,6 +77,7 @@
break
self.assertTrue(hit_limit, "Failed: Did not hit quota limit !")
+ @services('network')
def test_create_ports_until_quota_hit(self):
if not self.networks:
self.networks.append(
diff --git a/tempest/scenario/test_server_advanced_ops.py b/tempest/scenario/test_server_advanced_ops.py
index 8ee740e..cf72cd4 100644
--- a/tempest/scenario/test_server_advanced_ops.py
+++ b/tempest/scenario/test_server_advanced_ops.py
@@ -18,6 +18,7 @@
from tempest.common.utils.data_utils import rand_name
from tempest.openstack.common import log as logging
from tempest.scenario import manager
+from tempest.test import services
LOG = logging.getLogger(__name__)
@@ -45,6 +46,7 @@
msg = "Skipping test - flavor_ref and flavor_ref_alt are identical"
raise cls.skipException(msg)
+ @services('compute')
def test_resize_server_confirm(self):
# We create an instance for use in this test
i_name = rand_name('instance')
@@ -73,6 +75,7 @@
self.status_timeout(
self.compute_client.servers, instance_id, 'ACTIVE')
+ @services('compute')
def test_server_sequence_suspend_resume(self):
# We create an instance for use in this test
i_name = rand_name('instance')
diff --git a/tempest/scenario/test_server_basic_ops.py b/tempest/scenario/test_server_basic_ops.py
index 8e14b06..04204eb 100644
--- a/tempest/scenario/test_server_basic_ops.py
+++ b/tempest/scenario/test_server_basic_ops.py
@@ -18,6 +18,7 @@
from tempest.common.utils.data_utils import rand_name
from tempest.openstack.common import log as logging
from tempest.scenario import manager
+from tempest.test import services
LOG = logging.getLogger(__name__)
@@ -100,6 +101,7 @@
instance.delete()
self.remove_resource('instance')
+ @services('compute', 'network')
def test_server_basicops(self):
self.add_keypair()
self.create_security_group()
diff --git a/tempest/scenario/test_snapshot_pattern.py b/tempest/scenario/test_snapshot_pattern.py
index 003c264..8c2cc76 100644
--- a/tempest/scenario/test_snapshot_pattern.py
+++ b/tempest/scenario/test_snapshot_pattern.py
@@ -16,6 +16,7 @@
# under the License.
from tempest.scenario import manager
+from tempest.test import services
class TestSnapshotPattern(manager.OfficialClientTest):
@@ -61,6 +62,7 @@
def _set_floating_ip_to_server(self, server, floating_ip):
server.add_floating_ip(floating_ip)
+ @services('compute', 'network', 'image')
def test_snapshot_pattern(self):
# prepare for booting a instance
self._add_keypair()
diff --git a/tempest/scenario/test_stamp_pattern.py b/tempest/scenario/test_stamp_pattern.py
index 5af4bb2..c5a4aaf 100644
--- a/tempest/scenario/test_stamp_pattern.py
+++ b/tempest/scenario/test_stamp_pattern.py
@@ -144,6 +144,7 @@
self.assertEqual(self.timestamp, got_timestamp)
@testtools.skip("Skipped until the Bug #1205344 is resolved.")
+ @tempest.test.services('compute', 'network', 'volume', 'image')
def test_stamp_pattern(self):
# prepare for booting a instance
self._add_keypair()
diff --git a/tempest/scenario/test_volume_boot_pattern.py b/tempest/scenario/test_volume_boot_pattern.py
index 09c1cb7..3572166 100644
--- a/tempest/scenario/test_volume_boot_pattern.py
+++ b/tempest/scenario/test_volume_boot_pattern.py
@@ -14,6 +14,7 @@
from tempest.common.utils.data_utils import rand_name
from tempest.scenario import manager
+from tempest.test import services
class TestVolumeBootPattern(manager.OfficialClientTest):
@@ -117,6 +118,7 @@
actual = self._get_content(ssh_client)
self.assertEqual(expected, actual)
+ @services('compute', 'volume', 'image')
def test_volume_boot_pattern(self):
keypair = self.create_keypair()
self.create_loginable_secgroup_rule()