Merge "Clean up existing instances when not using tenant isolation"
diff --git a/HACKING.rst b/HACKING.rst
index 499b436..7871f60 100644
--- a/HACKING.rst
+++ b/HACKING.rst
@@ -8,16 +8,6 @@
Tempest Specific Commandments
------------------------------
-[T101] If a test is broken because of a bug it is appropriate to skip the test until
-bug has been fixed. However, the skip message should be formatted so that
-Tempest's skip tracking tool can watch the bug status. The skip message should
-contain the string 'Bug' immediately followed by a space. Then the bug number
-should be included in the message '#' in front of the number.
-
-Example::
-
- @testtools.skip("Skipped until the Bug #980688 is resolved")
-
- [T102] Cannot import OpenStack python clients in tempest/api tests
- [T103] tempest/tests is deprecated
- [T104] Scenario tests require a services decorator
@@ -115,6 +105,19 @@
in tempest.api.compute would require a service tag for those services, however
they do not need to be tagged as compute.
+Test skips because of Known Bugs
+--------------------------------
+
+If a test is broken because of a bug it is appropriate to skip the test until
+bug has been fixed. You should use the skip_because decorator so that
+Tempest's skip tracking tool can watch the bug status.
+
+Example::
+
+ @skip_because(bug="980688")
+ def test_this_and_that(self):
+ ...
+
Guidelines
----------
- Do not submit changesets with only testcases which are skipped as
diff --git a/tempest/api/compute/admin/test_flavors.py b/tempest/api/compute/admin/test_flavors.py
index 3ef75ba..004268e 100644
--- a/tempest/api/compute/admin/test_flavors.py
+++ b/tempest/api/compute/admin/test_flavors.py
@@ -15,14 +15,13 @@
# License for the specific language governing permissions and limitations
# under the License.
-import testtools
-
from tempest.api import compute
from tempest.api.compute import base
from tempest.common.utils.data_utils import rand_int_id
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class FlavorsAdminTestJSON(base.BaseComputeAdminTest):
@@ -195,7 +194,7 @@
flag = True
self.assertTrue(flag)
- @testtools.skip("Skipped until the Bug #1209101 is resolved")
+ @skip_because(bug="1209101")
@attr(type='gate')
def test_list_non_public_flavor(self):
# Create a flavor with os-flavor-access:is_public false should
diff --git a/tempest/api/compute/images/test_images_oneserver.py b/tempest/api/compute/images/test_images_oneserver.py
index 7df9010..800b2de 100644
--- a/tempest/api/compute/images/test_images_oneserver.py
+++ b/tempest/api/compute/images/test_images_oneserver.py
@@ -24,6 +24,7 @@
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class ImagesOneServerTestJSON(base.BaseComputeTest):
@@ -64,7 +65,7 @@
cls.alt_manager = clients.AltManager()
cls.alt_client = cls.alt_manager.images_client
- @testtools.skip("Skipped until the Bug #1006725 is resolved.")
+ @skip_because(bug="1006725")
@attr(type=['negative', 'gate'])
def test_create_image_specify_multibyte_character_image_name(self):
# Return an error if the image name has multi-byte characters
diff --git a/tempest/api/compute/security_groups/test_security_group_rules.py b/tempest/api/compute/security_groups/test_security_group_rules.py
index 61db61d..cbc0080 100644
--- a/tempest/api/compute/security_groups/test_security_group_rules.py
+++ b/tempest/api/compute/security_groups/test_security_group_rules.py
@@ -15,13 +15,12 @@
# License for the specific language governing permissions and limitations
# under the License.
-import testtools
-
from tempest.api.compute import base
from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class SecurityGroupRulesTestJSON(base.BaseComputeTest):
@@ -94,8 +93,8 @@
self.addCleanup(self.client.delete_security_group_rule, rule['id'])
self.assertEqual(200, resp.status)
- @testtools.skipIf(config.TempestConfig().service_available.neutron,
- "Skipped until the Bug #1182384 is resolved")
+ @skip_because(bug="1182384",
+ condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_rules_create_with_invalid_id(self):
# Negative test: Creation of Security Group rule should FAIL
@@ -186,8 +185,8 @@
self.client.create_security_group_rule,
secgroup_id, ip_protocol, from_port, to_port)
- @testtools.skipIf(config.TempestConfig().service_available.neutron,
- "Skipped until the Bug #1182384 is resolved")
+ @skip_because(bug="1182384",
+ condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_rules_delete_with_invalid_id(self):
# Negative test: Deletion of Security Group rule should be FAIL
diff --git a/tempest/api/compute/security_groups/test_security_groups.py b/tempest/api/compute/security_groups/test_security_groups.py
index 5cca3b2..fba2f53 100644
--- a/tempest/api/compute/security_groups/test_security_groups.py
+++ b/tempest/api/compute/security_groups/test_security_groups.py
@@ -22,6 +22,7 @@
from tempest import config
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class SecurityGroupsTestJSON(base.BaseComputeTest):
@@ -107,8 +108,8 @@
"The fetched Security Group is different "
"from the created Group")
- @testtools.skipIf(config.TempestConfig().service_available.neutron,
- "Skipped until the Bug #1182384 is resolved")
+ @skip_because(bug="1182384",
+ condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_get_nonexistant_group(self):
# Negative test:Should not be able to GET the details
@@ -125,8 +126,8 @@
self.assertRaises(exceptions.NotFound, self.client.get_security_group,
non_exist_id)
- @testtools.skipIf(config.TempestConfig().service_available.neutron,
- "Skipped until the Bug #1161411 is resolved")
+ @skip_because(bug="1161411",
+ condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_create_with_invalid_group_name(self):
# Negative test: Security Group should not be created with group name
@@ -145,8 +146,8 @@
self.client.create_security_group, s_name,
s_description)
- @testtools.skipIf(config.TempestConfig().service_available.neutron,
- "Skipped until the Bug #1161411 is resolved")
+ @skip_because(bug="1161411",
+ condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_security_group_create_with_invalid_group_description(self):
# Negative test:Security Group should not be created with description
@@ -197,8 +198,8 @@
self.client.delete_security_group,
default_security_group_id)
- @testtools.skipIf(config.TempestConfig().service_available.neutron,
- "Skipped until the Bug #1182384 is resolved")
+ @skip_because(bug="1182384",
+ condition=config.TempestConfig().service_available.neutron)
@attr(type=['negative', 'gate'])
def test_delete_nonexistant_security_group(self):
# Negative test:Deletion of a non-existent Security Group should Fail
diff --git a/tempest/api/compute/servers/test_list_server_filters.py b/tempest/api/compute/servers/test_list_server_filters.py
index a56bdf3..c469827 100644
--- a/tempest/api/compute/servers/test_list_server_filters.py
+++ b/tempest/api/compute/servers/test_list_server_filters.py
@@ -15,14 +15,13 @@
# License for the specific language governing permissions and limitations
# under the License.
-import testtools
-
from tempest.api.compute import base
from tempest.api import utils
from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class ListServerFiltersTestJSON(base.BaseComputeTest):
@@ -205,7 +204,7 @@
self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers))
self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers))
- @testtools.skip('Skipped until the Bug #1170718 is resolved.')
+ @skip_because(bug="1170718")
@attr(type='gate')
def test_list_servers_filtered_by_ip(self):
# Filter servers by ip
@@ -219,8 +218,8 @@
self.assertNotIn(self.s2_name, map(lambda x: x['name'], servers))
self.assertNotIn(self.s3_name, map(lambda x: x['name'], servers))
- @testtools.skipIf(config.TempestConfig().service_available.neutron,
- "Skipped until the Bug #1182883 is resolved")
+ @skip_because(bug="1182883",
+ condition=config.TempestConfig().service_available.neutron)
@attr(type='gate')
def test_list_servers_filtered_by_ip_regex(self):
# Filter servers by regex ip
diff --git a/tempest/api/compute/servers/test_server_actions.py b/tempest/api/compute/servers/test_server_actions.py
index 0d5a8fa..e9defe5 100644
--- a/tempest/api/compute/servers/test_server_actions.py
+++ b/tempest/api/compute/servers/test_server_actions.py
@@ -27,6 +27,7 @@
import tempest.config
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class ServerActionsTestJSON(base.BaseComputeTest):
@@ -86,7 +87,7 @@
new_boot_time = linux_client.get_boot_time()
self.assertGreater(new_boot_time, boot_time)
- @testtools.skip('Skipped until the Bug #1014647 is resolved.')
+ @skip_because(bug="1014647")
@attr(type='smoke')
def test_reboot_server_soft(self):
# The server should be signaled to reboot gracefully
@@ -250,7 +251,7 @@
self.servers_client.get_console_output,
'!@#$%^&*()', 10)
- @testtools.skip('Skipped until the Bug #1014683 is resolved.')
+ @skip_because(bug="1014683")
@attr(type='gate')
def test_get_console_output_server_id_in_reboot_status(self):
# Positive test:Should be able to GET the console output
diff --git a/tempest/api/compute/servers/test_virtual_interfaces.py b/tempest/api/compute/servers/test_virtual_interfaces.py
index b743a85..2c7ff32 100644
--- a/tempest/api/compute/servers/test_virtual_interfaces.py
+++ b/tempest/api/compute/servers/test_virtual_interfaces.py
@@ -16,13 +16,13 @@
# under the License.
import netaddr
-import testtools
from tempest.api.compute import base
from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class VirtualInterfacesTestJSON(base.BaseComputeTest):
@@ -37,8 +37,8 @@
resp, server = cls.create_server(wait_until='ACTIVE')
cls.server_id = server['id']
- @testtools.skipIf(CONF.service_available.neutron, "Not implemented by " +
- "Neutron. Skipped until the Bug #1183436 is resolved.")
+ @skip_because(bug="1183436",
+ condition=CONF.service_available.neutron)
@attr(type='gate')
def test_list_virtual_interfaces(self):
# Positive test:Should be able to GET the virtual interfaces list
diff --git a/tempest/api/object_storage/test_container_sync.py b/tempest/api/object_storage/test_container_sync.py
index a43b2b5..ff9f7bf 100644
--- a/tempest/api/object_storage/test_container_sync.py
+++ b/tempest/api/object_storage/test_container_sync.py
@@ -17,11 +17,10 @@
import time
-import testtools
-
from tempest.api.object_storage import base
from tempest.common.utils.data_utils import rand_name
from tempest.test import attr
+from tempest.test import skip_because
class ContainerSyncTest(base.BaseObjectTest):
@@ -52,7 +51,7 @@
cls.delete_containers(cls.containers, client[0], client[1])
super(ContainerSyncTest, cls).tearDownClass()
- @testtools.skip('Skipped until the Bug #1093743 is resolved.')
+ @skip_because(bug="1093743")
@attr(type='gate')
def test_container_synchronization(self):
# container to container synchronization
diff --git a/tempest/api/object_storage/test_object_expiry.py b/tempest/api/object_storage/test_object_expiry.py
index db38401..cb52d88 100644
--- a/tempest/api/object_storage/test_object_expiry.py
+++ b/tempest/api/object_storage/test_object_expiry.py
@@ -17,13 +17,12 @@
import time
-import testtools
-
from tempest.api.object_storage import base
from tempest.common.utils.data_utils import arbitrary_string
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
from tempest.test import attr
+from tempest.test import skip_because
class ObjectExpiryTest(base.BaseObjectTest):
@@ -43,7 +42,7 @@
cls.delete_containers([cls.container_name])
super(ObjectExpiryTest, cls).tearDownClass()
- @testtools.skip('Skipped until the Bug #1069849 is resolved.')
+ @skip_because(bug="1069849")
@attr(type='gate')
def test_get_object_after_expiry_time(self):
# TODO(harika-vakadi): similar test case has to be created for
diff --git a/tempest/cli/simple_read_only/test_keystone.py b/tempest/cli/simple_read_only/test_keystone.py
index 4c1c27f..a7e7147 100644
--- a/tempest/cli/simple_read_only/test_keystone.py
+++ b/tempest/cli/simple_read_only/test_keystone.py
@@ -50,7 +50,10 @@
self.assertTrue(svc['__label'].startswith('Service:'),
msg=('Invalid beginning of service block: '
'%s' % svc['__label']))
- self.assertIn('id', svc.keys())
+ # check that region and publicURL exists. One might also
+ # check for adminURL and internalURL. id seems to be optional
+ # and is missing in the catalog backend
+ self.assertIn('publicURL', svc.keys())
self.assertIn('region', svc.keys())
def test_admin_endpoint_list(self):
diff --git a/tempest/hacking/checks.py b/tempest/hacking/checks.py
index aa97211..4c1c107 100644
--- a/tempest/hacking/checks.py
+++ b/tempest/hacking/checks.py
@@ -19,28 +19,11 @@
PYTHON_CLIENTS = ['cinder', 'glance', 'keystone', 'nova', 'swift', 'neutron']
-SKIP_DECORATOR_RE = re.compile(r'\s*@testtools.skip\((.*)\)')
-SKIP_STR_RE = re.compile(r'.*Bug #\d+.*')
PYTHON_CLIENT_RE = re.compile('import (%s)client' % '|'.join(PYTHON_CLIENTS))
TEST_DEFINITION = re.compile(r'^\s*def test.*')
SCENARIO_DECORATOR = re.compile(r'\s*@.*services\(')
-def skip_bugs(physical_line):
- """Check skip lines for proper bug entries
-
- T101: skips must contain "Bug #<bug_number>"
- """
-
- res = SKIP_DECORATOR_RE.match(physical_line)
- if res:
- content = res.group(1)
- res = SKIP_STR_RE.match(content)
- if not res:
- return (physical_line.find(content),
- 'T101: skips must contain "Bug #<bug_number>"')
-
-
def import_no_clients_in_api(physical_line, filename):
"""Check for client imports from tempest/api tests
@@ -70,6 +53,5 @@
def factory(register):
- register(skip_bugs)
register(import_no_clients_in_api)
register(scenario_tests_need_service_tags)
diff --git a/tempest/test.py b/tempest/test.py
index 6acb1c9..0c7c916 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -103,6 +103,21 @@
return decorator
+def skip_because(*args, **kwargs):
+ """A decorator useful to skip tests hitting known bugs
+
+ @param bug: bug number causing the test to skip
+ @param condition: optional condition to be True for the skip to have place
+ """
+ def decorator(f):
+ if "bug" in kwargs:
+ if "condition" not in kwargs or kwargs["condition"] is True:
+ msg = "Skipped until Bug: %s is resolved." % kwargs["bug"]
+ raise testtools.TestCase.skipException(msg)
+ return f
+ return decorator
+
+
# there is a mis-match between nose and testtools for older pythons.
# testtools will set skipException to be either
# unittest.case.SkipTest, unittest2.case.SkipTest or an internal skip
diff --git a/tempest/thirdparty/boto/test_ec2_instance_run.py b/tempest/thirdparty/boto/test_ec2_instance_run.py
index bce544a..0f455e1 100644
--- a/tempest/thirdparty/boto/test_ec2_instance_run.py
+++ b/tempest/thirdparty/boto/test_ec2_instance_run.py
@@ -16,7 +16,6 @@
# under the License.
from boto import exception
-import testtools
from tempest import clients
from tempest.common.utils.data_utils import rand_name
@@ -24,6 +23,7 @@
from tempest import exceptions
from tempest.openstack.common import log as logging
from tempest.test import attr
+from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
from tempest.thirdparty.boto.utils.s3 import s3_upload_dir
from tempest.thirdparty.boto.utils.wait import re_search_wait
@@ -206,8 +206,8 @@
instance.terminate()
self.cancelResourceCleanUp(rcuk)
+ @skip_because(bug="1098891")
@attr(type='smoke')
- @testtools.skip("Skipped until the Bug #1098891 is resolved")
def test_run_terminate_instance(self):
# EC2 run, terminate immediately
image_ami = self.ec2_client.get_image(self.images["ami"]
@@ -233,7 +233,7 @@
# NOTE(afazekas): doctored test case,
# with normal validation it would fail
- @testtools.skip("Skipped until the Bug #1182679 is resolved.")
+ @skip_because(bug="1182679")
@attr(type='smoke')
def test_integration_1(self):
# EC2 1. integration test (not strict)
diff --git a/tempest/thirdparty/boto/test_ec2_keys.py b/tempest/thirdparty/boto/test_ec2_keys.py
index 85a99c0..5592d8c 100644
--- a/tempest/thirdparty/boto/test_ec2_keys.py
+++ b/tempest/thirdparty/boto/test_ec2_keys.py
@@ -15,11 +15,10 @@
# License for the specific language governing permissions and limitations
# under the License.
-import testtools
-
from tempest import clients
from tempest.common.utils.data_utils import rand_name
from tempest.test import attr
+from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
@@ -47,8 +46,8 @@
self.assertTrue(compare_key_pairs(keypair,
self.client.get_key_pair(key_name)))
+ @skip_because(bug="1072318")
@attr(type='smoke')
- @testtools.skip("Skipped until the Bug #1072318 is resolved")
def test_delete_ec2_keypair(self):
# EC2 delete KeyPair
key_name = rand_name("keypair-")
diff --git a/tempest/thirdparty/boto/test_ec2_network.py b/tempest/thirdparty/boto/test_ec2_network.py
index ae8c3c2..b4949c8 100644
--- a/tempest/thirdparty/boto/test_ec2_network.py
+++ b/tempest/thirdparty/boto/test_ec2_network.py
@@ -15,10 +15,9 @@
# License for the specific language governing permissions and limitations
# under the License.
-import testtools
-
from tempest import clients
from tempest.test import attr
+from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
@@ -30,8 +29,8 @@
cls.os = clients.Manager()
cls.client = cls.os.ec2api_client
-# Note(afazekas): these tests for things duable without an instance
- @testtools.skip("Skipped until the Bug #1080406 is resolved")
+ # Note(afazekas): these tests for things duable without an instance
+ @skip_because(bug="1080406")
@attr(type='smoke')
def test_disassociate_not_associated_floating_ip(self):
# EC2 disassociate not associated floating ip
diff --git a/tempest/thirdparty/boto/test_s3_buckets.py b/tempest/thirdparty/boto/test_s3_buckets.py
index e43cbaa..1a8fbe0 100644
--- a/tempest/thirdparty/boto/test_s3_buckets.py
+++ b/tempest/thirdparty/boto/test_s3_buckets.py
@@ -15,11 +15,10 @@
# License for the specific language governing permissions and limitations
# under the License.
-import testtools
-
from tempest import clients
from tempest.common.utils.data_utils import rand_name
from tempest.test import attr
+from tempest.test import skip_because
from tempest.thirdparty.boto.test import BotoTestCase
@@ -31,7 +30,7 @@
cls.os = clients.Manager()
cls.client = cls.os.s3_client
- @testtools.skip("Skipped until the Bug #1076965 is resolved")
+ @skip_because(bug="1076965")
@attr(type='smoke')
def test_create_and_get_delete_bucket(self):
# S3 Create, get and delete bucket
diff --git a/tools/skip_tracker.py b/tools/skip_tracker.py
index c38ccdb..ffaf134 100755
--- a/tools/skip_tracker.py
+++ b/tools/skip_tracker.py
@@ -61,8 +61,8 @@
"""
Return the skip tuples in a test file
"""
- BUG_RE = re.compile(r'.*skip.*bug:*\s*\#*(\d+)', re.IGNORECASE)
- DEF_RE = re.compile(r'.*def (\w+)\(')
+ BUG_RE = re.compile(r'\s*@.*skip_because\(bug=[\'"](\d+)[\'"]')
+ DEF_RE = re.compile(r'\s*def (\w+)\(')
bug_found = False
results = []
lines = open(path, 'rb').readlines()