Merge "Ensure all image cache volumes are removed before removing the volume type"
diff --git a/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml
new file mode 100644
index 0000000..2203fd1
--- /dev/null
+++ b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml
@@ -0,0 +1,11 @@
+---
+features:
+ - |
+ New decorator ``unstable_test`` is added to ``tempest.lib.decorators``.
+ It can be used to mark some test as unstable thus it will be still run
+ by tempest but job will not fail if this test will fail. Such test will
+ be skipped in case of failure.
+ It can be used for example when there is known bug related which cause
+ irregular tests failures. Marking such test as unstable will help other
+ developers to get their job done and still run this test to get additional
+ debug data or to confirm if some potential fix really solved the issue.
diff --git a/tempest/api/identity/admin/v3/test_roles.py b/tempest/api/identity/admin/v3/test_roles.py
index b67de95..5ba4c9f 100644
--- a/tempest/api/identity/admin/v3/test_roles.py
+++ b/tempest/api/identity/admin/v3/test_roles.py
@@ -25,6 +25,10 @@
class RolesV3TestJSON(base.BaseIdentityV3AdminTest):
+ # NOTE: force_tenant_isolation is true in the base class by default but
+ # overridden to false here to allow test execution for clouds using the
+ # pre-provisioned credentials provider.
+ force_tenant_isolation = False
@classmethod
def resource_setup(cls):
diff --git a/tempest/api/volume/test_volumes_backup.py b/tempest/api/volume/test_volumes_backup.py
index c178272..6ce5d3e 100644
--- a/tempest/api/volume/test_volumes_backup.py
+++ b/tempest/api/volume/test_volumes_backup.py
@@ -50,6 +50,7 @@
'available')
return restored_volume
+ @decorators.skip_because(bug="1483434")
@testtools.skipIf(CONF.volume.storage_protocol == 'ceph',
'ceph does not support arbitrary container names')
@decorators.idempotent_id('a66eb488-8ee1-47d4-8e9f-575a095728c6')
diff --git a/tempest/lib/decorators.py b/tempest/lib/decorators.py
index 4064401..808e0fb 100644
--- a/tempest/lib/decorators.py
+++ b/tempest/lib/decorators.py
@@ -154,3 +154,45 @@
return f
return decorator
+
+
+def unstable_test(*args, **kwargs):
+ """A decorator useful to run tests hitting known bugs and skip it if fails
+
+ This decorator can be used in cases like:
+
+ * We have skipped tests with some bug and now bug is claimed to be fixed.
+ Now we want to check the test stability so we use this decorator.
+ The number of skipped cases with that bug can be counted to mark test
+ stable again.
+ * There is test which is failing often, but not always. If there is known
+ bug related to it, and someone is working on fix, this decorator can be
+ used instead of "skip_because". That will ensure that test is still run
+ so new debug data can be collected from jobs' logs but it will not make
+ life of other developers harder by forcing them to recheck jobs more
+ often.
+
+ ``bug`` must be a number for the test to skip.
+
+ :param bug: bug number causing the test to skip (launchpad or storyboard)
+ :param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
+ :raises: testtools.TestCase.skipException if test actually fails,
+ and ``bug`` is included
+ """
+ def decor(f):
+ @functools.wraps(f)
+ def inner(self, *func_args, **func_kwargs):
+ try:
+ return f(self, *func_args, **func_kwargs)
+ except Exception as e:
+ if "bug" in kwargs:
+ bug = kwargs['bug']
+ bug_type = kwargs.get('bug_type', 'launchpad')
+ bug_url = _get_bug_url(bug, bug_type)
+ msg = ("Marked as unstable and skipped because of bug: "
+ "%s, failure was: %s") % (bug_url, e)
+ raise testtools.TestCase.skipException(msg)
+ else:
+ raise e
+ return inner
+ return decor
diff --git a/tempest/tests/lib/test_decorators.py b/tempest/tests/lib/test_decorators.py
index 3e6160e..9c6cac7 100644
--- a/tempest/tests/lib/test_decorators.py
+++ b/tempest/tests/lib/test_decorators.py
@@ -13,7 +13,10 @@
# License for the specific language governing permissions and limitations
# under the License.
+import abc
+
import mock
+import six
import testtools
from tempest.lib import base as test
@@ -66,9 +69,36 @@
condition=True)
-class TestSkipBecauseDecorator(base.TestCase):
- def _test_skip_because_helper(self, expected_to_skip=True,
- **decorator_args):
+@six.add_metaclass(abc.ABCMeta)
+class BaseSkipDecoratorTests(object):
+
+ @abc.abstractmethod
+ def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+ **decorator_args):
+ return
+
+ def test_skip_launchpad_bug(self):
+ self._test_skip_helper(bug='12345')
+
+ def test_skip_storyboard_bug(self):
+ self._test_skip_helper(bug='1992', bug_type='storyboard')
+
+ def test_skip_bug_without_bug_never_skips(self):
+ """Never skip without a bug parameter."""
+ self._test_skip_helper(
+ raise_exception=False, expected_to_skip=False, condition=True)
+ self._test_skip_helper(
+ raise_exception=False, expected_to_skip=False)
+
+ def test_skip_invalid_bug_number(self):
+ """Raise InvalidParam if with an invalid bug number"""
+ self.assertRaises(lib_exc.InvalidParam, self._test_skip_helper,
+ bug='critical_bug')
+
+
+class TestSkipBecauseDecorator(base.TestCase, BaseSkipDecoratorTests):
+ def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+ **decorator_args):
class TestFoo(test.BaseTestCase):
_interface = 'json'
@@ -90,38 +120,56 @@
# assert that test_bar returned 0
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
- def test_skip_because_launchpad_bug(self):
- self._test_skip_because_helper(bug='12345')
-
def test_skip_because_launchpad_bug_and_condition_true(self):
- self._test_skip_because_helper(bug='12348', condition=True)
+ self._test_skip_helper(bug='12348', condition=True)
def test_skip_because_launchpad_bug_and_condition_false(self):
- self._test_skip_because_helper(expected_to_skip=False,
- bug='12349', condition=False)
-
- def test_skip_because_storyboard_bug(self):
- self._test_skip_because_helper(bug='1992', bug_type='storyboard')
-
- def test_skip_because_storyboard_bug_and_condition_true(self):
- self._test_skip_because_helper(bug='1992', bug_type='storyboard',
- condition=True)
+ self._test_skip_helper(expected_to_skip=False,
+ bug='12349', condition=False)
def test_skip_because_storyboard_bug_and_condition_false(self):
- self._test_skip_because_helper(expected_to_skip=False,
- bug='1992', bug_type='storyboard',
- condition=False)
+ self._test_skip_helper(expected_to_skip=False,
+ bug='1992', bug_type='storyboard',
+ condition=False)
- def test_skip_because_bug_without_bug_never_skips(self):
- """Never skip without a bug parameter."""
- self._test_skip_because_helper(expected_to_skip=False,
- condition=True)
- self._test_skip_because_helper(expected_to_skip=False)
+ def test_skip_because_storyboard_bug_and_condition_true(self):
+ self._test_skip_helper(bug='1992', bug_type='storyboard',
+ condition=True)
- def test_skip_because_invalid_bug_number(self):
- """Raise InvalidParam if with an invalid bug number"""
- self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper,
- bug='critical_bug')
+
+class TestUnstableTestDecorator(base.TestCase, BaseSkipDecoratorTests):
+
+ def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+ **decorator_args):
+ fail_test_reason = "test_bar failed"
+
+ class TestFoo(test.BaseTestCase):
+
+ @decorators.unstable_test(**decorator_args)
+ def test_bar(self):
+ if raise_exception:
+ raise Exception(fail_test_reason)
+ else:
+ return 0
+
+ t = TestFoo('test_bar')
+ if expected_to_skip:
+ e = self.assertRaises(testtools.TestCase.skipException, t.test_bar)
+ bug = decorator_args['bug']
+ bug_type = decorator_args.get('bug_type', 'launchpad')
+ self.assertRegex(
+ str(e),
+ r'Marked as unstable and skipped because of bug\: %s.*, '
+ 'failure was: %s' % (decorators._get_bug_url(bug, bug_type),
+ fail_test_reason)
+ )
+ else:
+ # assert that test_bar returned 0
+ self.assertEqual(TestFoo('test_bar').test_bar(), 0)
+
+ def test_skip_bug_given_exception_not_raised(self):
+ self._test_skip_helper(raise_exception=False, expected_to_skip=False,
+ bug='1234')
class TestIdempotentIdDecorator(base.TestCase):