Add unstable_test decorator
This decorator can be used to temporarily mark some tests as unstable.
This may help sometimes to debug such test which is failing often
but not always in the gate because it will still be run but will
not cause all job failure when it fails.
This may be also used to easily track in logstash how often
such test is failing by looking for describption of unstability
reason set in decorator.
Change-Id: I79ce70f479506ec2b3216747d533ff2e450685aa
Related-Bug: #1813198
diff --git a/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml
new file mode 100644
index 0000000..2203fd1
--- /dev/null
+++ b/releasenotes/notes/add-unstable_test-decorator-a73cf97d4ffcc796.yaml
@@ -0,0 +1,11 @@
+---
+features:
+ - |
+ New decorator ``unstable_test`` is added to ``tempest.lib.decorators``.
+ It can be used to mark some test as unstable thus it will be still run
+ by tempest but job will not fail if this test will fail. Such test will
+ be skipped in case of failure.
+ It can be used for example when there is known bug related which cause
+ irregular tests failures. Marking such test as unstable will help other
+ developers to get their job done and still run this test to get additional
+ debug data or to confirm if some potential fix really solved the issue.
diff --git a/tempest/lib/decorators.py b/tempest/lib/decorators.py
index b399aa0..a716fa2 100644
--- a/tempest/lib/decorators.py
+++ b/tempest/lib/decorators.py
@@ -147,3 +147,45 @@
return f
return decorator
+
+
+def unstable_test(*args, **kwargs):
+ """A decorator useful to run tests hitting known bugs and skip it if fails
+
+ This decorator can be used in cases like:
+
+ * We have skipped tests with some bug and now bug is claimed to be fixed.
+ Now we want to check the test stability so we use this decorator.
+ The number of skipped cases with that bug can be counted to mark test
+ stable again.
+ * There is test which is failing often, but not always. If there is known
+ bug related to it, and someone is working on fix, this decorator can be
+ used instead of "skip_because". That will ensure that test is still run
+ so new debug data can be collected from jobs' logs but it will not make
+ life of other developers harder by forcing them to recheck jobs more
+ often.
+
+ ``bug`` must be a number for the test to skip.
+
+ :param bug: bug number causing the test to skip (launchpad or storyboard)
+ :param bug_type: 'launchpad' or 'storyboard', default 'launchpad'
+ :raises: testtools.TestCase.skipException if test actually fails,
+ and ``bug`` is included
+ """
+ def decor(f):
+ @functools.wraps(f)
+ def inner(self, *func_args, **func_kwargs):
+ try:
+ return f(self, *func_args, **func_kwargs)
+ except Exception as e:
+ if "bug" in kwargs:
+ bug = kwargs['bug']
+ bug_type = kwargs.get('bug_type', 'launchpad')
+ bug_url = _get_bug_url(bug, bug_type)
+ msg = ("Marked as unstable and skipped because of bug: "
+ "%s, failure was: %s") % (bug_url, e)
+ raise testtools.TestCase.skipException(msg)
+ else:
+ raise e
+ return inner
+ return decor
diff --git a/tempest/tests/lib/test_decorators.py b/tempest/tests/lib/test_decorators.py
index 0b1a599..59c4e2d 100644
--- a/tempest/tests/lib/test_decorators.py
+++ b/tempest/tests/lib/test_decorators.py
@@ -13,7 +13,10 @@
# License for the specific language governing permissions and limitations
# under the License.
+import abc
+
import mock
+import six
import testtools
from tempest.lib import base as test
@@ -51,9 +54,36 @@
self._test_attr_helper(expected_attrs=['foo'], type=['foo', 'foo'])
-class TestSkipBecauseDecorator(base.TestCase):
- def _test_skip_because_helper(self, expected_to_skip=True,
- **decorator_args):
+@six.add_metaclass(abc.ABCMeta)
+class BaseSkipDecoratorTests(object):
+
+ @abc.abstractmethod
+ def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+ **decorator_args):
+ return
+
+ def test_skip_launchpad_bug(self):
+ self._test_skip_helper(bug='12345')
+
+ def test_skip_storyboard_bug(self):
+ self._test_skip_helper(bug='1992', bug_type='storyboard')
+
+ def test_skip_bug_without_bug_never_skips(self):
+ """Never skip without a bug parameter."""
+ self._test_skip_helper(
+ raise_exception=False, expected_to_skip=False, condition=True)
+ self._test_skip_helper(
+ raise_exception=False, expected_to_skip=False)
+
+ def test_skip_invalid_bug_number(self):
+ """Raise InvalidParam if with an invalid bug number"""
+ self.assertRaises(lib_exc.InvalidParam, self._test_skip_helper,
+ bug='critical_bug')
+
+
+class TestSkipBecauseDecorator(base.TestCase, BaseSkipDecoratorTests):
+ def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+ **decorator_args):
class TestFoo(test.BaseTestCase):
_interface = 'json'
@@ -75,38 +105,56 @@
# assert that test_bar returned 0
self.assertEqual(TestFoo('test_bar').test_bar(), 0)
- def test_skip_because_launchpad_bug(self):
- self._test_skip_because_helper(bug='12345')
-
def test_skip_because_launchpad_bug_and_condition_true(self):
- self._test_skip_because_helper(bug='12348', condition=True)
+ self._test_skip_helper(bug='12348', condition=True)
def test_skip_because_launchpad_bug_and_condition_false(self):
- self._test_skip_because_helper(expected_to_skip=False,
- bug='12349', condition=False)
-
- def test_skip_because_storyboard_bug(self):
- self._test_skip_because_helper(bug='1992', bug_type='storyboard')
-
- def test_skip_because_storyboard_bug_and_condition_true(self):
- self._test_skip_because_helper(bug='1992', bug_type='storyboard',
- condition=True)
+ self._test_skip_helper(expected_to_skip=False,
+ bug='12349', condition=False)
def test_skip_because_storyboard_bug_and_condition_false(self):
- self._test_skip_because_helper(expected_to_skip=False,
- bug='1992', bug_type='storyboard',
- condition=False)
+ self._test_skip_helper(expected_to_skip=False,
+ bug='1992', bug_type='storyboard',
+ condition=False)
- def test_skip_because_bug_without_bug_never_skips(self):
- """Never skip without a bug parameter."""
- self._test_skip_because_helper(expected_to_skip=False,
- condition=True)
- self._test_skip_because_helper(expected_to_skip=False)
+ def test_skip_because_storyboard_bug_and_condition_true(self):
+ self._test_skip_helper(bug='1992', bug_type='storyboard',
+ condition=True)
- def test_skip_because_invalid_bug_number(self):
- """Raise InvalidParam if with an invalid bug number"""
- self.assertRaises(lib_exc.InvalidParam, self._test_skip_because_helper,
- bug='critical_bug')
+
+class TestUnstableTestDecorator(base.TestCase, BaseSkipDecoratorTests):
+
+ def _test_skip_helper(self, raise_exception=True, expected_to_skip=True,
+ **decorator_args):
+ fail_test_reason = "test_bar failed"
+
+ class TestFoo(test.BaseTestCase):
+
+ @decorators.unstable_test(**decorator_args)
+ def test_bar(self):
+ if raise_exception:
+ raise Exception(fail_test_reason)
+ else:
+ return 0
+
+ t = TestFoo('test_bar')
+ if expected_to_skip:
+ e = self.assertRaises(testtools.TestCase.skipException, t.test_bar)
+ bug = decorator_args['bug']
+ bug_type = decorator_args.get('bug_type', 'launchpad')
+ self.assertRegex(
+ str(e),
+ r'Marked as unstable and skipped because of bug\: %s.*, '
+ 'failure was: %s' % (decorators._get_bug_url(bug, bug_type),
+ fail_test_reason)
+ )
+ else:
+ # assert that test_bar returned 0
+ self.assertEqual(TestFoo('test_bar').test_bar(), 0)
+
+ def test_skip_bug_given_exception_not_raised(self):
+ self._test_skip_helper(raise_exception=False, expected_to_skip=False,
+ bug='1234')
class TestIdempotentIdDecorator(base.TestCase):