Retry on ServerFault in call_and_ignore_notfound_exc()
This makes us tolerate up to three ServerFault errors while trying to
do things like clean up resources during tearDown().
Change-Id: I3b2dac90fd6c71d66506d33aa5e35bb362d9bf87
Related-Bug: #1897907
diff --git a/tempest/lib/common/utils/test_utils.py b/tempest/lib/common/utils/test_utils.py
index 2a9f3a9..4cf8351 100644
--- a/tempest/lib/common/utils/test_utils.py
+++ b/tempest/lib/common/utils/test_utils.py
@@ -80,10 +80,19 @@
def call_and_ignore_notfound_exc(func, *args, **kwargs):
"""Call the given function and pass if a `NotFound` exception is raised."""
- try:
- return func(*args, **kwargs)
- except exceptions.NotFound:
- pass
+ attempt = 0
+ while True:
+ attempt += 1
+ try:
+ return func(*args, **kwargs)
+ except exceptions.NotFound:
+ return
+ except exceptions.ServerFault:
+ # NOTE(danms): Tolerate three ServerFault exceptions while trying
+ # to do this thing, and after that, assume it's legit.
+ if attempt >= 3:
+ raise
+ LOG.warning('Got ServerFault while running %s, retrying...', func)
def call_until_true(func, duration, sleep_for, *args, **kwargs):
diff --git a/tempest/tests/lib/common/utils/test_test_utils.py b/tempest/tests/lib/common/utils/test_test_utils.py
index bdc0ea4..d8e3745 100644
--- a/tempest/tests/lib/common/utils/test_test_utils.py
+++ b/tempest/tests/lib/common/utils/test_test_utils.py
@@ -74,6 +74,17 @@
self.assertRaises(ValueError, test_utils.call_and_ignore_notfound_exc,
raise_value_error)
+ def test_call_and_ignore_notfound_exc_when_serverfault_raised(self):
+ calls = []
+
+ def raise_serverfault():
+ calls.append('call')
+ raise exceptions.ServerFault()
+ self.assertRaises(exceptions.ServerFault,
+ test_utils.call_and_ignore_notfound_exc,
+ raise_serverfault)
+ self.assertEqual(3, len(calls))
+
def test_call_and_ignore_notfound_exc(self):
m = mock.Mock(return_value=42)
args, kwargs = (1,), {'1': None}