Retry glance add/update locations on BadRequest
If glance is unable to fetch the http image URL we give it due to
transient network problems, we currently fail the test because it
does not retry itself and we get a BadRequest. Because such problems
can happen in a CI run due to network interruptions, this adds a
retry loop on that condition when we go to add a location. If we fail
to add the location for some legit reason related to our actual
location URI or other parts of the request, all the retries will
fail and we'll still catch the problem.
Change-Id: I944eb076c9f9056200dc193b52f2004038a6942f
Related-Bug: #2004641
Related-Bug: #1999800
Related-Bug: #2006473
diff --git a/tempest/api/image/v2/test_images.py b/tempest/api/image/v2/test_images.py
index d590668..e8734e0 100644
--- a/tempest/api/image/v2/test_images.py
+++ b/tempest/api/image/v2/test_images.py
@@ -14,8 +14,10 @@
# License for the specific language governing permissions and limitations
# under the License.
+import contextlib
import io
import random
+import time
from oslo_log import log as logging
from tempest.api.image import base
@@ -29,6 +31,19 @@
LOG = logging.getLogger(__name__)
+@contextlib.contextmanager
+def retry_bad_request(fn):
+ retries = 3
+ for i in range(retries):
+ try:
+ yield
+ except lib_exc.BadRequest:
+ if i < retries:
+ time.sleep(1)
+ else:
+ raise
+
+
class ImportImagesTest(base.BaseV2ImageTest):
"""Here we test the import operations for image"""
@@ -817,8 +832,14 @@
# Add a new location
new_loc = {'metadata': {'foo': 'bar'},
'url': CONF.image.http_image}
- self.client.update_image(image['id'], [
- dict(add='/locations/-', value=new_loc)])
+
+ # NOTE(danms): If glance was unable to fetch the remote image via
+ # HTTP, it will return BadRequest. Because this can be transient in
+ # CI, we try this a few times before we agree that it has failed
+ # for a reason worthy of failing the test.
+ with retry_bad_request():
+ self.client.update_image(image['id'], [
+ dict(add='/locations/-', value=new_loc)])
# The image should now be active, with one location that looks
# like we expect
@@ -848,8 +869,14 @@
new_loc = {'metadata': {'speed': '88mph'},
'url': '%s#new' % CONF.image.http_image}
- self.client.update_image(image['id'], [
- dict(add='/locations/-', value=new_loc)])
+
+ # NOTE(danms): If glance was unable to fetch the remote image via
+ # HTTP, it will return BadRequest. Because this can be transient in
+ # CI, we try this a few times before we agree that it has failed
+ # for a reason worthy of failing the test.
+ with retry_bad_request():
+ self.client.update_image(image['id'], [
+ dict(add='/locations/-', value=new_loc)])
# The image should now have two locations and the last one
# (locations are ordered) should have the new URL.