Retry all API calls for >=500 responses
reraise original error otherwise.
Hardcoded 30 retries with 1 second interval.
Mostly useful to wake up the API dirung first cold start of Apache site
in the absence of haproxy probes.
Change-Id: I43dbc1f799913f010f87c73188be44fac07734fe
Related-Issue: PROD-32186
diff --git a/_modules/gnocchiv1/common.py b/_modules/gnocchiv1/common.py
index a75c26f..1e5282f 100644
--- a/_modules/gnocchiv1/common.py
+++ b/_modules/gnocchiv1/common.py
@@ -1,4 +1,5 @@
import logging
+import time
from uuid import UUID
try:
@@ -82,6 +83,8 @@
def send(method):
def wrap(func):
def wrapped_f(*args, **kwargs):
+ connect_retries = 30
+ connect_retry_delay = 1
cloud_name = kwargs.pop('cloud_name')
if not cloud_name:
e = NoCredentials()
@@ -94,9 +97,25 @@
if k.startswith('__'):
kwargs.pop(k)
url, request_kwargs = func(*args, **kwargs)
- response = getattr(adapter, method)(url, **request_kwargs)
- if not response.content:
- return {}
+ response = None
+ for i in range(connect_retries):
+ try:
+ response = getattr(adapter, method)(
+ url, connect_retries=connect_retries,
+ **request_kwargs)
+ except Exception as e:
+ if not hasattr(e, 'http_status') or (
+ e.http_status >= 500 or e.http_status == 0):
+ msg = ("Got retriable exception when contacting "
+ "Gnocchi API. Sleeping for %s seconds. "
+ "Attepmpts %s of %s")
+ log.error(msg % (connect_retry_delay, i,
+ connect_retries))
+ time.sleep(connect_retry_delay)
+ continue
+ else:
+ raise
+ break
return response.json()
return wrapped_f
return wrap