Merge "Adding retries for raw_session adapter" into release/2019.2.0
diff --git a/_modules/keystonev3/common.py b/_modules/keystonev3/common.py
index 93cfe6d..9f2c8b1 100644
--- a/_modules/keystonev3/common.py
+++ b/_modules/keystonev3/common.py
@@ -1,5 +1,9 @@
import logging
import os_client_config
+import functools
+import time
+
+from keystoneclient import exceptions as ks_exceptions
log = logging.getLogger(__name__)
@@ -64,33 +68,60 @@
def send(method, microversion_header=None):
def wrap(func):
+ @functools.wraps(func)
def wrapped_f(*args, **kwargs):
headers = kwargs.pop('headers', {})
if kwargs.get('microversion'):
headers.setdefault(microversion_header,
kwargs.get('microversion'))
cloud_name = kwargs.pop('cloud_name')
+ connect_retries = 30
+ connect_retry_delay = 1
if not cloud_name:
e = NoCredentials()
log.error('%s' % e)
raise e
- adapter = _get_raw_client(cloud_name)
+ adapter = None
+ for i in range(connect_retries):
+ try:
+ adapter = _get_raw_client(cloud_name)
+ except (ks_exceptions.DiscoveryFailure) as e:
+ msg = ("Got exception when determining a suitable "
+ "URL for Keystone plugin. Sleeping for %ss. Attepmpts "
+ "%s of %s")
+ log.error(msg % (connect_retry_delay, i, connect_retries))
+ time.sleep(connect_retry_delay)
+ continue
+ break
+ if not adapter:
+ raise ks_exceptions.DiscoveryFailure("Could not connect to Keystone API to determine a suitable URL for the plugin")
# Remove salt internal kwargs
kwarg_keys = list(kwargs.keys())
for k in kwarg_keys:
if k.startswith('__'):
kwargs.pop(k)
- url, json = func(*args, **kwargs)
- if json:
- response = getattr(adapter, method)(url, headers=headers,
- json=json)
- else:
- response = getattr(adapter, method)(url, headers=headers)
- if not response.content:
+ url, json = func(*args, **kwargs)
+ response = None
+ for i in range(connect_retries):
+ try:
+ response = getattr(adapter, method)(
+ url, connect_retries=connect_retries,
+ json=json)
+ except Exception as e:
+ if not hasattr(e, 'http_status') or (e.http_status >= 500
+ or e.http_status == 0):
+ msg = ("Got retriable exception when contacting "
+ "Keystone API. Sleeping for %ss. Attepmpts "
+ "%s of %s")
+ log.error(msg % (connect_retry_delay, i, connect_retries))
+ time.sleep(connect_retry_delay)
+ continue
+ break
+ if not response or not response.content:
return {}
try:
resp = response.json()
- except:
+ except ValueError:
resp = response.content
return resp
return wrapped_f