Adding retries for raw_session adapter
When race conditoin occures if keystone is brought up but HAProxy
doesn't detect that it is alive then DiscoveryFailure expection raises
to fix the issue connection retries have been implemented.
Change-Id: Ie4355a44be5501a8b404252a5ab304b5528d81ed
Related-PROD: PROD-27890 (PROD:27890)
diff --git a/_modules/keystonev3/common.py b/_modules/keystonev3/common.py
index 93cfe6d..9f2c8b1 100644
--- a/_modules/keystonev3/common.py
+++ b/_modules/keystonev3/common.py
@@ -1,5 +1,9 @@
import logging
import os_client_config
+import functools
+import time
+
+from keystoneclient import exceptions as ks_exceptions
log = logging.getLogger(__name__)
@@ -64,33 +68,60 @@
def send(method, microversion_header=None):
def wrap(func):
+ @functools.wraps(func)
def wrapped_f(*args, **kwargs):
headers = kwargs.pop('headers', {})
if kwargs.get('microversion'):
headers.setdefault(microversion_header,
kwargs.get('microversion'))
cloud_name = kwargs.pop('cloud_name')
+ connect_retries = 30
+ connect_retry_delay = 1
if not cloud_name:
e = NoCredentials()
log.error('%s' % e)
raise e
- adapter = _get_raw_client(cloud_name)
+ adapter = None
+ for i in range(connect_retries):
+ try:
+ adapter = _get_raw_client(cloud_name)
+ except (ks_exceptions.DiscoveryFailure) as e:
+ msg = ("Got exception when determining a suitable "
+ "URL for Keystone plugin. Sleeping for %ss. Attepmpts "
+ "%s of %s")
+ log.error(msg % (connect_retry_delay, i, connect_retries))
+ time.sleep(connect_retry_delay)
+ continue
+ break
+ if not adapter:
+ raise ks_exceptions.DiscoveryFailure("Could not connect to Keystone API to determine a suitable URL for the plugin")
# Remove salt internal kwargs
kwarg_keys = list(kwargs.keys())
for k in kwarg_keys:
if k.startswith('__'):
kwargs.pop(k)
- url, json = func(*args, **kwargs)
- if json:
- response = getattr(adapter, method)(url, headers=headers,
- json=json)
- else:
- response = getattr(adapter, method)(url, headers=headers)
- if not response.content:
+ url, json = func(*args, **kwargs)
+ response = None
+ for i in range(connect_retries):
+ try:
+ response = getattr(adapter, method)(
+ url, connect_retries=connect_retries,
+ json=json)
+ except Exception as e:
+ if not hasattr(e, 'http_status') or (e.http_status >= 500
+ or e.http_status == 0):
+ msg = ("Got retriable exception when contacting "
+ "Keystone API. Sleeping for %ss. Attepmpts "
+ "%s of %s")
+ log.error(msg % (connect_retry_delay, i, connect_retries))
+ time.sleep(connect_retry_delay)
+ continue
+ break
+ if not response or not response.content:
return {}
try:
resp = response.json()
- except:
+ except ValueError:
resp = response.content
return resp
return wrapped_f