Adding retries for raw_session adapter
When race conditoin occures if keystone is brought up but HAProxy
doesn't detect that it is alive then DiscoveryFailure expection raises
to fix the issue connection retries have been implemented.
Change-Id: Ie4355a44be5501a8b404252a5ab304b5528d81ed
Related-PROD: PROD-27890 (PROD:27890)
diff --git a/_modules/keystonev3/common.py b/_modules/keystonev3/common.py
index 0ce0907..9a7744b 100644
--- a/_modules/keystonev3/common.py
+++ b/_modules/keystonev3/common.py
@@ -1,4 +1,6 @@
+import functools
import logging
+import time
try:
import os_client_config
@@ -6,6 +8,8 @@
os_client_config = None
from salt import exceptions
+from keystoneclient import exceptions as ks_exceptions
+
log = logging.getLogger(__name__)
@@ -73,33 +77,60 @@
def send(method, microversion_header=None):
def wrap(func):
+ @functools.wraps(func)
def wrapped_f(*args, **kwargs):
headers = kwargs.pop('headers', {})
if kwargs.get('microversion'):
headers.setdefault(microversion_header,
kwargs.get('microversion'))
cloud_name = kwargs.pop('cloud_name')
+ connect_retries = 30
+ connect_retry_delay = 1
if not cloud_name:
e = NoCredentials()
log.error('%s' % e)
raise e
- adapter = _get_raw_client(cloud_name)
+ adapter = None
+ for i in range(connect_retries):
+ try:
+ adapter = _get_raw_client(cloud_name)
+ except (ks_exceptions.DiscoveryFailure) as e:
+ msg = ("Got exception when determining a suitable "
+ "URL for Keystone plugin. Sleeping for %ss. Attepmpts "
+ "%s of %s")
+ log.error(msg % (connect_retry_delay, i, connect_retries))
+ time.sleep(connect_retry_delay)
+ continue
+ break
+ if not adapter:
+ raise ks_exceptions.DiscoveryFailure("Could not connect to Keystone API to determine a suitable URL for the plugin")
# Remove salt internal kwargs
kwarg_keys = list(kwargs.keys())
for k in kwarg_keys:
if k.startswith('__'):
kwargs.pop(k)
- url, json = func(*args, **kwargs)
- if json:
- response = getattr(adapter, method)(url, headers=headers,
- json=json)
- else:
- response = getattr(adapter, method)(url, headers=headers)
- if not response.content:
+ url, json = func(*args, **kwargs)
+ response = None
+ for i in range(connect_retries):
+ try:
+ response = getattr(adapter, method)(
+ url, connect_retries=connect_retries,
+ json=json)
+ except Exception as e:
+ if not hasattr(e, 'http_status') or (e.http_status >= 500
+ or e.http_status == 0):
+ msg = ("Got retriable exception when contacting "
+ "Keystone API. Sleeping for %ss. Attepmpts "
+ "%s of %s")
+ log.error(msg % (connect_retry_delay, i, connect_retries))
+ time.sleep(connect_retry_delay)
+ continue
+ break
+ if not response or not response.content:
return {}
try:
resp = response.json()
- except:
+ except ValueError:
resp = response.content
return resp
return wrapped_f