Add retry for all requests to the Jenkins client on cluster
Change-Id: I69eccbebb58f3ce902abd11cf3926330e5fb2204
Related-Prod: #PROD-35718
diff --git a/tcp_tests/managers/jenkins/client.py b/tcp_tests/managers/jenkins/client.py
index 1256552..063d400 100644
--- a/tcp_tests/managers/jenkins/client.py
+++ b/tcp_tests/managers/jenkins/client.py
@@ -11,35 +11,53 @@
from requests.exceptions import ConnectionError
+def retry(max_count=6,
+ sleep_before_retry=10):
+ def _retry(func):
+ def __retry(*args, **kwargs):
+ """
+ Waits some time and retries the requests if it fails with
+ with any error
+ Raises Exceptions after all unsuccessful tries
+
+ :param func: callable
+ :param args, kwargs: parameters of decorated functions
+ :param max_count: times of retries
+ :param sleep_before_retry: how many seconds needs to wait before
+ the next retry
+ :return: response
+ :raise ConnectionError after several unsuccessful connections
+ """
+ err = None
+ for count in range(max_count):
+ try:
+ return func(*args, **kwargs)
+ except Exception as err:
+ print("Try {count}/{max_count} caught Exception: {err}. \
+ \n... repeat after {secs} secs".
+ format(err=err,
+ count=count+1,
+ max_count=max_count,
+ secs=sleep_before_retry))
+ time.sleep(sleep_before_retry)
+ print("Function failed in {total_time} seconds".format(
+ total_time=max_count*sleep_before_retry)
+ )
+ raise err
+ return __retry
+ return _retry
+
+
+@retry(max_count=5, sleep_before_retry=2)
def send_request(action, endpoint):
"""
- Wait 2 seconds time and retry requests if it fail with connection to
- endpoint
- Raises Connection exceptions after 5 unsuccessful steps
-
- :param action: string
- :param body: string, url to send request
- :return: response
- :raise ConnectionError after several unsuccessful connections
+ Makes request with described operation to endpoint
+ :param action: string, type of operation GET, POST, DELETE and so on
+ :param endpoint: string, url to send request
+ :return: response
"""
- max_retries = 5
- sleep_time_before_repeat = 2
-
- step = 0
- while step < max_retries:
- try:
- response = requests.Request(action, endpoint)
- return response
- except ConnectionError as error:
- step = step + 1
- print("Can't get {} due to error: {}.\nRetry {}/{}".format(
- endpoint,
- error,
- step,
- max_retries
- ))
- time.sleep(sleep_time_before_repeat)
- raise ConnectionError
+ response = requests.Request(action, endpoint)
+ return response
class JenkinsClient(object):
@@ -52,41 +70,23 @@
password=password)
self.__client._session.verify = False
+ @retry()
def jobs(self):
return self.__client.get_jobs()
def find_jobs(self, name):
return filter(lambda x: name in x['fullname'], self.jobs())
+ @retry()
def job_info(self, name):
- max_count = 6
- for count in range(max_count):
- try:
- return self.__client.get_job_info(name)
- except jenkins.JenkinsException as err:
- print("caught JenkinsException: {err}. \
- repeat {count}/{max_count}".
- format(err=err,
- count=count,
- max_count=max_count))
- time.sleep(10)
+ return self.__client.get_job_info(name)
def list_builds(self, name):
return self.job_info(name).get('builds')
+ @retry()
def build_info(self, name, build_id):
- max_count = 6
- for count in range(max_count):
- try:
- build = self.__client.get_build_info(name, build_id)
- return build
- except jenkins.JenkinsException as err:
- print("caught JenkinsException: {err}. \
- repeat {count}/{max_count}".
- format(err=err,
- count=count,
- max_count=max_count))
- time.sleep(10)
+ return self.__client.get_build_info(name, build_id)
def job_params(self, name):
job = self.job_info(name)
@@ -103,6 +103,7 @@
for j in job_params])
return def_params
+ @retry()
def run_build(self, name, params=None, timeout=600, verbose=False):
params = params or self.make_defults_params(name)
num = self.__client.build_job(name, params)
@@ -217,9 +218,11 @@
timeout_msg=('Timeout waiting the job {0}:{1} in {2} sec.'
.format(name, build_id, timeout)))
+ @retry()
def get_build_output(self, name, build_id):
return self.__client.get_build_console_output(name, build_id)
+ @retry(max_count=12)
def get_progressive_build_output(self, name, build_id, start=0):
'''Get build console text.
@@ -238,6 +241,7 @@
self.__client._build_url(PROGRESSIVE_CONSOLE_OUTPUT, locals()))
return(self.__client.jenkins_request(req))
+ @retry(max_count=12)
def get_workflow(self, name, build_id, enode=None, mode='describe'):
'''Get workflow results from pipeline job
@@ -262,6 +266,7 @@
response = self.__client.jenkins_open(req)
return json.loads(response)
+ @retry(max_count=12)
def get_artifact(self, name, build_id, artifact_path, destination_name):
'''Wait until the specified build is finished
diff --git a/tcp_tests/managers/reclass_manager.py b/tcp_tests/managers/reclass_manager.py
index f8895c6..137dd33 100644
--- a/tcp_tests/managers/reclass_manager.py
+++ b/tcp_tests/managers/reclass_manager.py
@@ -23,7 +23,7 @@
__config = None
__underlay = None
reclass_tools_cmd = ". venv-reclass-tools/bin/activate; reclass-tools "
- tgt = "cfg01" # place where the reclass-tools installed
+ tgt = "cfg01" # place where the reclass-tools installed
def __init__(self, config, underlay):
self.__config = config
@@ -38,11 +38,9 @@
def check_existence(self, key):
if key in self.ssh.check_call(
- "{reclass_tools} get-key {key} /srv/salt/reclass/classes"
- .format(
- reclass_tools=self.reclass_tools_cmd,
- key=key
- )):
+ "{reclass_tools} get-key {key} /srv/salt/reclass/classes".
+ format(reclass_tools=self.reclass_tools_cmd,
+ key=key)):
LOG.warning("({}) key already exists in reclass".format(key))
return True
return False
@@ -66,7 +64,7 @@
key=key,
value=value,
path=short_path
- ))
+ ))
def get_key(self, key, file_name):
"""Find a key in a YAML
@@ -75,6 +73,10 @@
:param file_name: name of YAML file to find a key
:return: str, key if found
"""
+ LOG.info("Try to get '{key}' key from '{file}' file".format(
+ file=file_name,
+ key=key
+ ))
request_key = self.ssh.check_call(
"{reclass_tools} get-key {key} /srv/salt/reclass/*/{file_name}".
format(reclass_tools=self.reclass_tools_cmd,
@@ -91,14 +93,13 @@
# '\n']
# So we have no chance to get value without dirty code like `stdout[3]`
- LOG.info("From reclass.get_key {}".format(request_key))
+ LOG.info("Raw output from reclass.get_key {}".format(request_key))
if len(request_key) < 4:
- assert "Can't find {key} in {file_name}. Got stdout {stdout}".\
- format(
- key=key,
- file_name=file_name,
- stdout=request_key
- )
+ print("Can't find {key} in {file_name}. Got stdout {stdout}".
+ format(key=key,
+ file_name=file_name,
+ stdout=request_key))
+ return None
value = request_key[3].strip('\n')
LOG.info("From reclass.get_key {}".format(value))
return value