Add retry for all requests to the Jenkins client on cluster
Change-Id: I69eccbebb58f3ce902abd11cf3926330e5fb2204
Related-Prod: #PROD-35718
diff --git a/tcp_tests/managers/jenkins/client.py b/tcp_tests/managers/jenkins/client.py
index 1256552..063d400 100644
--- a/tcp_tests/managers/jenkins/client.py
+++ b/tcp_tests/managers/jenkins/client.py
@@ -11,35 +11,53 @@
from requests.exceptions import ConnectionError
+def retry(max_count=6,
+ sleep_before_retry=10):
+ def _retry(func):
+ def __retry(*args, **kwargs):
+ """
+ Waits some time and retries the requests if it fails with
+ with any error
+ Raises Exceptions after all unsuccessful tries
+
+ :param func: callable
+ :param args, kwargs: parameters of decorated functions
+ :param max_count: times of retries
+ :param sleep_before_retry: how many seconds needs to wait before
+ the next retry
+ :return: response
+ :raise ConnectionError after several unsuccessful connections
+ """
+ err = None
+ for count in range(max_count):
+ try:
+ return func(*args, **kwargs)
+ except Exception as err:
+ print("Try {count}/{max_count} caught Exception: {err}. \
+ \n... repeat after {secs} secs".
+ format(err=err,
+ count=count+1,
+ max_count=max_count,
+ secs=sleep_before_retry))
+ time.sleep(sleep_before_retry)
+ print("Function failed in {total_time} seconds".format(
+ total_time=max_count*sleep_before_retry)
+ )
+ raise err
+ return __retry
+ return _retry
+
+
+@retry(max_count=5, sleep_before_retry=2)
def send_request(action, endpoint):
"""
- Wait 2 seconds time and retry requests if it fail with connection to
- endpoint
- Raises Connection exceptions after 5 unsuccessful steps
-
- :param action: string
- :param body: string, url to send request
- :return: response
- :raise ConnectionError after several unsuccessful connections
+ Makes request with described operation to endpoint
+ :param action: string, type of operation GET, POST, DELETE and so on
+ :param endpoint: string, url to send request
+ :return: response
"""
- max_retries = 5
- sleep_time_before_repeat = 2
-
- step = 0
- while step < max_retries:
- try:
- response = requests.Request(action, endpoint)
- return response
- except ConnectionError as error:
- step = step + 1
- print("Can't get {} due to error: {}.\nRetry {}/{}".format(
- endpoint,
- error,
- step,
- max_retries
- ))
- time.sleep(sleep_time_before_repeat)
- raise ConnectionError
+ response = requests.Request(action, endpoint)
+ return response
class JenkinsClient(object):
@@ -52,41 +70,23 @@
password=password)
self.__client._session.verify = False
+ @retry()
def jobs(self):
return self.__client.get_jobs()
def find_jobs(self, name):
return filter(lambda x: name in x['fullname'], self.jobs())
+ @retry()
def job_info(self, name):
- max_count = 6
- for count in range(max_count):
- try:
- return self.__client.get_job_info(name)
- except jenkins.JenkinsException as err:
- print("caught JenkinsException: {err}. \
- repeat {count}/{max_count}".
- format(err=err,
- count=count,
- max_count=max_count))
- time.sleep(10)
+ return self.__client.get_job_info(name)
def list_builds(self, name):
return self.job_info(name).get('builds')
+ @retry()
def build_info(self, name, build_id):
- max_count = 6
- for count in range(max_count):
- try:
- build = self.__client.get_build_info(name, build_id)
- return build
- except jenkins.JenkinsException as err:
- print("caught JenkinsException: {err}. \
- repeat {count}/{max_count}".
- format(err=err,
- count=count,
- max_count=max_count))
- time.sleep(10)
+ return self.__client.get_build_info(name, build_id)
def job_params(self, name):
job = self.job_info(name)
@@ -103,6 +103,7 @@
for j in job_params])
return def_params
+ @retry()
def run_build(self, name, params=None, timeout=600, verbose=False):
params = params or self.make_defults_params(name)
num = self.__client.build_job(name, params)
@@ -217,9 +218,11 @@
timeout_msg=('Timeout waiting the job {0}:{1} in {2} sec.'
.format(name, build_id, timeout)))
+ @retry()
def get_build_output(self, name, build_id):
return self.__client.get_build_console_output(name, build_id)
+ @retry(max_count=12)
def get_progressive_build_output(self, name, build_id, start=0):
'''Get build console text.
@@ -238,6 +241,7 @@
self.__client._build_url(PROGRESSIVE_CONSOLE_OUTPUT, locals()))
return(self.__client.jenkins_request(req))
+ @retry(max_count=12)
def get_workflow(self, name, build_id, enode=None, mode='describe'):
'''Get workflow results from pipeline job
@@ -262,6 +266,7 @@
response = self.__client.jenkins_open(req)
return json.loads(response)
+ @retry(max_count=12)
def get_artifact(self, name, build_id, artifact_path, destination_name):
'''Wait until the specified build is finished