Merge "Add retry for salttimeout errors and ceph sporadic Related-Prod:PROD-37137"
diff --git a/tcp_tests/utils/run_jenkins_job.py b/tcp_tests/utils/run_jenkins_job.py
index bf3bb01..403cc00 100755
--- a/tcp_tests/utils/run_jenkins_job.py
+++ b/tcp_tests/utils/run_jenkins_job.py
@@ -127,24 +127,39 @@
if job_parameters is not None: # job_parameters = {}
job_params.update(job_parameters)
- build = jenkins.run_build(job_name,
- job_params,
- verbose=verbose,
- timeout=start_timeout)
- if verbose:
- print_build_header(build, job_params, build_timeout)
+ job_attempts = 2
+ count = 1
+ while (job_attempts != 0):
+ print('Attempt ' + str(count))
+ count += 1
+ job_attempts -= 1
+ build = jenkins.run_build(job_name,
+ job_params,
+ verbose=verbose,
+ timeout=start_timeout)
- try:
- jenkins.wait_end_of_build(
- name=build[0],
- build_id=build[1],
- timeout=build_timeout,
- interval=1,
- verbose=verbose,
- job_output_prefix=job_output_prefix)
- except Exception as e:
- print(str(e))
- raise
+ if verbose:
+ print_build_header(build, job_params, build_timeout)
+
+ try:
+ jenkins.wait_end_of_build(
+ name=build[0],
+ build_id=build[1],
+ timeout=build_timeout,
+ interval=1,
+ verbose=verbose,
+ job_output_prefix=job_output_prefix)
+ except Exception as e:
+ print(str(e))
+ raise
+ job_log = jenkins.get_build_output(job_name, build[1])
+ # Workaround for restart jobs that failed by salt-timeout errors
+ # or by sporadic fail in attach disks
+ if ('SaltReqTimeoutError' not in job_log and
+ 'not a block device' not in job_log):
+ break
+ else:
+ print('Job returns known infra fail!')
result = jenkins.build_info(name=build[0],
build_id=build[1])['result']