Fix timeout processing in CICD jobs
- default timeout increased from 1800 to 2400 sec because 'core'
component deployment takes more than 30 min now
- print the timeout message in run_jenkins_job.py to stdout along
with stderr
- do not fail get_jenkins_job_stages.py if info['result'] contains
'None' in cases when the job is failed by timeout. Wait for few
seconds, then report the workflow stages 'as is':
Create infrastructure: SUCCESS
Install core infrastructure: SUCCESS
Install infra: IN_PROGRESS
Change-Id: I2ca592a8f5069bb38ec7659f15eb72e4bfba1722
Closes-Bug:#PROD-23815
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index 108ee0c..a2f5a4e 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -310,7 +310,7 @@
build_pipeline_job('create-cfg-config-drive', parameters)
}
-def run_job_on_day01_node(stack_to_install, timeout=1800) {
+def run_job_on_day01_node(stack_to_install, timeout=2400) {
// stack_to_install="core,cicd"
def stack = "${stack_to_install}"
try {
@@ -340,7 +340,7 @@
}
}
-def run_job_on_cicd_nodes(stack_to_install, timeout=1800) {
+def run_job_on_cicd_nodes(stack_to_install, timeout=2400) {
// stack_to_install="k8s,calico,stacklight"
def stack = "${stack_to_install}"
try {
diff --git a/tcp_tests/managers/jenkins/client.py b/tcp_tests/managers/jenkins/client.py
index fbd5c43..afc8900 100644
--- a/tcp_tests/managers/jenkins/client.py
+++ b/tcp_tests/managers/jenkins/client.py
@@ -143,8 +143,8 @@
building,
timeout=timeout,
interval=interval,
- timeout_msg='Timeout waiting, job {0} are not finished "{1}" build'
- ' still'.format(name, build_id))
+ timeout_msg=('Timeout waiting the job {0}:{1} in {2} sec.'
+ .format(name, build_id, timeout)))
def get_build_output(self, name, build_id):
return self.__client.get_build_console_output(name, build_id)
diff --git a/tcp_tests/utils/get_jenkins_job_stages.py b/tcp_tests/utils/get_jenkins_job_stages.py
index 361b8d1..883494f 100755
--- a/tcp_tests/utils/get_jenkins_job_stages.py
+++ b/tcp_tests/utils/get_jenkins_job_stages.py
@@ -111,16 +111,12 @@
for _ in range(3):
wf = jenkins.get_workflow(opts.job_name, opts.build_number)
info = jenkins.build_info(opts.job_name, int(wf['id']))
- if info is not None:
+ if info.get('result'):
break
time.sleep(3)
- if not info:
- raise("Cannot get info for the job {0}:{1}".format(opts.job_name,
- opts.build_number))
-
build_description = ("[" + info['fullDisplayName'] + "] " +
- info['url'] + " : " + info['result'])
+ info['url'] + " : " + (info['result'] or 'No result'))
stages = get_stages(wf['stages'], 0)
if not stages:
msg = wf['status'] + ":\n\n"
diff --git a/tcp_tests/utils/run_jenkins_job.py b/tcp_tests/utils/run_jenkins_job.py
index 00ebec6..b01f366 100755
--- a/tcp_tests/utils/run_jenkins_job.py
+++ b/tcp_tests/utils/run_jenkins_job.py
@@ -4,6 +4,7 @@
import os
import sys
+from devops import error
import json
sys.path.append(os.getcwd())
@@ -131,13 +132,18 @@
if opts.verbose:
print_build_header(build, job_params, opts)
- jenkins.wait_end_of_build(
- name=build[0],
- build_id=build[1],
- timeout=opts.build_timeout,
- interval=1,
- verbose=opts.verbose,
- job_output_prefix=opts.job_output_prefix)
+ try:
+ jenkins.wait_end_of_build(
+ name=build[0],
+ build_id=build[1],
+ timeout=opts.build_timeout,
+ interval=1,
+ verbose=opts.verbose,
+ job_output_prefix=opts.job_output_prefix)
+ except error.TimeoutError as e:
+ print(str(e))
+ raise
+
result = jenkins.build_info(name=build[0],
build_id=build[1])['result']
if opts.verbose: