blob: af4f4aff38d0a187a5ca9e7a93c9065279c376e0 [file] [log] [blame]
Dennis Dmitrievb3b37492018-07-08 21:23:00 +03001package com.mirantis.system_qa
2
Dennis Dmitriev27a96792018-07-30 07:52:03 +03003import groovy.xml.XmlUtil
Dennis Dmitrievb3b37492018-07-08 21:23:00 +03004
Vladimir Jigulin6a1ef812019-03-21 19:34:07 +04005def is_released_version(version) {
6 return Character.isDigit(version.charAt(0))
7}
8
Dennis Dmitrieveb50ce12018-09-27 13:34:32 +03009def run_sh(String cmd) {
10 // run shell script without catching any output
11 def common = new com.mirantis.mk.Common()
12 common.printMsg("Run shell command:\n" + cmd, "blue")
13 def VENV_PATH='/home/jenkins/fuel-devops30'
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020014 def script = """\
Dennis Dmitrieveb50ce12018-09-27 13:34:32 +030015 set -ex;
16 . ${VENV_PATH}/bin/activate;
17 bash -c '${cmd.stripIndent()}'
18 """
19 return sh(script: script)
20}
21
Dennis Dmitriev8df35442018-07-31 08:40:20 +030022def run_cmd(String cmd, Boolean returnStdout=false) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030023 def common = new com.mirantis.mk.Common()
24 common.printMsg("Run shell command:\n" + cmd, "blue")
25 def VENV_PATH='/home/jenkins/fuel-devops30'
Dennis Dmitriev27a96792018-07-30 07:52:03 +030026 def stderr_path = "/tmp/${JOB_NAME}_${BUILD_NUMBER}_stderr.log"
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020027 def script = """#!/bin/bash
28 set +x
29 echo 'activate python virtualenv ${VENV_PATH}'
30 . ${VENV_PATH}/bin/activate
31 bash -c -e -x '${cmd.stripIndent()}' 2>${stderr_path}
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030032 """
Dennis Dmitriev27a96792018-07-30 07:52:03 +030033 try {
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020034 def stdout = sh(script: script, returnStdout: returnStdout)
35 def stderr = readFile("${stderr_path}")
36 def error_message = "\n<<<<<< STDERR: >>>>>>\n" + stderr
37 common.printMsg(error_message, "yellow")
38 common.printMsg("", "reset")
39 return stdout
Dennis Dmitriev27a96792018-07-30 07:52:03 +030040 } catch (e) {
Dennis Dmitriev8df35442018-07-31 08:40:20 +030041 def stderr = readFile("${stderr_path}")
42 def error_message = e.message + "\n<<<<<< STDERR: >>>>>>\n" + stderr
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020043 common.printMsg(error_message, "red")
44 common.printMsg("", "reset")
Dennis Dmitriev8df35442018-07-31 08:40:20 +030045 throw new Exception(error_message)
Dennis Dmitriev27a96792018-07-30 07:52:03 +030046 } finally {
Dennis Dmitriev8df35442018-07-31 08:40:20 +030047 sh(script: "rm ${stderr_path} || true")
Dennis Dmitriev27a96792018-07-30 07:52:03 +030048 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030049}
50
51def run_cmd_stdout(cmd) {
Dennis Dmitriev8df35442018-07-31 08:40:20 +030052 return run_cmd(cmd, true)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030053}
54
Dennis Dmitriev27a96792018-07-30 07:52:03 +030055def build_pipeline_job(job_name, parameters) {
56 //Build a job, grab the results if failed and use the results in exception
57 def common = new com.mirantis.mk.Common()
58 common.printMsg("Start building job '${job_name}' with parameters:", "purple")
59 common.prettyPrint(parameters)
60
61 def job_info = build job: "${job_name}",
62 parameters: parameters,
63 propagate: false
64
65 if (job_info.getResult() != "SUCCESS") {
66 currentBuild.result = job_info.getResult()
67 def build_number = job_info.getNumber()
Dennis Dmitrievb08c0772018-10-17 15:10:26 +030068 common.printMsg("Job '${job_name}' failed, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +030069 def workflow_details=run_cmd_stdout("""\
70 export JOB_NAME=${job_name}
71 export BUILD_NUMBER=${build_number}
72 python ./tcp_tests/utils/get_jenkins_job_stages.py
73 """)
74 throw new Exception(workflow_details)
75 }
76}
77
78def build_shell_job(job_name, parameters, junit_report_filename=null, junit_report_source_dir='**/') {
79 //Build a job, grab the results if failed and use the results in exception
80 //junit_report_filename: if not null, try to copy this JUnit report first from remote job
81 def common = new com.mirantis.mk.Common()
82 common.printMsg("Start building job '${job_name}' with parameters:", "purple")
83 common.prettyPrint(parameters)
84
85 def job_info = build job: "${job_name}",
86 parameters: parameters,
87 propagate: false
88
Dennis Dmitriev4115ae72018-11-20 13:43:35 +020089 def build_number = job_info.getNumber()
90 def build_url = job_info.getAbsoluteUrl()
91 def build_status = job_info.getResult()
92 try {
93 // Try to grab 'tar.gz' articacts from the shell job'
94 step($class: 'hudson.plugins.copyartifact.CopyArtifact',
95 projectName: job_name,
96 selector: specific("${build_number}"),
97 filter: "**/*.tar.gz",
98 target: '.',
99 flatten: true,
100 fingerprintArtifacts: true)
101 } catch (none) {
102 common.printMsg("No *.tar.gz files found in artifacts of the build ${build_url}", "purple")
103 }
104
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300105 if (job_info.getResult() != "SUCCESS") {
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300106 def job_url = "${build_url}"
107 currentBuild.result = build_status
108 if (junit_report_filename) {
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300109 common.printMsg("Job '${job_url}' failed with status ${build_status}, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300110 step($class: 'hudson.plugins.copyartifact.CopyArtifact',
111 projectName: job_name,
112 selector: specific("${build_number}"),
113 filter: "${junit_report_source_dir}/${junit_report_filename}",
114 target: '.',
115 flatten: true,
116 fingerprintArtifacts: true)
117
118 def String junit_report_xml = readFile("${junit_report_filename}")
119 def String junit_report_xml_pretty = new XmlUtil().serialize(junit_report_xml)
120 def String msg = "Job '${job_url}' failed with status ${build_status}, JUnit report:\n"
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300121 throw new Exception(msg + junit_report_xml_pretty)
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300122 } else {
123 throw new Exception("Job '${job_url}' failed with status ${build_status}, please check the console output.")
124 }
125 }
126}
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300127
128def prepare_working_dir() {
129 println "Clean the working directory ${env.WORKSPACE}"
130 deleteDir()
131
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300132 // do not fail if environment doesn't exists
133 println "Remove environment ${ENV_NAME}"
134 run_cmd("""\
135 dos.py erase ${ENV_NAME} || true
136 """)
137 println "Remove config drive ISO"
138 run_cmd("""\
139 rm /home/jenkins/images/${CFG01_CONFIG_IMAGE_NAME} || true
140 """)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300141
142 run_cmd("""\
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300143 git clone https://github.com/Mirantis/tcp-qa.git ${env.WORKSPACE}
144 if [ -n "$TCP_QA_REFS" ]; then
145 set -e
146 git fetch https://review.gerrithub.io/Mirantis/tcp-qa $TCP_QA_REFS && git checkout FETCH_HEAD || exit \$?
147 fi
148 pip install --upgrade --upgrade-strategy=only-if-needed -r tcp_tests/requirements.txt
Dennis Dmitriev8df35442018-07-31 08:40:20 +0300149 """)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300150}
151
Dennis Dmitrieveb50ce12018-09-27 13:34:32 +0300152def update_working_dir() {
153 // Use to fetch a patchset from gerrit to the working dir
154 run_cmd("""\
155 if [ -n "$TCP_QA_REFS" ]; then
156 set -e
157 git fetch https://review.gerrithub.io/Mirantis/tcp-qa $TCP_QA_REFS && git checkout FETCH_HEAD || exit \$?
158 fi
159 pip install -r tcp_tests/requirements.txt
160 """)
161}
162
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300163def swarm_bootstrap_salt_cluster_devops() {
164 def common = new com.mirantis.mk.Common()
Vladimir Jigulin6a1ef812019-03-21 19:34:07 +0400165 def cookiecutter_template_commit = env.COOKIECUTTER_TEMPLATE_COMMIT ?: is_released_version(env.MCP_VERSION) ? "release/${env.MCP_VERSION}" : 'master'
Dennis Dmitrieve3884652019-03-05 14:26:44 +0200166 def salt_models_system_commit = env.SALT_MODELS_SYSTEM_COMMIT ?: "release/${env.MCP_VERSION}"
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300167 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
168 def mk_pipelines_ref = env.MK_PIPELINES_REF ?: ''
169 def pipeline_library_ref = env.PIPELINE_LIBRARY_REF ?: ''
170 def cookiecutter_ref_change = env.COOKIECUTTER_REF_CHANGE ?: ''
171 def environment_template_ref_change = env.ENVIRONMENT_TEMPLATE_REF_CHANGE ?: ''
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200172 def mcp_salt_repo_url = env.MCP_SALT_REPO_URL ?: ''
173 def mcp_salt_repo_key = env.MCP_SALT_REPO_KEY ?: ''
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300174 def deploy_network_mask = env.DEPLOY_NETWORK_NETMASK ?: ''
Dennis Dmitriev63460042018-12-11 13:08:11 +0200175 def env_ipmi_user = env.IPMI_USER ?: ''
176 def env_ipmi_pass = env.IPMI_PASS ?: ''
177 def env_lab_mgm_iface = env.LAB_MANAGEMENT_IFACE ?: ''
178 def env_lab_ctl_iface = env.LAB_CONTROL_IFACE ?: ''
sgudzb7953382019-02-18 17:59:30 +0200179 def update_repo_custom_tag = env.UPDATE_REPO_CUSTOM_TAG ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300180 def parameters = [
181 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
182 string(name: 'PARENT_WORKSPACE', value: pwd()),
183 string(name: 'LAB_CONFIG_NAME', value: "${LAB_CONFIG_NAME}"),
184 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
185 string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
186 string(name: 'MCP_IMAGE_PATH1604', value: "${MCP_IMAGE_PATH1604}"),
187 string(name: 'IMAGE_PATH_CFG01_DAY01', value: "${IMAGE_PATH_CFG01_DAY01}"),
188 string(name: 'CFG01_CONFIG_IMAGE_NAME', value: "${CFG01_CONFIG_IMAGE_NAME}"),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300189 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
190 string(name: 'PIPELINE_LIBRARY_REF', value: "${pipeline_library_ref}"),
191 string(name: 'MK_PIPELINES_REF', value: "${mk_pipelines_ref}"),
192 string(name: 'COOKIECUTTER_TEMPLATE_COMMIT', value: "${cookiecutter_template_commit}"),
193 string(name: 'SALT_MODELS_SYSTEM_COMMIT', value: "${salt_models_system_commit}"),
194 string(name: 'COOKIECUTTER_REF_CHANGE', value: "${cookiecutter_ref_change}"),
195 string(name: 'ENVIRONMENT_TEMPLATE_REF_CHANGE', value: "${environment_template_ref_change}"),
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200196 string(name: 'MCP_SALT_REPO_URL', value: "${mcp_salt_repo_url}"),
197 string(name: 'MCP_SALT_REPO_KEY', value: "${mcp_salt_repo_key}"),
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300198 string(name: 'DEPLOY_NETWORK_NETMASK', value: "${deploy_network_mask}"),
Dennis Dmitriev63460042018-12-11 13:08:11 +0200199 string(name: 'IPMI_USER', value: env_ipmi_user),
200 string(name: 'IPMI_PASS', value: env_ipmi_pass),
201 string(name: 'LAB_MANAGEMENT_IFACE', value: env_lab_mgm_iface),
202 string(name: 'LAB_CONTROL_IFACE', value: env_lab_ctl_iface),
sgudzb7953382019-02-18 17:59:30 +0200203 string(name: 'UPDATE_REPO_CUSTOM_TAG', value: "${update_repo_custom_tag}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300204 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
205 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300206
207 build_pipeline_job('swarm-bootstrap-salt-cluster-devops', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300208}
209
Dennis Dmitriev07d5b8a2018-10-29 19:43:00 +0200210def swarm_deploy_cicd(String stack_to_install, String install_timeout) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300211 // Run openstack_deploy job on cfg01 Jenkins for specified stacks
212 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300213 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300214 def parameters = [
215 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
216 string(name: 'PARENT_WORKSPACE', value: pwd()),
217 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
218 string(name: 'STACK_INSTALL', value: stack_to_install),
Dennis Dmitriev07d5b8a2018-10-29 19:43:00 +0200219 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300220 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300221 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
222 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300223 build_pipeline_job('swarm-deploy-cicd', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300224}
225
Dennis Dmitriev07d5b8a2018-10-29 19:43:00 +0200226def swarm_deploy_platform(String stack_to_install, String install_timeout) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300227 // Run openstack_deploy job on CICD Jenkins for specified stacks
228 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300229 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300230 def parameters = [
231 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
232 string(name: 'PARENT_WORKSPACE', value: pwd()),
233 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
234 string(name: 'STACK_INSTALL', value: stack_to_install),
Dennis Dmitriev07d5b8a2018-10-29 19:43:00 +0200235 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300236 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300237 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
238 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300239 build_pipeline_job('swarm-deploy-platform', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300240}
241
sgudz2ed95852019-03-01 14:48:52 +0200242def swarm_deploy_platform_non_cicd(String stack_to_install, String install_timeout) {
243 // Run openstack_deploy job on day01 Jenkins for specified stacks
244 def common = new com.mirantis.mk.Common()
245 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
246 def parameters = [
247 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
248 string(name: 'PARENT_WORKSPACE', value: pwd()),
249 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
250 string(name: 'STACK_INSTALL', value: stack_to_install),
251 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
252 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
253 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
254 ]
255 build_pipeline_job('swarm-deploy-platform-without-cicd', parameters)
256}
257
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300258def swarm_run_pytest(String passed_steps) {
259 // Run pytest tests
260 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300261 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Oleksii Butenkoec78c9f2019-04-04 16:26:20 +0300262 def tempest_extra_args = env.TEMPEST_EXTRA_ARGS ?: ''
Tatyana Leontovich05f79402018-11-16 15:04:02 +0200263 def tempest_image_version = env.TEMPEST_IMAGE_VERSION ?: 'pike'
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200264 def tempest_target=env.TEMPEST_TARGET ?: 'gtw01'
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300265 def parameters = [
266 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
267 string(name: 'PASSED_STEPS', value: passed_steps),
268 string(name: 'RUN_TEST_OPTS', value: "${RUN_TEST_OPTS}"),
269 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
270 string(name: 'PARENT_WORKSPACE', value: pwd()),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300271 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300272 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
273 string(name: 'LAB_CONFIG_NAME', value: "${LAB_CONFIG_NAME}"),
274 string(name: 'REPOSITORY_SUITE', value: "${MCP_VERSION}"),
275 string(name: 'MCP_IMAGE_PATH1604', value: "${MCP_IMAGE_PATH1604}"),
276 string(name: 'IMAGE_PATH_CFG01_DAY01', value: "${IMAGE_PATH_CFG01_DAY01}"),
Tatyana Leontovich05f79402018-11-16 15:04:02 +0200277 string(name: 'TEMPEST_IMAGE_VERSION', value: "${tempest_image_version}"),
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200278 string(name: 'TEMPEST_TARGET', value: "${tempest_target}"),
Oleksii Butenkoec78c9f2019-04-04 16:26:20 +0300279 string(name: 'TEMPEST_EXTRA_ARGS', value: "${tempest_extra_args}"),
280
Tatyana Leontovichf3718442018-10-31 13:36:13 +0200281
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300282 ]
283 common.printMsg("Start building job 'swarm-run-pytest' with parameters:", "purple")
284 common.prettyPrint(parameters)
285 build job: 'swarm-run-pytest',
286 parameters: parameters
287}
288
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300289def swarm_testrail_report(String passed_steps) {
290 // Run pytest tests
291 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300292 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Tatyana Leontovichf3718442018-10-31 13:36:13 +0200293 def tempest_test_suite_name = env.TEMPEST_TEST_SUITE_NAME
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300294 def parameters = [
295 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200296 string(name: 'LAB_CONFIG_NAME', value: "${LAB_CONFIG_NAME}"),
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300297 string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
298 string(name: 'PASSED_STEPS', value: passed_steps),
299 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
300 string(name: 'PARENT_WORKSPACE', value: pwd()),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300301 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Tatyana Leontovichf3718442018-10-31 13:36:13 +0200302 string(name: 'TEMPEST_TEST_SUITE_NAME', value: "${tempest_test_suite_name}"),
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300303 ]
304 common.printMsg("Start building job 'swarm-testrail-report' with parameters:", "purple")
305 common.prettyPrint(parameters)
306 build job: 'swarm-testrail-report',
307 parameters: parameters
308}
309
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300310def generate_cookied_model() {
311 def common = new com.mirantis.mk.Common()
312 // do not fail if environment doesn't exists
313 def IPV4_NET_ADMIN=run_cmd_stdout("dos.py net-list ${ENV_NAME} | grep admin-pool01").trim().split().last()
314 def IPV4_NET_CONTROL=run_cmd_stdout("dos.py net-list ${ENV_NAME} | grep private-pool01").trim().split().last()
315 def IPV4_NET_TENANT=run_cmd_stdout("dos.py net-list ${ENV_NAME} | grep tenant-pool01").trim().split().last()
316 def IPV4_NET_EXTERNAL=run_cmd_stdout("dos.py net-list ${ENV_NAME} | grep external-pool01").trim().split().last()
317 println("IPV4_NET_ADMIN=" + IPV4_NET_ADMIN)
318 println("IPV4_NET_CONTROL=" + IPV4_NET_CONTROL)
319 println("IPV4_NET_TENANT=" + IPV4_NET_TENANT)
320 println("IPV4_NET_EXTERNAL=" + IPV4_NET_EXTERNAL)
321
Vladimir Jigulin6a1ef812019-03-21 19:34:07 +0400322 def cookiecuttertemplate_commit = env.COOKIECUTTER_TEMPLATE_COMMIT ?: is_released_version(env.MCP_VERSION) ? "release/${env.MCP_VERSION}" : 'master'
Dennis Dmitrieve3884652019-03-05 14:26:44 +0200323 def saltmodels_system_commit = env.SALT_MODELS_SYSTEM_COMMIT ?: "release/${env.MCP_VERSION}"
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300324 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
325 def environment_template_ref_change = env.ENVIRONMENT_TEMPLATE_REF_CHANGE ?: ''
326 def cookiecutter_ref_change = env.COOKIECUTTER_REF_CHANGE ?: ''
sgudzb7953382019-02-18 17:59:30 +0200327 def update_repo_custom_tag = env.UPDATE_REPO_CUSTOM_TAG ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300328
329 def parameters = [
330 string(name: 'LAB_CONTEXT_NAME', value: "${LAB_CONFIG_NAME}"),
331 string(name: 'CLUSTER_NAME', value: "${LAB_CONFIG_NAME}"),
332 string(name: 'DOMAIN_NAME', value: "${LAB_CONFIG_NAME}.local"),
333 string(name: 'REPOSITORY_SUITE', value: "${env.MCP_VERSION}"),
334 string(name: 'SALT_MODELS_SYSTEM_COMMIT', value: "${saltmodels_system_commit}"),
335 string(name: 'COOKIECUTTER_TEMPLATE_COMMIT', value: "${cookiecuttertemplate_commit}"),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300336 string(name: 'COOKIECUTTER_REF_CHANGE', value: "${cookiecutter_ref_change}"),
337 string(name: 'ENVIRONMENT_TEMPLATE_REF_CHANGE', value: "${environment_template_ref_change}"),
338 string(name: 'TCP_QA_REVIEW', value: "${tcp_qa_refs}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300339 string(name: 'IPV4_NET_ADMIN', value: IPV4_NET_ADMIN),
340 string(name: 'IPV4_NET_CONTROL', value: IPV4_NET_CONTROL),
341 string(name: 'IPV4_NET_TENANT', value: IPV4_NET_TENANT),
342 string(name: 'IPV4_NET_EXTERNAL', value: IPV4_NET_EXTERNAL),
sgudzc97385a2018-11-29 17:01:53 +0200343 string(name: 'IPMI_USER', value: env.IPMI_USER),
344 string(name: 'IPMI_PASS', value: env.IPMI_PASS),
sgudzb7953382019-02-18 17:59:30 +0200345 string(name: 'UPDATE_REPO_CUSTOM_TAG', value: "${update_repo_custom_tag}"),
sgudzc97385a2018-11-29 17:01:53 +0200346 string(name: 'IMAGE_PATH_CFG01_DAY01', value: env.IMAGE_PATH_CFG01_DAY01),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300347 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300348
349 build_shell_job('swarm-cookied-model-generator', parameters, "deploy_generate_model.xml")
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300350}
351
352def generate_configdrive_iso() {
353 def common = new com.mirantis.mk.Common()
354 def SALT_MASTER_IP=run_cmd_stdout("""\
355 export ENV_NAME=${ENV_NAME}
356 . ./tcp_tests/utils/env_salt
357 echo \$SALT_MASTER_IP
358 """).trim().split().last()
359 println("SALT_MASTER_IP=" + SALT_MASTER_IP)
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300360
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300361 def dhcp_ranges_json=run_cmd_stdout("""\
362 fgrep dhcp_ranges ${ENV_NAME}_hardware.ini |
363 fgrep "admin-pool01"|
364 cut -d"=" -f2
365 """).trim().split("\n").last()
366 def dhcp_ranges = new groovy.json.JsonSlurperClassic().parseText(dhcp_ranges_json)
367 def ADMIN_NETWORK_GW = dhcp_ranges['admin-pool01']['gateway']
368 println("ADMIN_NETWORK_GW=" + ADMIN_NETWORK_GW)
369
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300370 def mk_pipelines_ref = env.MK_PIPELINES_REF ?: ''
371 def pipeline_library_ref = env.PIPELINE_LIBRARY_REF ?: ''
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300372 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200373 def mcp_salt_repo_url = env.MCP_SALT_REPO_URL ?: ''
374 def mcp_salt_repo_key = env.MCP_SALT_REPO_KEY ?: ''
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300375 def deploy_network_mask = env.DEPLOY_NETWORK_NETMASK ?: ''
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300376
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300377 def parameters = [
378 string(name: 'CLUSTER_NAME', value: "${LAB_CONFIG_NAME}"),
379 string(name: 'MODEL_URL', value: "http://cz8133.bud.mirantis.net:8098/${LAB_CONFIG_NAME}.git"),
380 string(name: 'MODEL_URL_OBJECT_TYPE', value: "git"),
381 booleanParam(name: 'DOWNLOAD_CONFIG_DRIVE', value: true),
382 string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
Dennis Dmitrieve3884652019-03-05 14:26:44 +0200383 string(name: 'COMMON_SCRIPTS_COMMIT', value: "release/${env.MCP_VERSION}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300384 string(name: 'NODE_NAME', value: "${NODE_NAME}"),
385 string(name: 'CONFIG_DRIVE_ISO_NAME', value: "${CFG01_CONFIG_IMAGE_NAME}"),
386 string(name: 'SALT_MASTER_DEPLOY_IP', value: SALT_MASTER_IP),
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300387 string(name: 'DEPLOY_NETWORK_GW', value: "${ADMIN_NETWORK_GW}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300388 string(name: 'PIPELINE_REPO_URL', value: "https://github.com/Mirantis"),
389 booleanParam(name: 'PIPELINES_FROM_ISO', value: true),
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200390 string(name: 'MCP_SALT_REPO_URL', value: "${mcp_salt_repo_url}"),
391 string(name: 'MCP_SALT_REPO_KEY', value: "${mcp_salt_repo_key}"),
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300392 string(name: 'DEPLOY_NETWORK_NETMASK', value: "${deploy_network_mask}"),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300393 string(name: 'PIPELINE_LIBRARY_REF', value: "${pipeline_library_ref}"),
394 string(name: 'MK_PIPELINES_REF', value: "${mk_pipelines_ref}"),
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300395 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300396 ]
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300397 build_pipeline_job('swarm-create-cfg-config-drive', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300398}
399
Dennis Dmitriev13e804b2018-10-09 19:25:14 +0300400def run_job_on_day01_node(stack_to_install, timeout=2400) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300401 // stack_to_install="core,cicd"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200402 def common = new com.mirantis.mk.Common()
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300403 def stack = "${stack_to_install}"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200404 common.printMsg("Deploy DriveTrain CICD components: ${stack_to_install}", "blue")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300405 try {
406 run_cmd("""\
407 export ENV_NAME=${ENV_NAME}
408 . ./tcp_tests/utils/env_salt
409 . ./tcp_tests/utils/env_jenkins_day01
410 export JENKINS_BUILD_TIMEOUT=${timeout}
411 JOB_PARAMETERS=\"{
412 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
413 \\\"STACK_INSTALL\\\": \\\"${stack}\\\"
414 }\"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200415 JOB_PREFIX="[ ${ENV_NAME}/{build_number}:drivetrain {time} ] "
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300416 python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
417 """)
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200418 // Wait for IO calm down on cluster nodes
419 sleep(60)
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300420 } catch (e) {
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300421 common.printMsg("Product job 'deploy_openstack' failed, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300422 def workflow_details=run_cmd_stdout("""\
423 . ./tcp_tests/utils/env_salt
424 . ./tcp_tests/utils/env_jenkins_day01
425 export JOB_NAME=deploy_openstack
426 export BUILD_NUMBER=lastBuild
427 python ./tcp_tests/utils/get_jenkins_job_stages.py
428 """)
429 throw new Exception(workflow_details)
430 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300431}
432
Dennis Dmitriev13e804b2018-10-09 19:25:14 +0300433def run_job_on_cicd_nodes(stack_to_install, timeout=2400) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300434 // stack_to_install="k8s,calico,stacklight"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200435 def common = new com.mirantis.mk.Common()
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300436 def stack = "${stack_to_install}"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200437 common.printMsg("Deploy Platform components: ${stack_to_install}", "blue")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300438 try {
439 run_cmd("""\
440 export ENV_NAME=${ENV_NAME}
441 . ./tcp_tests/utils/env_salt
442 . ./tcp_tests/utils/env_jenkins_cicd
443 export JENKINS_BUILD_TIMEOUT=${timeout}
444 JOB_PARAMETERS=\"{
445 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
446 \\\"STACK_INSTALL\\\": \\\"${stack}\\\"
447 }\"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200448 JOB_PREFIX="[ ${ENV_NAME}/{build_number}:platform {time} ] "
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300449 python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300450 """)
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200451 // Wait for IO calm down on cluster nodes
452 sleep(60)
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300453 } catch (e) {
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300454 common.printMsg("Product job 'deploy_openstack' failed, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300455 def workflow_details=run_cmd_stdout("""\
456 . ./tcp_tests/utils/env_salt
457 . ./tcp_tests/utils/env_jenkins_cicd
458 export JOB_NAME=deploy_openstack
459 export BUILD_NUMBER=lastBuild
460 python ./tcp_tests/utils/get_jenkins_job_stages.py
461 """)
462 throw new Exception(workflow_details)
463 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300464}
465
466def sanity_check_component(stack) {
467 // Run sanity check for the component ${stack}.
468 // Result will be stored in JUnit XML file deploy_${stack}.xml
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300469 try {
470 run_cmd("""\
471 py.test --junit-xml=deploy_${stack}.xml -m check_${stack}
472 """)
473 } catch (e) {
474 def String junit_report_xml = readFile("deploy_${stack}.xml")
475 def String junit_report_xml_pretty = new XmlUtil().serialize(junit_report_xml)
476 def String msg = "Sanity check for '${stack}' failed, JUnit report:\n"
477 throw new Exception(msg + junit_report_xml_pretty)
478 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300479}
480
Dennis Dmitrievefe5c0b2018-10-24 20:35:26 +0300481def download_logs(archive_name_prefix) {
482 // Archive and download logs and debug info from salt nodes in the lab
483 // Do not fail in case of error to not lose the original error from the parent exception.
484 def common = new com.mirantis.mk.Common()
485 common.printMsg("Downloading nodes logs by ${archive_name_prefix}", "blue")
486 run_cmd("""\
487 export TESTS_CONFIGS=\$(pwd)/${ENV_NAME}_salt_deployed.ini
488 ./tcp_tests/utils/get_logs.py --archive-name-prefix ${archive_name_prefix} || true
489 """)
490}
491
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300492def devops_snapshot_info(snapshot_name) {
493 // Print helper message after snapshot
494 def common = new com.mirantis.mk.Common()
495
496 def SALT_MASTER_IP=run_cmd_stdout("""\
497 . ./tcp_tests/utils/env_salt
498 echo \$SALT_MASTER_IP
499 """).trim().split().last()
500 def login = "root" // set fixed 'root' login for now
501 def password = "r00tme" // set fixed 'root' login for now
502 def key_file = "${env.WORKSPACE}/id_rsa" // set fixed path in the WORKSPACE
503 def VENV_PATH='/home/jenkins/fuel-devops30'
504
505 common.printMsg("""\
506#########################
507# To revert the snapshot:
508#########################
509. ${VENV_PATH}/bin/activate;
510dos.py revert ${ENV_NAME} ${snapshot_name};
511dos.py resume ${ENV_NAME};
512# dos.py time-sync ${ENV_NAME}; # Optional\n
513ssh -i ${key_file} ${login}@${SALT_MASTER_IP} # Optional password: ${password}
514""", "cyan")
515}
516
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300517def devops_snapshot(stack) {
518 // Make the snapshot with name "${stack}_deployed"
519 // for all VMs in the environment.
520 // If oslo_config INI file ${ENV_NAME}_salt_deployed.ini exists,
521 // then make a copy for the created snapshot to allow the system
522 // tests to revert this snapshot along with the metadata from the INI file.
523 run_cmd("""\
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300524 set -ex
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300525 dos.py suspend ${ENV_NAME}
526 dos.py snapshot ${ENV_NAME} ${stack}_deployed
527 dos.py resume ${ENV_NAME}
Dennis Dmitriev1fed6662018-07-27 18:22:13 +0300528 sleep 20 # Wait for I/O on the host calms down
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300529
530 CFG01_NAME=\$(dos.py show-resources ${ENV_NAME} | grep ^cfg01 | cut -d" " -f1)
531 dos.py time-sync ${ENV_NAME} --skip-sync \${CFG01_NAME}
532
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300533 if [ -f \$(pwd)/${ENV_NAME}_salt_deployed.ini ]; then
534 cp \$(pwd)/${ENV_NAME}_salt_deployed.ini \$(pwd)/${ENV_NAME}_${stack}_deployed.ini
535 fi
Dennis Dmitriev8df35442018-07-31 08:40:20 +0300536 """)
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300537 devops_snapshot_info("${stack}_deployed")
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300538}
539
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300540def get_steps_list(steps) {
541 // Make a list from comma separated string
542 return steps.split(',').collect { it.split(':')[0] }
543}
544
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300545def create_xml_report(String filename, String classname, String name, String status='success', String status_message='', String text='', String stdout='', String stderr='') {
546 // <filename> is name of the XML report file that will be created
547 // <status> is one of the 'success', 'skipped', 'failure' or 'error'
548 // 'error' status is assumed as 'Blocker' in TestRail reporter
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300549
550 // Replace '<' and '>' to '&lt;' and '&gt;' to avoid conflicts between xml tags in the message and JUnit report
551 def String text_filtered = text.replaceAll("<","&lt;").replaceAll(">", "&gt;")
552
Dennis Dmitriev39666082018-08-29 15:30:45 +0300553 def script = """\
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300554<?xml version=\"1.0\" encoding=\"utf-8\"?>
555 <testsuite>
556 <testcase classname=\"${classname}\" name=\"${name}\" time=\"0\">
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300557 <${status} message=\"${status_message}\">${text_filtered}</${status}>
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300558 <system-out>${stdout}</system-out>
559 <system-err>${stderr}</system-err>
560 </testcase>
561 </testsuite>
Dennis Dmitriev39666082018-08-29 15:30:45 +0300562"""
563 writeFile(file: filename, text: script, encoding: "UTF-8")
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300564}
565
Dennis Dmitrievaf59f7e2019-04-02 11:54:06 +0000566def upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300567 def venvPath = '/home/jenkins/venv_testrail_reporter'
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200568 def testPlanDesc = env.LAB_CONFIG_NAME
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300569 def testrailURL = "https://mirantis.testrail.com"
570 def testrailProject = "Mirantis Cloud Platform"
Vladimir Jigulinb02dcc52019-04-02 15:57:53 +0400571 def testPlanNamePrefix = env.TEST_PLAN_NAME_PREFIX ?: "[MCP-Q2]System"
572 def testPlanName = "${testPlanNamePrefix}-${MCP_VERSION}-${new Date().format('yyyy-MM-dd')}"
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300573 def testrailMilestone = "MCP1.1"
Dennis Dmitriev707b2152018-10-23 19:12:48 +0300574 def testrailCaseMaxNameLenght = 250
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300575 def jobURL = env.BUILD_URL
576
577 def reporterOptions = [
578 "--verbose",
579 "--env-description \"${testPlanDesc}\"",
580 "--testrail-run-update",
581 "--testrail-url \"${testrailURL}\"",
582 "--testrail-user \"\${TESTRAIL_USER}\"",
583 "--testrail-password \"\${TESTRAIL_PASSWORD}\"",
584 "--testrail-project \"${testrailProject}\"",
585 "--testrail-plan-name \"${testPlanName}\"",
586 "--testrail-milestone \"${testrailMilestone}\"",
587 "--testrail-suite \"${testSuiteName}\"",
588 "--xunit-name-template \"${methodname}\"",
589 "--testrail-name-template \"${testrail_name_template}\"",
590 "--test-results-link \"${jobURL}\"",
Dennis Dmitriev707b2152018-10-23 19:12:48 +0300591 "--testrail-case-max-name-lenght ${testrailCaseMaxNameLenght}",
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300592 ] + reporter_extra_options
593
594 def script = """
595 . ${venvPath}/bin/activate
596 set -ex
Dmitry Tyzhnenko80ce0202019-02-07 13:27:19 +0200597 report ${reporterOptions.join(' ')} '${report_name}'
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300598 """
599
600 def testrail_cred_id = params.TESTRAIL_CRED ?: 'testrail_system_tests'
601
602 withCredentials([
603 [$class : 'UsernamePasswordMultiBinding',
604 credentialsId : testrail_cred_id,
605 passwordVariable: 'TESTRAIL_PASSWORD',
606 usernameVariable: 'TESTRAIL_USER']
607 ]) {
Dennis Dmitrievfbf42272018-10-23 00:19:50 +0300608 return run_cmd_stdout(script)
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300609 }
610}
611
612
613def create_deploy_result_report(deploy_expected_stacks, result, text) {
614 def STATUS_MAP = ['SUCCESS': 'success', 'FAILURE': 'failure', 'UNSTABLE': 'failure', 'ABORTED': 'error']
615 def classname = "Deploy"
616 def name = "deployment_${ENV_NAME}"
Dennis Dmitriev39666082018-08-29 15:30:45 +0300617 def filename = "${name}.xml"
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300618 def status = STATUS_MAP[result ?: 'FAILURE'] // currentBuild.result *must* be set at the finish of the try/catch
619 create_xml_report(filename, classname, name, status, "Deploy components: ${deploy_expected_stacks}", text, '', '')
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300620}