blob: c42d43f09fc5506149355445bc354d153b0b45ad [file] [log] [blame]
Dennis Dmitrievb3b37492018-07-08 21:23:00 +03001package com.mirantis.system_qa
2
Dennis Dmitriev27a96792018-07-30 07:52:03 +03003import groovy.xml.XmlUtil
Dennis Dmitrievb3b37492018-07-08 21:23:00 +03004
Vladimir Jigulin6a1ef812019-03-21 19:34:07 +04005def is_released_version(version) {
6 return Character.isDigit(version.charAt(0))
7}
8
Dennis Dmitriev4c383472019-04-12 13:58:06 +03009def verbose_sh(String script, Boolean returnStatus=false, Boolean returnStdout=false, Boolean verboseStdout=false) {
10 def common = new com.mirantis.mk.Common()
11 common.printMsg("Run shell command:\n" + script, "blue")
12 def result = sh(script: script, returnStatus: returnStatus, returnStdout: returnStdout)
13 if (verboseStdout) {
14 common.printMsg("Output:\n" + result, "cyan")
15 }
16 return result
17}
18
Dennis Dmitrieveb50ce12018-09-27 13:34:32 +030019def run_sh(String cmd) {
20 // run shell script without catching any output
21 def common = new com.mirantis.mk.Common()
22 common.printMsg("Run shell command:\n" + cmd, "blue")
23 def VENV_PATH='/home/jenkins/fuel-devops30'
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020024 def script = """\
Dennis Dmitrieveb50ce12018-09-27 13:34:32 +030025 set -ex;
26 . ${VENV_PATH}/bin/activate;
27 bash -c '${cmd.stripIndent()}'
28 """
29 return sh(script: script)
30}
31
Dennis Dmitriev8df35442018-07-31 08:40:20 +030032def run_cmd(String cmd, Boolean returnStdout=false) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030033 def common = new com.mirantis.mk.Common()
34 common.printMsg("Run shell command:\n" + cmd, "blue")
35 def VENV_PATH='/home/jenkins/fuel-devops30'
Dennis Dmitriev27a96792018-07-30 07:52:03 +030036 def stderr_path = "/tmp/${JOB_NAME}_${BUILD_NUMBER}_stderr.log"
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020037 def script = """#!/bin/bash
38 set +x
39 echo 'activate python virtualenv ${VENV_PATH}'
40 . ${VENV_PATH}/bin/activate
41 bash -c -e -x '${cmd.stripIndent()}' 2>${stderr_path}
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030042 """
Dennis Dmitriev27a96792018-07-30 07:52:03 +030043 try {
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020044 def stdout = sh(script: script, returnStdout: returnStdout)
45 def stderr = readFile("${stderr_path}")
46 def error_message = "\n<<<<<< STDERR: >>>>>>\n" + stderr
47 common.printMsg(error_message, "yellow")
48 common.printMsg("", "reset")
49 return stdout
Dennis Dmitriev27a96792018-07-30 07:52:03 +030050 } catch (e) {
Dennis Dmitriev8df35442018-07-31 08:40:20 +030051 def stderr = readFile("${stderr_path}")
52 def error_message = e.message + "\n<<<<<< STDERR: >>>>>>\n" + stderr
Dmitry Tyzhnenkob39de052019-03-21 17:05:07 +020053 common.printMsg(error_message, "red")
54 common.printMsg("", "reset")
Dennis Dmitriev8df35442018-07-31 08:40:20 +030055 throw new Exception(error_message)
Dennis Dmitriev27a96792018-07-30 07:52:03 +030056 } finally {
Dennis Dmitriev8df35442018-07-31 08:40:20 +030057 sh(script: "rm ${stderr_path} || true")
Dennis Dmitriev27a96792018-07-30 07:52:03 +030058 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030059}
60
61def run_cmd_stdout(cmd) {
Dennis Dmitriev8df35442018-07-31 08:40:20 +030062 return run_cmd(cmd, true)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +030063}
64
Dennis Dmitriev27a96792018-07-30 07:52:03 +030065def build_pipeline_job(job_name, parameters) {
66 //Build a job, grab the results if failed and use the results in exception
67 def common = new com.mirantis.mk.Common()
68 common.printMsg("Start building job '${job_name}' with parameters:", "purple")
69 common.prettyPrint(parameters)
70
71 def job_info = build job: "${job_name}",
72 parameters: parameters,
73 propagate: false
74
75 if (job_info.getResult() != "SUCCESS") {
76 currentBuild.result = job_info.getResult()
77 def build_number = job_info.getNumber()
Dennis Dmitrievb08c0772018-10-17 15:10:26 +030078 common.printMsg("Job '${job_name}' failed, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +030079 def workflow_details=run_cmd_stdout("""\
80 export JOB_NAME=${job_name}
81 export BUILD_NUMBER=${build_number}
82 python ./tcp_tests/utils/get_jenkins_job_stages.py
83 """)
84 throw new Exception(workflow_details)
85 }
86}
87
88def build_shell_job(job_name, parameters, junit_report_filename=null, junit_report_source_dir='**/') {
89 //Build a job, grab the results if failed and use the results in exception
90 //junit_report_filename: if not null, try to copy this JUnit report first from remote job
91 def common = new com.mirantis.mk.Common()
92 common.printMsg("Start building job '${job_name}' with parameters:", "purple")
93 common.prettyPrint(parameters)
94
95 def job_info = build job: "${job_name}",
96 parameters: parameters,
97 propagate: false
98
Dennis Dmitriev4115ae72018-11-20 13:43:35 +020099 def build_number = job_info.getNumber()
100 def build_url = job_info.getAbsoluteUrl()
101 def build_status = job_info.getResult()
102 try {
103 // Try to grab 'tar.gz' articacts from the shell job'
104 step($class: 'hudson.plugins.copyartifact.CopyArtifact',
105 projectName: job_name,
106 selector: specific("${build_number}"),
107 filter: "**/*.tar.gz",
108 target: '.',
109 flatten: true,
110 fingerprintArtifacts: true)
111 } catch (none) {
112 common.printMsg("No *.tar.gz files found in artifacts of the build ${build_url}", "purple")
113 }
114
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300115 if (job_info.getResult() != "SUCCESS") {
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300116 def job_url = "${build_url}"
117 currentBuild.result = build_status
118 if (junit_report_filename) {
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300119 common.printMsg("Job '${job_url}' failed with status ${build_status}, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300120 step($class: 'hudson.plugins.copyartifact.CopyArtifact',
121 projectName: job_name,
122 selector: specific("${build_number}"),
123 filter: "${junit_report_source_dir}/${junit_report_filename}",
124 target: '.',
125 flatten: true,
126 fingerprintArtifacts: true)
127
128 def String junit_report_xml = readFile("${junit_report_filename}")
129 def String junit_report_xml_pretty = new XmlUtil().serialize(junit_report_xml)
130 def String msg = "Job '${job_url}' failed with status ${build_status}, JUnit report:\n"
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300131 throw new Exception(msg + junit_report_xml_pretty)
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300132 } else {
133 throw new Exception("Job '${job_url}' failed with status ${build_status}, please check the console output.")
134 }
135 }
136}
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300137
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300138def prepare_working_dir(env_manager) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300139 println "Clean the working directory ${env.WORKSPACE}"
140 deleteDir()
141
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300142 if (env_manager == 'devops') {
143 // do not fail if environment doesn't exists
144 println "Remove fuel-devops environment '${ENV_NAME}'"
145 run_cmd("""\
146 dos.py erase ${ENV_NAME} || true
147 """)
148 } else if (env_manager == 'heat') {
149 // delete heat stack
150 println "Remove heat stack '${ENV_NAME}'"
151 withCredentials([
152 [$class : 'UsernamePasswordMultiBinding',
153 credentialsId : env.OS_CREDENTIALS,
154 passwordVariable: 'OS_PASSWORD',
155 usernameVariable: 'OS_USERNAME']
156 ]) {
157 run_cmd("""\
158 export OS_IDENTITY_API_VERSION=3
159 export OS_AUTH_URL=${OS_AUTH_URL}
160 export OS_USERNAME=${OS_USERNAME}
161 export OS_PASSWORD=${OS_PASSWORD}
162 export OS_PROJECT_NAME=${OS_PROJECT_NAME}
163 export OS_USER_DOMAIN_NAME=${OS_USER_DOMAIN_NAME}
164 openstack --insecure stack delete -y ${ENV_NAME} || true
Hanna Arhipova8b45a712020-05-25 14:21:58 +0300165 timeout 20m /bin/bash -c "while openstack --insecure stack show ${ENV_NAME} -f value -c stack_status; do sleep 10; done"
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300166 """)
167 }
168
169 } else {
170 throw new Exception("Unknown env_manager: '${env_manager}'")
171 }
172
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300173 println "Remove config drive ISO"
174 run_cmd("""\
175 rm /home/jenkins/images/${CFG01_CONFIG_IMAGE_NAME} || true
176 """)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300177
178 run_cmd("""\
Hanna Arhipova5173aad2019-11-11 12:42:31 +0200179 git clone https://gerrit.mcp.mirantis.com/mcp/tcp-qa ${env.WORKSPACE}
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300180 if [ -n "$TCP_QA_REFS" ]; then
181 set -e
Hanna Arhipova5173aad2019-11-11 12:42:31 +0200182 git fetch https://gerrit.mcp.mirantis.com/mcp/tcp-qa $TCP_QA_REFS && git checkout FETCH_HEAD || exit \$?
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300183 fi
184 pip install --upgrade --upgrade-strategy=only-if-needed -r tcp_tests/requirements.txt
Dennis Dmitriev8df35442018-07-31 08:40:20 +0300185 """)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300186}
187
Dennis Dmitriev056ecbf2019-05-20 15:36:29 +0300188def update_working_dir(Boolean updateRequirements=true) {
Dennis Dmitrieveb50ce12018-09-27 13:34:32 +0300189 // Use to fetch a patchset from gerrit to the working dir
190 run_cmd("""\
191 if [ -n "$TCP_QA_REFS" ]; then
192 set -e
Hanna Arhipova5173aad2019-11-11 12:42:31 +0200193 git reset --hard && git fetch https://gerrit.mcp.mirantis.com/mcp/tcp-qa $TCP_QA_REFS && git checkout FETCH_HEAD || exit \$?
Dennis Dmitriev056ecbf2019-05-20 15:36:29 +0300194 fi""")
195 if (updateRequirements) {
196 run_cmd("""\
197 pip install -r tcp_tests/requirements.txt
198 """)
199 }
Dennis Dmitrieveb50ce12018-09-27 13:34:32 +0300200}
201
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300202def swarm_bootstrap_salt_cluster_devops() {
203 def common = new com.mirantis.mk.Common()
Hanna Arhipovac2965a42020-01-21 12:27:33 +0200204 def cookiecutter_template_commit = env.COOKIECUTTER_TEMPLATE_COMMIT ?: is_released_version(env.MCP_VERSION) ? "refs/tags/${env.MCP_VERSION}" : 'master'
205 def salt_models_system_commit = env.SALT_MODELS_SYSTEM_COMMIT ?: "refs/tags/${env.MCP_VERSION}"
Tatyana Leontovich8ac26d72019-05-15 23:33:38 +0300206 def jenkins_pipelines_branch = env.JENKINS_PIPELINE_BRANCH ?: ''
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300207 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
208 def mk_pipelines_ref = env.MK_PIPELINES_REF ?: ''
209 def pipeline_library_ref = env.PIPELINE_LIBRARY_REF ?: ''
210 def cookiecutter_ref_change = env.COOKIECUTTER_REF_CHANGE ?: ''
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300211 def mcp_common_scripts_refs = env.MCP_COMMON_SCRIPTS_REFS ?: ''
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300212 def environment_template_ref_change = env.ENVIRONMENT_TEMPLATE_REF_CHANGE ?: ''
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200213 def mcp_salt_repo_url = env.MCP_SALT_REPO_URL ?: ''
214 def mcp_salt_repo_key = env.MCP_SALT_REPO_KEY ?: ''
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300215 def deploy_network_mask = env.DEPLOY_NETWORK_NETMASK ?: ''
Dennis Dmitriev63460042018-12-11 13:08:11 +0200216 def env_ipmi_user = env.IPMI_USER ?: ''
217 def env_ipmi_pass = env.IPMI_PASS ?: ''
Oleksii Butenkoab538de2019-09-23 18:26:57 +0300218 def env_cisco_pass = env.CISCO_PASS ?: ''
Dennis Dmitriev63460042018-12-11 13:08:11 +0200219 def env_lab_mgm_iface = env.LAB_MANAGEMENT_IFACE ?: ''
220 def env_lab_ctl_iface = env.LAB_CONTROL_IFACE ?: ''
sgudzb7953382019-02-18 17:59:30 +0200221 def update_repo_custom_tag = env.UPDATE_REPO_CUSTOM_TAG ?: ''
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300222 def update_version = env.UPDATE_VERSION ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300223 def parameters = [
224 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
225 string(name: 'PARENT_WORKSPACE', value: pwd()),
226 string(name: 'LAB_CONFIG_NAME', value: "${LAB_CONFIG_NAME}"),
227 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
228 string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
229 string(name: 'MCP_IMAGE_PATH1604', value: "${MCP_IMAGE_PATH1604}"),
230 string(name: 'IMAGE_PATH_CFG01_DAY01', value: "${IMAGE_PATH_CFG01_DAY01}"),
231 string(name: 'CFG01_CONFIG_IMAGE_NAME', value: "${CFG01_CONFIG_IMAGE_NAME}"),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300232 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
233 string(name: 'PIPELINE_LIBRARY_REF', value: "${pipeline_library_ref}"),
234 string(name: 'MK_PIPELINES_REF', value: "${mk_pipelines_ref}"),
235 string(name: 'COOKIECUTTER_TEMPLATE_COMMIT', value: "${cookiecutter_template_commit}"),
236 string(name: 'SALT_MODELS_SYSTEM_COMMIT', value: "${salt_models_system_commit}"),
237 string(name: 'COOKIECUTTER_REF_CHANGE', value: "${cookiecutter_ref_change}"),
238 string(name: 'ENVIRONMENT_TEMPLATE_REF_CHANGE', value: "${environment_template_ref_change}"),
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200239 string(name: 'MCP_SALT_REPO_URL', value: "${mcp_salt_repo_url}"),
240 string(name: 'MCP_SALT_REPO_KEY', value: "${mcp_salt_repo_key}"),
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300241 string(name: 'DEPLOY_NETWORK_NETMASK', value: "${deploy_network_mask}"),
Dennis Dmitriev63460042018-12-11 13:08:11 +0200242 string(name: 'IPMI_USER', value: env_ipmi_user),
243 string(name: 'IPMI_PASS', value: env_ipmi_pass),
Oleksii Butenkoab538de2019-09-23 18:26:57 +0300244 string(name: 'CISCO_PASS', value: env_cisco_pass),
Dennis Dmitriev63460042018-12-11 13:08:11 +0200245 string(name: 'LAB_MANAGEMENT_IFACE', value: env_lab_mgm_iface),
246 string(name: 'LAB_CONTROL_IFACE', value: env_lab_ctl_iface),
sgudzb7953382019-02-18 17:59:30 +0200247 string(name: 'UPDATE_REPO_CUSTOM_TAG', value: "${update_repo_custom_tag}"),
Tatyana Leontovich8ac26d72019-05-15 23:33:38 +0300248 string(name: 'JENKINS_PIPELINE_BRANCH', value: "${jenkins_pipelines_branch}"),
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300249 string(name: 'MCP_COMMON_SCRIPTS_REFS', value: "${mcp_common_scripts_refs}"),
250 string(name: 'UPDATE_VERSION', value: "${update_version}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300251 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
252 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300253
254 build_pipeline_job('swarm-bootstrap-salt-cluster-devops', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300255}
256
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300257def swarm_bootstrap_salt_cluster_heat(String jenkins_slave_node_name) {
258 // jenkins_slave_node_name
259 def common = new com.mirantis.mk.Common()
Hanna Arhipovac2965a42020-01-21 12:27:33 +0200260 def cookiecutter_template_commit = env.COOKIECUTTER_TEMPLATE_COMMIT ?: "refs/tags/${env.MCP_VERSION}"
261 def salt_models_system_commit = env.SALT_MODELS_SYSTEM_COMMIT ?: "refs/tags/${env.MCP_VERSION}"
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300262 def mcp_common_scripts_refs = env.MCP_COMMON_SCRIPTS_REFS ?: ''
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300263 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
264 def mk_pipelines_ref = env.MK_PIPELINES_REF ?: ''
Tatyana Leontovich8ac26d72019-05-15 23:33:38 +0300265 def jenkins_pipelines_branch = env.JENKINS_PIPELINE_BRANCH ?: ''
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300266 def pipeline_library_ref = env.PIPELINE_LIBRARY_REF ?: ''
267 def cookiecutter_ref_change = env.COOKIECUTTER_REF_CHANGE ?: ''
268 def environment_template_ref_change = env.ENVIRONMENT_TEMPLATE_REF_CHANGE ?: ''
269 def mcp_salt_repo_url = env.MCP_SALT_REPO_URL ?: ''
270 def mcp_salt_repo_key = env.MCP_SALT_REPO_KEY ?: ''
271 def env_ipmi_user = env.IPMI_USER ?: ''
272 def env_ipmi_pass = env.IPMI_PASS ?: ''
Oleksii Butenkoab538de2019-09-23 18:26:57 +0300273 def env_cisco_pass = env.CISCO_PASS ?: ''
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300274 def env_lab_mgm_iface = env.LAB_MANAGEMENT_IFACE ?: ''
275 def env_lab_ctl_iface = env.LAB_CONTROL_IFACE ?: ''
276 def update_repo_custom_tag = env.UPDATE_REPO_CUSTOM_TAG ?: ''
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300277 def update_version = env.UPDATE_VERSION ?: ''
Hanna Arhipova258094b2021-02-11 14:49:52 +0200278 def deploy_empty_node = env.DEPLOY_EMPTY_NODE ?: false
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300279 def parameters = [
280 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
281 string(name: 'JENKINS_SLAVE_NODE_NAME', value: jenkins_slave_node_name),
282 string(name: 'PARENT_WORKSPACE', value: pwd()),
283 string(name: 'LAB_CONFIG_NAME', value: "${LAB_CONFIG_NAME}"),
284 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
285 string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
286 string(name: 'MCP_IMAGE_PATH1604', value: "${MCP_IMAGE_PATH1604}"),
287 string(name: 'IMAGE_PATH_CFG01_DAY01', value: "${IMAGE_PATH_CFG01_DAY01}"),
288 string(name: 'CFG01_CONFIG_IMAGE_NAME', value: "${CFG01_CONFIG_IMAGE_NAME}"),
289 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
290 string(name: 'PIPELINE_LIBRARY_REF', value: "${pipeline_library_ref}"),
291 string(name: 'MK_PIPELINES_REF', value: "${mk_pipelines_ref}"),
292 string(name: 'COOKIECUTTER_TEMPLATE_COMMIT', value: "${cookiecutter_template_commit}"),
293 string(name: 'SALT_MODELS_SYSTEM_COMMIT', value: "${salt_models_system_commit}"),
294 string(name: 'COOKIECUTTER_REF_CHANGE', value: "${cookiecutter_ref_change}"),
295 string(name: 'ENVIRONMENT_TEMPLATE_REF_CHANGE', value: "${environment_template_ref_change}"),
296 string(name: 'MCP_SALT_REPO_URL', value: "${mcp_salt_repo_url}"),
297 string(name: 'MCP_SALT_REPO_KEY', value: "${mcp_salt_repo_key}"),
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300298 string(name: 'MCP_COMMON_SCRIPTS_REFS', value: "${mcp_common_scripts_refs}"),
299 string(name: 'UPDATE_VERSION', value: "${update_version}"),
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300300 string(name: 'IPMI_USER', value: env_ipmi_user),
301 string(name: 'IPMI_PASS', value: env_ipmi_pass),
Oleksii Butenkoab538de2019-09-23 18:26:57 +0300302 string(name: 'CISCO_PASS', value: env_cisco_pass),
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300303 string(name: 'LAB_MANAGEMENT_IFACE', value: env_lab_mgm_iface),
304 string(name: 'LAB_CONTROL_IFACE', value: env_lab_ctl_iface),
305 string(name: 'UPDATE_REPO_CUSTOM_TAG', value: "${update_repo_custom_tag}"),
306 string(name: 'OS_AUTH_URL', value: "${OS_AUTH_URL}"),
307 string(name: 'OS_PROJECT_NAME', value: "${OS_PROJECT_NAME}"),
308 string(name: 'OS_USER_DOMAIN_NAME', value: "${OS_USER_DOMAIN_NAME}"),
309 string(name: 'OS_CREDENTIALS', value: "${OS_CREDENTIALS}"),
310 string(name: 'LAB_PARAM_DEFAULTS', value: "${LAB_PARAM_DEFAULTS}"),
Tatyana Leontovich8ac26d72019-05-15 23:33:38 +0300311 string(name: 'JENKINS_PIPELINE_BRANCH', value: "${jenkins_pipelines_branch}"),
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300312 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
Hanna Arhipova258094b2021-02-11 14:49:52 +0200313 booleanParam(name: 'DEPLOY_EMPTY_NODE', value: deploy_empty_node),
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300314 ]
315
316 build_pipeline_job('swarm-bootstrap-salt-cluster-heat', parameters)
317}
318
PGlazoved19b352020-05-21 16:42:27 +0400319def swarm_deploy_cicd(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, String batch_size, Boolean dist_upgrade_nodes, Boolean upgrade_saltstack) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300320 // Run openstack_deploy job on cfg01 Jenkins for specified stacks
321 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300322 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300323 def parameters = [
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300324 string(name: 'PARENT_NODE_NAME', value: jenkins_slave_node_name),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300325 string(name: 'PARENT_WORKSPACE', value: pwd()),
326 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
327 string(name: 'STACK_INSTALL', value: stack_to_install),
Dennis Dmitriev07d5b8a2018-10-29 19:43:00 +0200328 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300329 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Hanna Arhipova6f493522019-12-25 04:23:28 +0200330 string(name: 'BATCH_SIZE', value: batch_size),
Dmitriy Kruglov758f99c2020-02-04 12:04:31 +0100331 booleanParam(name: 'DIST_UPGRADE_NODES', value: dist_upgrade_nodes),
PGlazoved19b352020-05-21 16:42:27 +0400332 booleanParam(name: 'UPGRADE_SALTSTACK', value: upgrade_saltstack),
Dennis Dmitriev02447412019-04-17 18:02:46 +0300333 booleanParam(name: 'MAKE_SNAPSHOT_STAGES', value: make_snapshot_stages),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300334 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
335 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300336 build_pipeline_job('swarm-deploy-cicd', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300337}
338
Hanna Arhipovaddd35a22020-05-27 12:39:26 +0300339def swarm_deploy_platform(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, String batch_size, Boolean dist_upgrade_nodes, Boolean upgrade_saltstack) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300340 // Run openstack_deploy job on CICD Jenkins for specified stacks
341 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300342 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300343 def parameters = [
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300344 string(name: 'PARENT_NODE_NAME', value: jenkins_slave_node_name),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300345 string(name: 'PARENT_WORKSPACE', value: pwd()),
346 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
347 string(name: 'STACK_INSTALL', value: stack_to_install),
Dennis Dmitriev07d5b8a2018-10-29 19:43:00 +0200348 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300349 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Hanna Arhipova6f493522019-12-25 04:23:28 +0200350 string(name: 'BATCH_SIZE', value: batch_size),
Dmitriy Kruglov758f99c2020-02-04 12:04:31 +0100351 booleanParam(name: 'DIST_UPGRADE_NODES', value: dist_upgrade_nodes),
PGlazoved19b352020-05-21 16:42:27 +0400352 booleanParam(name: 'UPGRADE_SALTSTACK', value: upgrade_saltstack),
Dennis Dmitriev02447412019-04-17 18:02:46 +0300353 booleanParam(name: 'MAKE_SNAPSHOT_STAGES', value: make_snapshot_stages),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300354 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
355 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300356 build_pipeline_job('swarm-deploy-platform', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300357}
358
sgudz2ed95852019-03-01 14:48:52 +0200359def swarm_deploy_platform_non_cicd(String stack_to_install, String install_timeout) {
360 // Run openstack_deploy job on day01 Jenkins for specified stacks
361 def common = new com.mirantis.mk.Common()
362 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
363 def parameters = [
364 string(name: 'PARENT_NODE_NAME', value: "${NODE_NAME}"),
365 string(name: 'PARENT_WORKSPACE', value: pwd()),
366 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
367 string(name: 'STACK_INSTALL', value: stack_to_install),
368 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
369 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
370 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
371 ]
372 build_pipeline_job('swarm-deploy-platform-without-cicd', parameters)
373}
374
Dennis Dmitriev02447412019-04-17 18:02:46 +0300375def swarm_run_pytest(String passed_steps, String jenkins_slave_node_name, Boolean make_snapshot_stages) {
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300376 // Run pytest tests
377 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300378 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Oleksii Butenkoec78c9f2019-04-04 16:26:20 +0300379 def tempest_extra_args = env.TEMPEST_EXTRA_ARGS ?: ''
Tatyana Leontovich05f79402018-11-16 15:04:02 +0200380 def tempest_image_version = env.TEMPEST_IMAGE_VERSION ?: 'pike'
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200381 def tempest_target=env.TEMPEST_TARGET ?: 'gtw01'
Andrew Baraniukd1586342019-06-12 11:41:20 +0300382 def tempest_pattern=env.TEMPEST_PATTERN ?: 'tempest'
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300383 def parameters = [
384 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
385 string(name: 'PASSED_STEPS', value: passed_steps),
Hanna Arhipova798759d2020-02-19 14:07:50 +0200386 text(name: 'RUN_TEST_OPTS', value: "${RUN_TEST_OPTS}"),
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300387 string(name: 'PARENT_NODE_NAME', value: jenkins_slave_node_name),
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300388 string(name: 'PARENT_WORKSPACE', value: pwd()),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300389 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300390 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
Tatyana Leontovich05f79402018-11-16 15:04:02 +0200391 string(name: 'TEMPEST_IMAGE_VERSION', value: "${tempest_image_version}"),
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200392 string(name: 'TEMPEST_TARGET', value: "${tempest_target}"),
Hanna Arhipovac8875442020-07-10 13:22:59 +0300393 string(name: 'TEMPEST_PATTERN', value: "${tempest_pattern}"),
394 string(name: 'TEMPEST_EXTRA_ARGS', value: "${tempest_extra_args}"),
Dennis Dmitriev02447412019-04-17 18:02:46 +0300395 booleanParam(name: 'MAKE_SNAPSHOT_STAGES', value: make_snapshot_stages),
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300396 ]
397 common.printMsg("Start building job 'swarm-run-pytest' with parameters:", "purple")
398 common.prettyPrint(parameters)
399 build job: 'swarm-run-pytest',
400 parameters: parameters
401}
402
Tatyana Leontovichc18c8142019-05-16 15:20:33 +0300403def swarm_testrail_report(String passed_steps, String node_with_reports) {
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300404 // Run pytest tests
405 def common = new com.mirantis.mk.Common()
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300406 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Tatyana Leontovichf3718442018-10-31 13:36:13 +0200407 def tempest_test_suite_name = env.TEMPEST_TEST_SUITE_NAME
Vladimir Jigulin91bb7c92019-04-04 18:15:45 +0400408 def test_plan_name_prefix = env.TEST_PLAN_NAME_PREFIX ?: ''
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300409 def parameters = [
410 string(name: 'ENV_NAME', value: "${ENV_NAME}"),
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200411 string(name: 'LAB_CONFIG_NAME', value: "${LAB_CONFIG_NAME}"),
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300412 string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
413 string(name: 'PASSED_STEPS', value: passed_steps),
Tatyana Leontovichc18c8142019-05-16 15:20:33 +0300414 string(name: 'PARENT_NODE_NAME', value: node_with_reports),
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300415 string(name: 'PARENT_WORKSPACE', value: pwd()),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300416 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Tatyana Leontovichf3718442018-10-31 13:36:13 +0200417 string(name: 'TEMPEST_TEST_SUITE_NAME', value: "${tempest_test_suite_name}"),
Hanna Arhipova705a2892020-09-25 12:40:39 +0300418 string(name: 'TEST_PLAN_NAME_PREFIX', value: "${test_plan_name_prefix}")
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300419 ]
420 common.printMsg("Start building job 'swarm-testrail-report' with parameters:", "purple")
421 common.prettyPrint(parameters)
422 build job: 'swarm-testrail-report',
423 parameters: parameters
424}
425
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300426def generate_cookied_model(IPV4_NET_ADMIN, IPV4_NET_CONTROL, IPV4_NET_TENANT, IPV4_NET_EXTERNAL) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300427 def common = new com.mirantis.mk.Common()
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300428 println("IPV4_NET_ADMIN=" + IPV4_NET_ADMIN)
429 println("IPV4_NET_CONTROL=" + IPV4_NET_CONTROL)
430 println("IPV4_NET_TENANT=" + IPV4_NET_TENANT)
431 println("IPV4_NET_EXTERNAL=" + IPV4_NET_EXTERNAL)
432
Hanna Arhipovac2965a42020-01-21 12:27:33 +0200433 def cookiecuttertemplate_commit = env.COOKIECUTTER_TEMPLATE_COMMIT ?: is_released_version(env.MCP_VERSION) ? "refs/tags/${env.MCP_VERSION}" : 'master'
434 def saltmodels_system_commit = env.SALT_MODELS_SYSTEM_COMMIT ?: "refs/tags/${env.MCP_VERSION}"
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300435 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
436 def environment_template_ref_change = env.ENVIRONMENT_TEMPLATE_REF_CHANGE ?: ''
437 def cookiecutter_ref_change = env.COOKIECUTTER_REF_CHANGE ?: ''
Tatyana Leontovich8ac26d72019-05-15 23:33:38 +0300438 def jenkins_pipelines_branch=env.JENKINS_PIPELINE_BRANCH ?: ''
sgudzb7953382019-02-18 17:59:30 +0200439 def update_repo_custom_tag = env.UPDATE_REPO_CUSTOM_TAG ?: ''
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300440 def update_version = env.UPDATE_VERSION ?: ''
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300441
442 def parameters = [
443 string(name: 'LAB_CONTEXT_NAME', value: "${LAB_CONFIG_NAME}"),
444 string(name: 'CLUSTER_NAME', value: "${LAB_CONFIG_NAME}"),
445 string(name: 'DOMAIN_NAME', value: "${LAB_CONFIG_NAME}.local"),
446 string(name: 'REPOSITORY_SUITE', value: "${env.MCP_VERSION}"),
447 string(name: 'SALT_MODELS_SYSTEM_COMMIT', value: "${saltmodels_system_commit}"),
448 string(name: 'COOKIECUTTER_TEMPLATE_COMMIT', value: "${cookiecuttertemplate_commit}"),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300449 string(name: 'COOKIECUTTER_REF_CHANGE', value: "${cookiecutter_ref_change}"),
450 string(name: 'ENVIRONMENT_TEMPLATE_REF_CHANGE', value: "${environment_template_ref_change}"),
451 string(name: 'TCP_QA_REVIEW', value: "${tcp_qa_refs}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300452 string(name: 'IPV4_NET_ADMIN', value: IPV4_NET_ADMIN),
453 string(name: 'IPV4_NET_CONTROL', value: IPV4_NET_CONTROL),
454 string(name: 'IPV4_NET_TENANT', value: IPV4_NET_TENANT),
455 string(name: 'IPV4_NET_EXTERNAL', value: IPV4_NET_EXTERNAL),
sgudzc97385a2018-11-29 17:01:53 +0200456 string(name: 'IPMI_USER', value: env.IPMI_USER),
457 string(name: 'IPMI_PASS', value: env.IPMI_PASS),
Oleksii Butenkoab538de2019-09-23 18:26:57 +0300458 string(name: 'CISCO_PASS', value: env.CISCO_PASS),
sgudzb7953382019-02-18 17:59:30 +0200459 string(name: 'UPDATE_REPO_CUSTOM_TAG', value: "${update_repo_custom_tag}"),
Tatyana Leontovich8ac26d72019-05-15 23:33:38 +0300460 string(name: 'JENKINS_PIPELINE_BRANCH', value: "${jenkins_pipelines_branch}"),
sgudzc97385a2018-11-29 17:01:53 +0200461 string(name: 'IMAGE_PATH_CFG01_DAY01', value: env.IMAGE_PATH_CFG01_DAY01),
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300462 string(name: 'UPDATE_VERSION', value: "${update_version}"),
463
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300464 ]
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300465
466 build_shell_job('swarm-cookied-model-generator', parameters, "deploy_generate_model.xml")
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300467}
468
Dennis Dmitriev4c383472019-04-12 13:58:06 +0300469def generate_configdrive_iso(SALT_MASTER_IP, ADMIN_NETWORK_GW) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300470 def common = new com.mirantis.mk.Common()
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300471 println("SALT_MASTER_IP=" + SALT_MASTER_IP)
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300472 println("ADMIN_NETWORK_GW=" + ADMIN_NETWORK_GW)
473
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300474 def mk_pipelines_ref = env.MK_PIPELINES_REF ?: ''
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300475 def mcp_common_scripts_ref = env.MCP_COMMON_SCRIPTS_REFS ?: ''
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300476 def pipeline_library_ref = env.PIPELINE_LIBRARY_REF ?: ''
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300477 def tcp_qa_refs = env.TCP_QA_REFS ?: ''
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300478 def update_version = env.UPDATE_VERSION?: 'proposed'
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200479 def mcp_salt_repo_url = env.MCP_SALT_REPO_URL ?: ''
480 def mcp_salt_repo_key = env.MCP_SALT_REPO_KEY ?: ''
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300481 def deploy_network_mask = env.DEPLOY_NETWORK_NETMASK ?: ''
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300482
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300483 def parameters = [
484 string(name: 'CLUSTER_NAME', value: "${LAB_CONFIG_NAME}"),
Hanna Arhipovac2965a42020-01-21 12:27:33 +0200485 string(name: 'MODEL_URL', value: "http://sre-ci.scc.mirantis.net:8098/${LAB_CONFIG_NAME}.git"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300486 string(name: 'MODEL_URL_OBJECT_TYPE', value: "git"),
487 booleanParam(name: 'DOWNLOAD_CONFIG_DRIVE', value: true),
488 string(name: 'MCP_VERSION', value: "${MCP_VERSION}"),
Hanna Arhipovac2965a42020-01-21 12:27:33 +0200489 string(name: 'COMMON_SCRIPTS_COMMIT', value: "refs/tags/${env.MCP_VERSION}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300490 string(name: 'NODE_NAME', value: "${NODE_NAME}"),
491 string(name: 'CONFIG_DRIVE_ISO_NAME', value: "${CFG01_CONFIG_IMAGE_NAME}"),
492 string(name: 'SALT_MASTER_DEPLOY_IP', value: SALT_MASTER_IP),
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300493 string(name: 'DEPLOY_NETWORK_GW', value: "${ADMIN_NETWORK_GW}"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300494 string(name: 'PIPELINE_REPO_URL', value: "https://github.com/Mirantis"),
495 booleanParam(name: 'PIPELINES_FROM_ISO', value: true),
Dennis Dmitrievd9168b52018-12-19 18:53:52 +0200496 string(name: 'MCP_SALT_REPO_URL', value: "${mcp_salt_repo_url}"),
497 string(name: 'MCP_SALT_REPO_KEY', value: "${mcp_salt_repo_key}"),
Tatyana Leontovich1f069162019-04-02 19:57:55 +0300498 string(name: 'DEPLOY_NETWORK_NETMASK', value: "${deploy_network_mask}"),
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300499 string(name: 'PIPELINE_LIBRARY_REF', value: "${pipeline_library_ref}"),
500 string(name: 'MK_PIPELINES_REF', value: "${mk_pipelines_ref}"),
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300501 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
Tatyana Leontovich4718bdf2019-05-24 12:37:06 +0300502 string(name: 'UPDATE_VERSION', value: "${update_version}"),
503 string(name: 'MCP_COMMON_SCRIPTS_REFS', value: "${mcp_common_scripts_ref}"),
504 string(name: 'MCP_SALT_REPO_UPDATES', value: "'deb [arch=amd64] http://mirror.mirantis.com/update/${UPDATE_VERSION}/salt-formulas/xenial xenial main'"),
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300505 ]
Dennis Dmitrievf220d972018-10-10 15:19:14 +0300506 build_pipeline_job('swarm-create-cfg-config-drive', parameters)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300507}
508
Dennis Dmitriev13e804b2018-10-09 19:25:14 +0300509def run_job_on_day01_node(stack_to_install, timeout=2400) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300510 // stack_to_install="core,cicd"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200511 def common = new com.mirantis.mk.Common()
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300512 def stack = "${stack_to_install}"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200513 common.printMsg("Deploy DriveTrain CICD components: ${stack_to_install}", "blue")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300514 try {
515 run_cmd("""\
516 export ENV_NAME=${ENV_NAME}
517 . ./tcp_tests/utils/env_salt
518 . ./tcp_tests/utils/env_jenkins_day01
519 export JENKINS_BUILD_TIMEOUT=${timeout}
520 JOB_PARAMETERS=\"{
521 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
Hanna Arhipova6f493522019-12-25 04:23:28 +0200522 \\\"BATCH_SIZE\\\": \\\"\${BATCH_SIZE}\\\",
Dmitriy Kruglov758f99c2020-02-04 12:04:31 +0100523 \\\"STACK_INSTALL\\\": \\\"${stack}\\\",
PGlazoved19b352020-05-21 16:42:27 +0400524 \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\",
525 \\\"UPGRADE_SALTSTACK\\\": \\\"\${UPGRADE_SALTSTACK}\\\"
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300526 }\"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200527 JOB_PREFIX="[ ${ENV_NAME}/{build_number}:drivetrain {time} ] "
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300528 python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
529 """)
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200530 // Wait for IO calm down on cluster nodes
531 sleep(60)
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300532 } catch (e) {
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300533 common.printMsg("Product job 'deploy_openstack' failed, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300534 def workflow_details=run_cmd_stdout("""\
535 . ./tcp_tests/utils/env_salt
536 . ./tcp_tests/utils/env_jenkins_day01
537 export JOB_NAME=deploy_openstack
538 export BUILD_NUMBER=lastBuild
539 python ./tcp_tests/utils/get_jenkins_job_stages.py
540 """)
541 throw new Exception(workflow_details)
542 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300543}
544
Dennis Dmitriev13e804b2018-10-09 19:25:14 +0300545def run_job_on_cicd_nodes(stack_to_install, timeout=2400) {
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300546 // stack_to_install="k8s,calico,stacklight"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200547 def common = new com.mirantis.mk.Common()
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300548 def stack = "${stack_to_install}"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200549 common.printMsg("Deploy Platform components: ${stack_to_install}", "blue")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300550 try {
551 run_cmd("""\
552 export ENV_NAME=${ENV_NAME}
553 . ./tcp_tests/utils/env_salt
554 . ./tcp_tests/utils/env_jenkins_cicd
555 export JENKINS_BUILD_TIMEOUT=${timeout}
556 JOB_PARAMETERS=\"{
557 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
Hanna Arhipova6f493522019-12-25 04:23:28 +0200558 \\\"BATCH_SIZE\\\": \\\"\${BATCH_SIZE}\\\",
Dmitriy Kruglov758f99c2020-02-04 12:04:31 +0100559 \\\"STACK_INSTALL\\\": \\\"${stack}\\\",
PGlazoved19b352020-05-21 16:42:27 +0400560 \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\",
561 \\\"UPGRADE_SALTSTACK\\\": \\\"\${UPGRADE_SALTSTACK}\\\"
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300562 }\"
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200563 JOB_PREFIX="[ ${ENV_NAME}/{build_number}:platform {time} ] "
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300564 python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300565 """)
Dennis Dmitriev44f6db22018-10-31 16:07:56 +0200566 // Wait for IO calm down on cluster nodes
567 sleep(60)
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300568 } catch (e) {
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300569 common.printMsg("Product job 'deploy_openstack' failed, getting details", "purple")
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300570 def workflow_details=run_cmd_stdout("""\
571 . ./tcp_tests/utils/env_salt
572 . ./tcp_tests/utils/env_jenkins_cicd
573 export JOB_NAME=deploy_openstack
574 export BUILD_NUMBER=lastBuild
575 python ./tcp_tests/utils/get_jenkins_job_stages.py
576 """)
577 throw new Exception(workflow_details)
578 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300579}
580
581def sanity_check_component(stack) {
582 // Run sanity check for the component ${stack}.
583 // Result will be stored in JUnit XML file deploy_${stack}.xml
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300584 try {
585 run_cmd("""\
Dennis Dmitriev27007322019-05-03 19:21:44 +0300586 export LOG_NAME=deploy_${stack}_test.log
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300587 py.test --junit-xml=deploy_${stack}.xml -m check_${stack}
588 """)
589 } catch (e) {
590 def String junit_report_xml = readFile("deploy_${stack}.xml")
591 def String junit_report_xml_pretty = new XmlUtil().serialize(junit_report_xml)
592 def String msg = "Sanity check for '${stack}' failed, JUnit report:\n"
593 throw new Exception(msg + junit_report_xml_pretty)
594 }
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300595}
596
Dennis Dmitrievefe5c0b2018-10-24 20:35:26 +0300597def download_logs(archive_name_prefix) {
598 // Archive and download logs and debug info from salt nodes in the lab
599 // Do not fail in case of error to not lose the original error from the parent exception.
600 def common = new com.mirantis.mk.Common()
601 common.printMsg("Downloading nodes logs by ${archive_name_prefix}", "blue")
602 run_cmd("""\
603 export TESTS_CONFIGS=\$(pwd)/${ENV_NAME}_salt_deployed.ini
604 ./tcp_tests/utils/get_logs.py --archive-name-prefix ${archive_name_prefix} || true
605 """)
606}
607
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300608def devops_snapshot_info(snapshot_name) {
609 // Print helper message after snapshot
610 def common = new com.mirantis.mk.Common()
611
612 def SALT_MASTER_IP=run_cmd_stdout("""\
613 . ./tcp_tests/utils/env_salt
614 echo \$SALT_MASTER_IP
615 """).trim().split().last()
616 def login = "root" // set fixed 'root' login for now
617 def password = "r00tme" // set fixed 'root' login for now
618 def key_file = "${env.WORKSPACE}/id_rsa" // set fixed path in the WORKSPACE
619 def VENV_PATH='/home/jenkins/fuel-devops30'
620
621 common.printMsg("""\
622#########################
623# To revert the snapshot:
624#########################
625. ${VENV_PATH}/bin/activate;
626dos.py revert ${ENV_NAME} ${snapshot_name};
627dos.py resume ${ENV_NAME};
628# dos.py time-sync ${ENV_NAME}; # Optional\n
629ssh -i ${key_file} ${login}@${SALT_MASTER_IP} # Optional password: ${password}
630""", "cyan")
631}
632
Dennis Dmitriev2bacadf2019-05-03 17:26:11 +0300633def devops_snapshot(stacks) {
634 // Make snapshots with names "${stack}_deployed" for each stack
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300635 // for all VMs in the environment.
Dennis Dmitriev2bacadf2019-05-03 17:26:11 +0300636
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300637 run_cmd("""\
638 dos.py suspend ${ENV_NAME}
Dennis Dmitriev2bacadf2019-05-03 17:26:11 +0300639 """)
640
641 for (stack in "${stacks}".split(",")) {
642 run_cmd("""\
643 dos.py snapshot ${ENV_NAME} ${stack}_deployed
644 """)
645 devops_snapshot_info("${stack}_deployed")
646 }
647
648 run_cmd("""\
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300649 dos.py resume ${ENV_NAME}
Dennis Dmitriev1fed6662018-07-27 18:22:13 +0300650 sleep 20 # Wait for I/O on the host calms down
Dennis Dmitrievee5ef232018-08-31 13:53:18 +0300651
652 CFG01_NAME=\$(dos.py show-resources ${ENV_NAME} | grep ^cfg01 | cut -d" " -f1)
653 dos.py time-sync ${ENV_NAME} --skip-sync \${CFG01_NAME}
Dennis Dmitriev8df35442018-07-31 08:40:20 +0300654 """)
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300655}
656
Dennis Dmitrievfde667f2018-07-23 16:26:50 +0300657def get_steps_list(steps) {
658 // Make a list from comma separated string
659 return steps.split(',').collect { it.split(':')[0] }
660}
661
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300662def create_xml_report(String filename, String classname, String name, String status='success', String status_message='', String text='', String stdout='', String stderr='') {
663 // <filename> is name of the XML report file that will be created
664 // <status> is one of the 'success', 'skipped', 'failure' or 'error'
665 // 'error' status is assumed as 'Blocker' in TestRail reporter
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300666
667 // Replace '<' and '>' to '&lt;' and '&gt;' to avoid conflicts between xml tags in the message and JUnit report
Hanna Arhipova70685072021-01-25 16:11:57 +0200668 def String text_filtered = text.replaceAll("&", "&amp;").
669 replaceAll("<","&lt;").
Hanna Arhipova413c1db2020-02-11 15:37:48 +0200670 replaceAll(">", "&gt;").
671 replaceAll("[^\\x{0009}\\x{000a}\\x{000d}\\x{0020}-\\x{D7FF}\\x{E000}-\\x{FFFD}]+", "")
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300672
Dennis Dmitriev39666082018-08-29 15:30:45 +0300673 def script = """\
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300674<?xml version=\"1.0\" encoding=\"utf-8\"?>
675 <testsuite>
676 <testcase classname=\"${classname}\" name=\"${name}\" time=\"0\">
Dennis Dmitrievb08c0772018-10-17 15:10:26 +0300677 <${status} message=\"${status_message}\">${text_filtered}</${status}>
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300678 <system-out>${stdout}</system-out>
679 <system-err>${stderr}</system-err>
680 </testcase>
681 </testsuite>
Dennis Dmitriev39666082018-08-29 15:30:45 +0300682"""
683 writeFile(file: filename, text: script, encoding: "UTF-8")
Dennis Dmitrievb3b37492018-07-08 21:23:00 +0300684}
685
Hanna Arhipova1b92f9b2020-10-05 15:50:25 +0300686def upload_results_to_testrail(report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300687 def venvPath = '/home/jenkins/venv_testrail_reporter'
Tatyana Leontovichbfbc4832018-12-27 12:47:23 +0200688 def testPlanDesc = env.LAB_CONFIG_NAME
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300689 def testrailURL = "https://mirantis.testrail.com"
690 def testrailProject = "Mirantis Cloud Platform"
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300691 def testrailMilestone = "MCP1.1"
Dennis Dmitriev707b2152018-10-23 19:12:48 +0300692 def testrailCaseMaxNameLenght = 250
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300693 def jobURL = env.BUILD_URL
694
695 def reporterOptions = [
696 "--verbose",
697 "--env-description \"${testPlanDesc}\"",
698 "--testrail-run-update",
699 "--testrail-url \"${testrailURL}\"",
700 "--testrail-user \"\${TESTRAIL_USER}\"",
701 "--testrail-password \"\${TESTRAIL_PASSWORD}\"",
702 "--testrail-project \"${testrailProject}\"",
703 "--testrail-plan-name \"${testPlanName}\"",
704 "--testrail-milestone \"${testrailMilestone}\"",
705 "--testrail-suite \"${testSuiteName}\"",
706 "--xunit-name-template \"${methodname}\"",
707 "--testrail-name-template \"${testrail_name_template}\"",
708 "--test-results-link \"${jobURL}\"",
Dennis Dmitriev707b2152018-10-23 19:12:48 +0300709 "--testrail-case-max-name-lenght ${testrailCaseMaxNameLenght}",
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300710 ] + reporter_extra_options
711
712 def script = """
713 . ${venvPath}/bin/activate
714 set -ex
Ekaterina Chernova0d81efc2019-10-28 16:23:17 +0300715 report ${reporterOptions.join(' ')} ${report_name}
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300716 """
717
718 def testrail_cred_id = params.TESTRAIL_CRED ?: 'testrail_system_tests'
719
720 withCredentials([
721 [$class : 'UsernamePasswordMultiBinding',
722 credentialsId : testrail_cred_id,
723 passwordVariable: 'TESTRAIL_PASSWORD',
724 usernameVariable: 'TESTRAIL_USER']
725 ]) {
Dennis Dmitrievfd0b78c2019-05-30 00:16:04 +0300726 def ret = [:]
727 ret.stdout = ''
728 ret.exception = ''
729 try {
730 ret.stdout = run_cmd_stdout(script)
731 } catch (Exception ex) {
732 ret.exception = ("""\
733##### Report to '${testSuiteName}' failed: #####\n""" + ex.message + "\n\n")
734 }
735 return ret
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300736 }
737}
738
739
740def create_deploy_result_report(deploy_expected_stacks, result, text) {
741 def STATUS_MAP = ['SUCCESS': 'success', 'FAILURE': 'failure', 'UNSTABLE': 'failure', 'ABORTED': 'error']
742 def classname = "Deploy"
743 def name = "deployment_${ENV_NAME}"
Dennis Dmitriev39666082018-08-29 15:30:45 +0300744 def filename = "${name}.xml"
Dennis Dmitrieve4b831b2018-08-15 17:16:10 +0300745 def status = STATUS_MAP[result ?: 'FAILURE'] // currentBuild.result *must* be set at the finish of the try/catch
746 create_xml_report(filename, classname, name, status, "Deploy components: ${deploy_expected_stacks}", text, '', '')
Dennis Dmitriev27a96792018-07-30 07:52:03 +0300747}
Hanna Arhipova1b92f9b2020-10-05 15:50:25 +0300748
749def mark_test_results(testPlanName, testSuiteName) {
750 def venvPath = '/home/jenkins/venv_testrail_analyzer'
751 def testrailURL = "https://mirantis.testrail.com"
752 def testrailProject = "Mirantis Cloud Platform"
753 def configName = env.LAB_CONFIG_NAME
754 def testRunName = "${configName} <${testSuiteName}>"
755
756 def script = """
757 . ${venvPath}/bin/activate
758 set -ex
759 export TESTRAIL_URL=${testrailURL}
760 testrail-reporter analyze -p "${testrailProject}" -t "${testPlanName}" -r "${testRunName}" checklist.yaml
761 """
762
763 def testrail_cred_id = params.TESTRAIL_CRED ?: 'testrail_system_tests'
764
765 withCredentials([
766 [$class : 'UsernamePasswordMultiBinding',
767 credentialsId : testrail_cred_id,
768 passwordVariable: 'TESTRAIL_PASSWORD',
769 usernameVariable: 'TESTRAIL_USER']
770 ]) {
771 def ret = [:]
772 ret.stdout = ''
773 ret.exception = ''
774 try {
775 ret.stdout = run_cmd_stdout(script)
776 } catch (Exception ex) {
777 ret.exception = ("""\
778##### Report to '${testRunName}' failed: #####\n""" + ex.message + "\n\n")
779 }
780 return ret
781 }
Hanna Arhipova3be50622021-01-11 13:11:45 +0200782}
783
784def reboot_hardware_nodes() {
785 bm_ips = [
786 "185.8.59.227",
787 "185.8.59.229",
788 "5.43.225.88",
789 "5.43.225.112",
790 "5.43.225.208",
791 "5.43.227.118",
792 "185.8.58.248",
793 "185.8.59.222",
794 "5.43.225.228",
795 "5.43.229.28",
796 "5.43.225.23",
797 "185.8.58.9",
798 "185.8.58.246",
799 "185.8.58.243",
800 "185.8.58.244"
801 ]
802
803 withCredentials([
804 [$class : 'UsernamePasswordMultiBinding',
805 credentialsId : 'lab_engineer',
806 passwordVariable: 'lab_pass',
807 usernameVariable: 'lab_user']
808 ]) {
809 for (ip in bm_ips) { sh ("ipmitool -H ${ip} -U ${lab_user} -P ${lab_pass} chassis power off")}
810 for (ip in bm_ips) { sh ("ipmitool -H ${ip} -U ${lab_user} -P ${lab_pass} chassis bootdev pxe")}
811 }
Hanna Arhipova1b92f9b2020-10-05 15:50:25 +0300812}