blob: 6a6604b73f3e4889cd981e8db7b536e78b0f5b0a [file] [log] [blame]
chnydaf14ea2a2017-05-26 15:07:47 +02001package com.mirantis.mk
2
3/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04004 * Setup Docker to run some tests. Returns true/false based on
5 were tests successful or not.
6 * @param config - LinkedHashMap with configuration params:
7 * dockerHostname - (required) Hostname to use for Docker container.
azvyagintsevabcf42e2018-10-05 20:40:27 +03008 * distribRevision - (optional) Revision of packages to use (default proposed).
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04009 * runCommands - (optional) Dict with closure structure of body required tests. For example:
10 * [ '001_Test': { sh("./run-some-test") }, '002_Test': { sh("./run-another-test") } ]
11 * Before execution runCommands will be sorted by key names. Alpabetical order is preferred.
12 * runFinally - (optional) Dict with closure structure of body required commands, which should be
13 * executed in any case of test results. Same format as for runCommands
14 * updateRepo - (optional) Whether to run common repo update step.
15 * dockerContainerName - (optional) Docker container name.
16 * dockerImageName - (optional) Docker image name
17 * dockerMaxCpus - (optional) Number of CPUS to use in Docker.
18 * dockerExtraOpts - (optional) Array of Docker extra opts for container
19 * envOpts - (optional) Array of variables that should be passed as ENV vars to Docker container.
20 * Return true | false
21 */
22
23def setupDockerAndTest(LinkedHashMap config) {
24 def common = new com.mirantis.mk.Common()
25 def TestMarkerResult = false
26 // setup options
27 def defaultContainerName = 'test-' + UUID.randomUUID().toString()
28 def dockerHostname = config.get('dockerHostname', defaultContainerName)
azvyagintsevabcf42e2018-10-05 20:40:27 +030029 def distribRevision = config.get('distribRevision', 'proposed')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040030 def runCommands = config.get('runCommands', [:])
31 def runFinally = config.get('runFinally', [:])
32 def baseRepoPreConfig = config.get('baseRepoPreConfig', true)
33 def dockerContainerName = config.get('dockerContainerName', defaultContainerName)
azvyagintseve716a452018-11-25 13:41:52 +020034 // def dockerImageName = config.get('image', "mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
35 // FIXME /PROD-25244
Denis Egorenkoe8115c42019-04-15 20:21:25 +040036 def dockerImageName = config.get('image', "docker-prod-virtual.docker.mirantis.net/mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040037 def dockerMaxCpus = config.get('dockerMaxCpus', 4)
38 def dockerExtraOpts = config.get('dockerExtraOpts', [])
39 def envOpts = config.get('envOpts', [])
azvyagintsevabcf42e2018-10-05 20:40:27 +030040 envOpts.add("DISTRIB_REVISION=${distribRevision}")
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040041 def dockerBaseOpts = [
42 '-u root:root',
43 "--hostname=${dockerHostname}",
44 '--ulimit nofile=4096:8192',
45 "--name=${dockerContainerName}",
46 "--cpus=${dockerMaxCpus}"
47 ]
Denis Egorenko649cf7d2018-10-18 16:36:33 +040048 def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ')
Denis Egorenko5cea1412018-10-18 16:40:11 +040049 def extraReposConfig = null
50 if (baseRepoPreConfig) {
51 // extra repo on mirror.mirantis.net, which is not supported before 2018.11.0 release
52 def extraRepoSource = "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/extra/xenial xenial main"
Denis Egorenko7269cf32019-03-27 19:12:06 +040053 def releaseVersionQ4 = '2018.11.0'
54 def oldRelease = false
Denis Egorenko5cea1412018-10-18 16:40:11 +040055 try {
56 def releaseNaming = 'yyyy.MM.dd'
57 def repoDateUsed = new Date().parse(releaseNaming, distribRevision)
Denis Egorenko7269cf32019-03-27 19:12:06 +040058 def extraAvailableFrom = new Date().parse(releaseNaming, releaseVersionQ4)
Denis Egorenko5cea1412018-10-18 16:40:11 +040059 if (repoDateUsed < extraAvailableFrom) {
azvyagintsevb9afd232019-02-11 14:22:31 +020060 extraRepoSource = "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
Denis Egorenko7269cf32019-03-27 19:12:06 +040061 oldRelease = true
Denis Egorenko5cea1412018-10-18 16:40:11 +040062 }
63 } catch (Exception e) {
64 common.warningMsg(e)
azvyagintsevb9afd232019-02-11 14:22:31 +020065 if (!(distribRevision in ['nightly', 'proposed', 'testing'])) {
azvyagintsev62c97742019-01-02 17:53:54 +020066 extraRepoSource = "deb [arch=amd64] http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
Denis Egorenko7269cf32019-03-27 19:12:06 +040067 oldRelease = true
Denis Egorenko5cea1412018-10-18 16:40:11 +040068 }
69 }
70
71 def defaultExtraReposYaml = """
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000072---
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000073aprConfD: |-
74 APT::Get::AllowUnauthenticated 'true';
75 APT::Get::Install-Suggests 'false';
76 APT::Get::Install-Recommends 'false';
77repo:
78 mcp_saltstack:
Denis Egorenko395aa212018-10-11 15:11:28 +040079 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/saltstack-2017.7/xenial xenial main"
Denis Egorenkoe02a1b22018-10-19 17:47:53 +040080 pin:
81 - package: "libsodium18"
82 pin: "release o=SaltStack"
83 priority: 50
84 - package: "*"
85 pin: "release o=SaltStack"
86 priority: "1100"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000087 mcp_extra:
Denis Egorenko3c752a52018-10-12 12:21:29 +040088 source: "${extraRepoSource}"
Denis Egorenko395aa212018-10-11 15:11:28 +040089 mcp_saltformulas:
azvyagintsev058c1f42018-12-14 18:32:33 +020090 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial xenial main"
91 repo_key: "http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial/archive-salt-formulas.key"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000092 ubuntu:
Denis Egorenko395aa212018-10-11 15:11:28 +040093 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000094 ubuntu-upd:
Denis Egorenko395aa212018-10-11 15:11:28 +040095 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-updates main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000096 ubuntu-sec:
Denis Egorenko395aa212018-10-11 15:11:28 +040097 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-security main restricted universe"
98"""
Denis Egorenko5cea1412018-10-18 16:40:11 +040099 // override for now
100 def extraRepoMergeStrategy = config.get('extraRepoMergeStrategy', 'override')
101 def extraRepos = config.get('extraRepos', [:])
102 def defaultRepos = readYaml text: defaultExtraReposYaml
Denis Egorenko7269cf32019-03-27 19:12:06 +0400103 if (! oldRelease && distribRevision != releaseVersionQ4) {
104 defaultRepos['repo']['mcp_saltformulas_update'] = [
105 'source': "deb [arch=amd64] http://mirror.mirantis.com/update/${distribRevision}/salt-formulas/xenial xenial main",
106 'repo_key': "http://mirror.mirantis.com/update/${distribRevision}/salt-formulas/xenial/archive-salt-formulas.key"
107 ]
108 }
Denis Egorenko5cea1412018-10-18 16:40:11 +0400109 if (extraRepoMergeStrategy == 'merge') {
110 extraReposConfig = common.mergeMaps(defaultRepos, extraRepos)
111 } else {
112 extraReposConfig = extraRepos ? extraRepos : defaultRepos
113 }
114 }
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400115 def img = docker.image(dockerImageName)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000116
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400117 img.pull()
118
119 try {
120 img.inside(dockerOptsFinal) {
121 withEnv(envOpts) {
122 try {
azvyagintsev4a8ccfa2019-04-08 16:25:08 +0300123 sh('printenv |sort -u')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400124 // Currently, we don't have any other point to install
125 // runtime dependencies for tests.
126 if (baseRepoPreConfig) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000127 // Warning! POssible point of 'allow-downgrades' issue
128 // Probably, need to add such flag into apt.prefs
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400129 sh("""#!/bin/bash -xe
130 echo "Installing extra-deb dependencies inside docker:"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000131 echo > /etc/apt/sources.list
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400132 rm -vf /etc/apt/sources.list.d/* || true
Denis Egorenkoe02a1b22018-10-19 17:47:53 +0400133 rm -vf /etc/apt/preferences.d/* || true
Aleksey Zvyagintsevc5453342018-10-05 15:03:59 +0000134 """)
Denis Egorenko5cea1412018-10-18 16:40:11 +0400135 common.debianExtraRepos(extraReposConfig)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000136 sh('''#!/bin/bash -xe
137 apt-get update
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400138 apt-get install -y python-netaddr
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000139 ''')
140
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400141 }
142 runCommands.sort().each { command, body ->
143 common.warningMsg("Running command: ${command}")
144 // doCall is the closure implementation in groovy, allow to pass arguments to closure
145 body.call()
146 }
147 // If we didn't dropped for now - test has been passed.
148 TestMarkerResult = true
149 }
150 finally {
151 runFinally.sort().each { command, body ->
152 common.warningMsg("Running ${command} command.")
153 // doCall is the closure implementation in groovy, allow to pass arguments to closure
154 body.call()
155 }
156 }
157 }
158 }
159 }
160 catch (Exception er) {
161 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
162 }
163
164 try {
165 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
166 timeout(time: 10, unit: 'SECONDS') {
167 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
168 }
169 timeout(time: 10, unit: 'SECONDS') {
170 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
171 }
172 }
173 catch (Exception er) {
174 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
175 }
176
177 if (TestMarkerResult) {
178 common.infoMsg("Test finished: SUCCESS")
179 } else {
180 common.warningMsg("Test finished: FAILURE")
181 }
182 return TestMarkerResult
183}
184
185/**
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000186 * Wrapper around setupDockerAndTest, to run checks against new Reclass version
187 * that current model is compatible with new Reclass.
188 *
189 * @param config - LinkedHashMap with configuration params:
190 * dockerHostname - (required) Hostname to use for Docker container.
191 * distribRevision - (optional) Revision of packages to use (default proposed).
192 * extraRepo - (optional) Extra repo to use to install new Reclass version. Has
193 * high priority on distribRevision
194 * targetNodes - (required) List nodes to check pillar data.
Denis Egorenkob090a762018-09-12 19:25:41 +0400195 */
196def compareReclassVersions(config) {
197 def common = new com.mirantis.mk.Common()
198 def salt = new com.mirantis.mk.Salt()
199 common.infoMsg("Going to test new reclass for CFG node")
200 def distribRevision = config.get('distribRevision', 'proposed')
201 def venv = config.get('venv')
202 def extraRepo = config.get('extraRepo', '')
203 def extraRepoKey = config.get('extraRepoKey', '')
204 def targetNodes = config.get('targetNodes')
205 sh "rm -rf ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
206 sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
207 def configRun = [
azvyagintsevabcf42e2018-10-05 20:40:27 +0300208 'distribRevision': distribRevision,
azvyagintsevb9afd232019-02-11 14:22:31 +0200209 'dockerExtraOpts': [
Denis Egorenkob090a762018-09-12 19:25:41 +0400210 "-v /srv/salt/reclass:/srv/salt/reclass:ro",
211 "-v /etc/salt:/etc/salt:ro",
212 "-v /usr/share/salt-formulas/:/usr/share/salt-formulas/:ro"
213 ],
azvyagintsevb9afd232019-02-11 14:22:31 +0200214 'envOpts' : [
Denis Egorenkob090a762018-09-12 19:25:41 +0400215 "WORKSPACE=${env.WORKSPACE}",
216 "NODES_LIST=${targetNodes.join(' ')}"
217 ],
azvyagintsevb9afd232019-02-11 14:22:31 +0200218 'runCommands' : [
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000219 '001_Update_Reclass_package' : {
220 sh('apt-get update && apt-get install -y reclass')
Denis Egorenkob090a762018-09-12 19:25:41 +0400221 },
222 '002_Test_Reclass_Compatibility': {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000223 sh('''
Denis Egorenkob090a762018-09-12 19:25:41 +0400224 reclass-salt -b /srv/salt/reclass -t > ${WORKSPACE}/new/inventory || exit 1
225 for node in $NODES_LIST; do
226 reclass-salt -b /srv/salt/reclass -p $node > ${WORKSPACE}/new/$node || exit 1
227 done
228 ''')
229 }
230 ]
231 ]
232 if (extraRepo) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000233 // FIXME
Denis Egorenkob090a762018-09-12 19:25:41 +0400234 configRun['runCommands']['0001_Additional_Extra_Repo_Passed'] = {
235 sh("""
236 echo "${extraRepo}" > /etc/apt/sources.list.d/mcp_extra.list
237 [ "${extraRepoKey}" ] && wget -O - ${extraRepoKey} | apt-key add -
238 """)
239 }
Denis Egorenkob090a762018-09-12 19:25:41 +0400240 }
241 if (setupDockerAndTest(configRun)) {
242 common.infoMsg("New reclass version is compatible with current model: SUCCESS")
243 def inventoryOld = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -t", true, null, true).get("return")[0].values()[0]
244 // [0..-31] to exclude 'echo Salt command execution success' from output
245 writeFile(file: "${env.WORKSPACE}/old/inventory", text: inventoryOld[0..-31])
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000246 for (String node in targetNodes) {
Denis Egorenkob090a762018-09-12 19:25:41 +0400247 def nodeOut = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -p ${node}", true, null, true).get("return")[0].values()[0]
248 writeFile(file: "${env.WORKSPACE}/old/${node}", text: nodeOut[0..-31])
249 }
250 def reclassDiff = common.comparePillars(env.WORKSPACE, env.BUILD_URL, '')
251 currentBuild.description = reclassDiff
252 if (reclassDiff != '<b>No job changes</b>') {
253 throw new RuntimeException("Pillars with new reclass version has been changed: FAILED")
254 } else {
255 common.infoMsg("Pillars not changed with new reclass version: SUCCESS")
256 }
257 } else {
258 throw new RuntimeException("New reclass version is not compatible with current model: FAILED")
259 }
260}
261
262/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400263 * Wrapper over setupDockerAndTest, to test CC model.
264 *
265 * @param config - dict with params:
266 * dockerHostname - (required) salt master's name
267 * clusterName - (optional) model cluster name
268 * extraFormulas - (optional) extraFormulas to install. DEPRECATED
269 * formulasSource - (optional) formulas source (git or pkg, default pkg)
270 * reclassVersion - (optional) Version of used reclass (branch, tag, ...) (optional, default master)
271 * reclassEnv - (require) directory of model
272 * ignoreClassNotfound - (optional) Ignore missing classes for reclass model (default false)
273 * aptRepoUrl - (optional) package repository with salt formulas
274 * aptRepoGPG - (optional) GPG key for apt repository with formulas
275 * testContext - (optional) Description of test
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000276 Return: true\exception
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400277 */
278
279def testNode(LinkedHashMap config) {
280 def common = new com.mirantis.mk.Common()
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400281 def dockerHostname = config.get('dockerHostname')
Denis Egorenkoe3509c12019-02-19 14:06:26 +0400282 def domain = config.get('domain')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400283 def reclassEnv = config.get('reclassEnv')
284 def clusterName = config.get('clusterName', "")
285 def formulasSource = config.get('formulasSource', 'pkg')
286 def extraFormulas = config.get('extraFormulas', 'linux')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400287 def ignoreClassNotfound = config.get('ignoreClassNotfound', false)
288 def aptRepoUrl = config.get('aptRepoUrl', "")
289 def aptRepoGPG = config.get('aptRepoGPG', "")
290 def testContext = config.get('testContext', 'test')
Denis Egorenko66876fc2019-03-07 15:57:56 +0400291 def nodegenerator = config.get('nodegenerator', false)
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400292 config['envOpts'] = [
293 "RECLASS_ENV=${reclassEnv}", "SALT_STOPSTART_WAIT=5",
Denis Egorenkoe3509c12019-02-19 14:06:26 +0400294 "HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}",
295 "DOMAIN=${domain}", "FORMULAS_SOURCE=${formulasSource}",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400296 "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400297 "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "DEBUG=1",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400298 "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}"
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400299 ]
300
301 config['runCommands'] = [
302 '001_Clone_salt_formulas_scripts': {
azvyagintsev62c97742019-01-02 17:53:54 +0200303 sh(script: 'git clone http://gerrit.mcp.mirantis.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400304 },
305
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000306 '002_Prepare_something' : {
azvyagintsevb9afd232019-02-11 14:22:31 +0200307 sh('''#!/bin/bash -x
308 rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
Denis Egorenkoe3509c12019-02-19 14:06:26 +0400309 echo "127.0.0.1 ${HOSTNAME}.${DOMAIN}" >> /etc/hosts
azvyagintsevb9afd232019-02-11 14:22:31 +0200310 if [ -f '/srv/salt/reclass/salt_master_pillar.asc' ] ; then
311 mkdir -p /etc/salt/gpgkeys
312 chmod 700 /etc/salt/gpgkeys
313 GNUPGHOME=/etc/salt/gpgkeys gpg --import /srv/salt/reclass/salt_master_pillar.asc
314 fi
Alexander Evseev4983dec2018-10-25 15:49:39 +0200315 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
316 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400317 ''')
318 },
319
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400320 '003_Install_Reclass_package' : {
321 sh('apt-get install -y reclass')
322 },
323
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000324 '004_Run_tests' : {
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400325 def testTimeout = 40 * 60
326 timeout(time: testTimeout, unit: 'SECONDS') {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000327 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400328 source /srv/salt/scripts/bootstrap.sh
329 cd /srv/salt/scripts
330 source_local_envs
331 configure_salt_master
332 configure_salt_minion
333 install_salt_formula_pkg
334 source /srv/salt/scripts/bootstrap.sh
335 cd /srv/salt/scripts
336 saltservice_restart''')
337
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000338 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400339 source /srv/salt/scripts/bootstrap.sh
340 cd /srv/salt/scripts
341 source_local_envs
342 saltmaster_init''')
343
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000344 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400345 source /srv/salt/scripts/bootstrap.sh
346 cd /srv/salt/scripts
347 verify_salt_minions''')
348 }
349 }
350 ]
351 config['runFinally'] = [
352 '001_Archive_artefacts': {
353 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
354 archiveArtifacts artifacts: "nodesinfo.tar.gz"
355 }
356 ]
Denis Egorenko66876fc2019-03-07 15:57:56 +0400357 // this tool should be tested in master branch only
358 // and not for all jobs, as pilot will be used cc-reclass-chunk
359 if (nodegenerator) {
360 config['runCommands']['005_Test_new_nodegenerator'] = {
361 try {
362 sh('''#!/bin/bash
363 new_generated_dir=/srv/salt/_new_generated
364 mkdir -p ${new_generated_dir}
365 nodegenerator -b /srv/salt/reclass/classes/ -o ${new_generated_dir} ${CLUSTER_NAME}
366 diff -r /srv/salt/reclass/nodes/_generated ${new_generated_dir} > /tmp/nodegenerator.diff
367 tar -czf /tmp/_generated.tar.gz /srv/salt/reclass/nodes/_generated/
368 tar -czf /tmp/_new_generated.tar.gz ${new_generated_dir}/
369 tar -czf /tmp/_model.tar.gz /srv/salt/reclass/classes/cluster/*
370 ''')
371 } catch (Exception e) {
372 print "Test new nodegenerator tool is failed: ${e}"
373 }
374 }
375 config['runFinally']['002_Archive_nodegenerator_artefact'] = {
376 sh(script: "cd /tmp; [ -f nodegenerator.diff ] && tar -czf ${env.WORKSPACE}/nodegenerator.tar.gz nodegenerator.diff _generated.tar.gz _new_generated.tar.gz _model.tar.gz", returnStatus: true)
377 if (fileExists('nodegenerator.tar.gz')) {
378 archiveArtifacts artifacts: "nodegenerator.tar.gz"
379 }
380 }
381 }
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400382 testResult = setupDockerAndTest(config)
383 if (testResult) {
384 common.infoMsg("Node test for context: ${testContext} model: ${reclassEnv} finished: SUCCESS")
385 } else {
386 throw new RuntimeException("Node test for context: ${testContext} model: ${reclassEnv} finished: FAILURE")
387 }
388 return testResult
389}
390
391/**
chnydaf14ea2a2017-05-26 15:07:47 +0200392 * setup and test salt-master
393 *
azvyagintsevb4e0c442018-09-12 17:00:04 +0300394 * @param masterName salt master's name
395 * @param clusterName model cluster name
396 * @param extraFormulas extraFormulas to install. DEPRECATED
397 * @param formulasSource formulas source (git or pkg)
398 * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master)
399 * @param testDir directory of model
400 * @param formulasSource Salt formulas source type (optional, default pkg)
401 * @param formulasRevision APT revision for formulas (optional default stable)
Petr Michalec6414aa52017-08-17 14:32:52 +0200402 * @param ignoreClassNotfound Ignore missing classes for reclass model
azvyagintsevb4e0c442018-09-12 17:00:04 +0300403 * @param dockerMaxCpus max cpus passed to docker (default 0, disabled)
404 * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models)
405 * @param aptRepoUrl package repository with salt formulas
406 * @param aptRepoGPG GPG key for apt repository with formulas
azvyagintsev28fa9d92018-06-26 14:31:49 +0300407 * Return true | false
chnydaf14ea2a2017-05-26 15:07:47 +0200408 */
409
azvyagintsevb4e0c442018-09-12 17:00:04 +0300410def setupAndTestNode(masterName, clusterName, extraFormulas = '*', testDir, formulasSource = 'pkg',
Vasyl Saienko369ed902018-07-23 11:49:32 +0000411 formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0,
azvyagintsev28fa9d92018-06-26 14:31:49 +0300412 ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300413 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300414 // TODO
415 common.errorMsg('You are using deprecated function!Please migrate to "setupDockerAndTest".' +
416 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
417 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300418 // timeout for test execution (40min)
419 def testTimeout = 40 * 60
420 def TestMarkerResult = false
421 def saltOpts = "--retcode-passthrough --force-color"
422 def workspace = common.getWorkspace()
423 def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
424 img.pull()
chnydaf14ea2a2017-05-26 15:07:47 +0200425
azvyagintsev635affb2018-09-13 13:02:54 +0300426 if (formulasSource == 'pkg') {
427 if (extraFormulas) {
428 common.warningMsg("You have passed deprecated variable:extraFormulas=${extraFormulas}. " +
429 "\n It would be ignored, and all formulas would be installed anyway")
430 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300431 }
432 if (!dockerContainerName) {
433 dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString()
434 }
435 def dockerMaxCpusOpt = "--cpus=4"
436 if (dockerMaxCpus > 0) {
437 dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}"
438 }
439 try {
440 img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") {
azvyagintsev635affb2018-09-13 13:02:54 +0300441 withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
azvyagintsevb4e0c442018-09-12 17:00:04 +0300442 "DISTRIB_REVISION=${formulasRevision}",
443 "DEBUG=1", "MASTER_HOSTNAME=${masterName}",
444 "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
445 "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}",
446 "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5",
447 "APT_REPOSITORY_GPG=${aptRepoGPG}"]) {
448 try {
449 // Currently, we don't have any other point to install
450 // runtime dependencies for tests.
451 sh("""#!/bin/bash -xe
azvyagintsev1bfe6842018-08-09 18:40:17 +0200452 echo "Installing extra-deb dependencies inside docker:"
453 echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
454 echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
455 echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
456 rm -vf /etc/apt/sources.list.d/* || true
457 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list
458 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
459 apt-get update
460 apt-get install -y python-netaddr
461 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300462 sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
463 sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
Alexander Evseev4983dec2018-10-25 15:49:39 +0200464 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
465 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
azvyagintsev1bfe6842018-08-09 18:40:17 +0200466 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300467 // FIXME: should be changed to use reclass from mcp_extra_nigtly?
468 sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
azvyagintsev1bfe6842018-08-09 18:40:17 +0200469 sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
470 -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
471 done""")
azvyagintsevb4e0c442018-09-12 17:00:04 +0300472 timeout(time: testTimeout, unit: 'SECONDS') {
473 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200474 source /srv/salt/scripts/bootstrap.sh
475 cd /srv/salt/scripts
476 source_local_envs
477 configure_salt_master
478 configure_salt_minion
479 install_salt_formula_pkg
480 source /srv/salt/scripts/bootstrap.sh
481 cd /srv/salt/scripts
482 saltservice_restart''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300483 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200484 source /srv/salt/scripts/bootstrap.sh
485 cd /srv/salt/scripts
486 source_local_envs
487 saltmaster_init''')
488
azvyagintsevb4e0c442018-09-12 17:00:04 +0300489 if (!legacyTestingMode.toBoolean()) {
490 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200491 source /srv/salt/scripts/bootstrap.sh
492 cd /srv/salt/scripts
493 verify_salt_minions
494 ''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300495 }
496 }
497 // If we didn't dropped for now - test has been passed.
498 TestMarkerResult = true
499 }
500
501 finally {
502 // Collect rendered per-node data.Those info could be simply used
503 // for diff processing. Data was generated via reclass.cli --nodeinfo,
504 /// during verify_salt_minions.
505 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
506 archiveArtifacts artifacts: "nodesinfo.tar.gz"
507 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200508 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200509 }
chnydaf14ea2a2017-05-26 15:07:47 +0200510 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300511 catch (Exception er) {
512 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
513 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300514
azvyagintsevb4e0c442018-09-12 17:00:04 +0300515 if (legacyTestingMode.toBoolean()) {
516 common.infoMsg("Running legacy mode test for master hostname ${masterName}")
517 def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true)
518 for (minion in nodes.tokenize()) {
519 def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true)
520 if (!basename.trim().contains(masterName)) {
521 testMinion(basename.trim())
522 }
523 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300524 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300525
azvyagintsevb4e0c442018-09-12 17:00:04 +0300526 try {
527 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
528 timeout(time: 10, unit: 'SECONDS') {
529 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
530 }
531 timeout(time: 10, unit: 'SECONDS') {
532 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
533 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300534 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300535 catch (Exception er) {
536 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
azvyagintsev28fa9d92018-06-26 14:31:49 +0300537 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300538
azvyagintsevb4e0c442018-09-12 17:00:04 +0300539 if (TestMarkerResult) {
540 common.infoMsg("Test finished: SUCCESS")
541 } else {
542 common.warningMsg("Test finished: FAILURE")
543 }
544 return TestMarkerResult
azvyagintsev28fa9d92018-06-26 14:31:49 +0300545
chnydaf14ea2a2017-05-26 15:07:47 +0200546}
547
548/**
549 * Test salt-minion
550 *
azvyagintsev28fa9d92018-06-26 14:31:49 +0300551 * @param minion salt minion
chnydaf14ea2a2017-05-26 15:07:47 +0200552 */
553
azvyagintsev28fa9d92018-06-26 14:31:49 +0300554def testMinion(minionName) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300555 sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
Jakub Joseffa6ad8d2017-06-26 18:29:55 +0200556}
azvyagintsev2b279d82018-08-07 17:22:54 +0200557
azvyagintsev2b279d82018-08-07 17:22:54 +0200558/**
559 * Wrapper over setupAndTestNode, to test exactly one CC model.
azvyagintsevb4e0c442018-09-12 17:00:04 +0300560 Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
561 Flow: grab all data, and pass to setupAndTestNode function
562 under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
azvyagintsev2b279d82018-08-07 17:22:54 +0200563 *
564 * @param cfg - dict with params:
azvyagintsevb4e0c442018-09-12 17:00:04 +0300565 MODELS_TARGZ http link to arch with (models|contexts|global_reclass)
566 modelFile
567 DockerCName directly passed to setupAndTestNode
568 EXTRA_FORMULAS directly passed to setupAndTestNode
569 DISTRIB_REVISION directly passed to setupAndTestNode
570 reclassVersion directly passed to setupAndTestNode
azvyagintsev2b279d82018-08-07 17:22:54 +0200571
azvyagintsevb4e0c442018-09-12 17:00:04 +0300572 Return: true\exception
azvyagintsev2b279d82018-08-07 17:22:54 +0200573 */
574
575def testCCModel(cfg) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300576 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300577 common.errorMsg('You are using deprecated function!Please migrate to "testNode".' +
578 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
579 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300580 sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
581 sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
582 // unpack data
583 sh(script: "tar -xzf models.tar.gz ")
584 common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
585 content = readFile(file: cfg.modelFile)
586 templateContext = readYaml text: content
587 clusterName = templateContext.default_context.cluster_name
588 clusterDomain = templateContext.default_context.cluster_domain
azvyagintsev2b279d82018-08-07 17:22:54 +0200589
azvyagintsevb4e0c442018-09-12 17:00:04 +0300590 def testResult = false
591 testResult = setupAndTestNode(
592 "cfg01.${clusterDomain}",
593 clusterName,
594 '',
595 cfg.testReclassEnv, // Sync into image exactly one env
596 'pkg',
597 cfg.DISTRIB_REVISION,
598 cfg.reclassVersion,
599 0,
600 false,
601 false,
602 '',
603 '',
604 cfg.DockerCName)
605 if (testResult) {
606 common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
607 } else {
608 throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
609 }
610 return testResult
azvyagintsev2b279d82018-08-07 17:22:54 +0200611}