blob: c93587f0642d3e6a5c3c2f9ca4d22f4629502e85 [file] [log] [blame]
chnydaf14ea2a2017-05-26 15:07:47 +02001package com.mirantis.mk
2
3/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04004 * Setup Docker to run some tests. Returns true/false based on
5 were tests successful or not.
6 * @param config - LinkedHashMap with configuration params:
7 * dockerHostname - (required) Hostname to use for Docker container.
azvyagintsevabcf42e2018-10-05 20:40:27 +03008 * distribRevision - (optional) Revision of packages to use (default proposed).
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04009 * runCommands - (optional) Dict with closure structure of body required tests. For example:
10 * [ '001_Test': { sh("./run-some-test") }, '002_Test': { sh("./run-another-test") } ]
11 * Before execution runCommands will be sorted by key names. Alpabetical order is preferred.
12 * runFinally - (optional) Dict with closure structure of body required commands, which should be
13 * executed in any case of test results. Same format as for runCommands
14 * updateRepo - (optional) Whether to run common repo update step.
15 * dockerContainerName - (optional) Docker container name.
16 * dockerImageName - (optional) Docker image name
17 * dockerMaxCpus - (optional) Number of CPUS to use in Docker.
18 * dockerExtraOpts - (optional) Array of Docker extra opts for container
19 * envOpts - (optional) Array of variables that should be passed as ENV vars to Docker container.
20 * Return true | false
21 */
22
23def setupDockerAndTest(LinkedHashMap config) {
24 def common = new com.mirantis.mk.Common()
25 def TestMarkerResult = false
26 // setup options
27 def defaultContainerName = 'test-' + UUID.randomUUID().toString()
28 def dockerHostname = config.get('dockerHostname', defaultContainerName)
azvyagintsevabcf42e2018-10-05 20:40:27 +030029 def distribRevision = config.get('distribRevision', 'proposed')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040030 def runCommands = config.get('runCommands', [:])
31 def runFinally = config.get('runFinally', [:])
32 def baseRepoPreConfig = config.get('baseRepoPreConfig', true)
33 def dockerContainerName = config.get('dockerContainerName', defaultContainerName)
azvyagintseve716a452018-11-25 13:41:52 +020034 // def dockerImageName = config.get('image', "mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
35 // FIXME /PROD-25244
36 def dockerImageName = config.get('image', "docker-dev-virtual.docker.mirantis.net/mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040037 def dockerMaxCpus = config.get('dockerMaxCpus', 4)
38 def dockerExtraOpts = config.get('dockerExtraOpts', [])
39 def envOpts = config.get('envOpts', [])
azvyagintsevabcf42e2018-10-05 20:40:27 +030040 envOpts.add("DISTRIB_REVISION=${distribRevision}")
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040041 def dockerBaseOpts = [
42 '-u root:root',
43 "--hostname=${dockerHostname}",
44 '--ulimit nofile=4096:8192',
45 "--name=${dockerContainerName}",
46 "--cpus=${dockerMaxCpus}"
47 ]
Denis Egorenko649cf7d2018-10-18 16:36:33 +040048 def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ')
Denis Egorenko5cea1412018-10-18 16:40:11 +040049 def extraReposConfig = null
50 if (baseRepoPreConfig) {
51 // extra repo on mirror.mirantis.net, which is not supported before 2018.11.0 release
52 def extraRepoSource = "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/extra/xenial xenial main"
53 try {
54 def releaseNaming = 'yyyy.MM.dd'
55 def repoDateUsed = new Date().parse(releaseNaming, distribRevision)
56 def extraAvailableFrom = new Date().parse(releaseNaming, '2018.11.0')
57 if (repoDateUsed < extraAvailableFrom) {
Alexander Evseev4983dec2018-10-25 15:49:39 +020058 extraRepoSource = "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
Denis Egorenko5cea1412018-10-18 16:40:11 +040059 }
60 } catch (Exception e) {
61 common.warningMsg(e)
62 if ( !(distribRevision in [ 'nightly', 'proposed', 'testing' ] )) {
azvyagintsev62c97742019-01-02 17:53:54 +020063 extraRepoSource = "deb [arch=amd64] http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
Denis Egorenko5cea1412018-10-18 16:40:11 +040064 }
65 }
66
67 def defaultExtraReposYaml = """
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000068---
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000069aprConfD: |-
70 APT::Get::AllowUnauthenticated 'true';
71 APT::Get::Install-Suggests 'false';
72 APT::Get::Install-Recommends 'false';
73repo:
74 mcp_saltstack:
Denis Egorenko395aa212018-10-11 15:11:28 +040075 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/saltstack-2017.7/xenial xenial main"
Denis Egorenkoe02a1b22018-10-19 17:47:53 +040076 pin:
77 - package: "libsodium18"
78 pin: "release o=SaltStack"
79 priority: 50
80 - package: "*"
81 pin: "release o=SaltStack"
82 priority: "1100"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000083 mcp_extra:
Denis Egorenko3c752a52018-10-12 12:21:29 +040084 source: "${extraRepoSource}"
Denis Egorenko395aa212018-10-11 15:11:28 +040085 mcp_saltformulas:
azvyagintsev058c1f42018-12-14 18:32:33 +020086 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial xenial main"
87 repo_key: "http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial/archive-salt-formulas.key"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000088 ubuntu:
Denis Egorenko395aa212018-10-11 15:11:28 +040089 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000090 ubuntu-upd:
Denis Egorenko395aa212018-10-11 15:11:28 +040091 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-updates main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000092 ubuntu-sec:
Denis Egorenko395aa212018-10-11 15:11:28 +040093 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-security main restricted universe"
94"""
Denis Egorenko5cea1412018-10-18 16:40:11 +040095 // override for now
96 def extraRepoMergeStrategy = config.get('extraRepoMergeStrategy', 'override')
97 def extraRepos = config.get('extraRepos', [:])
98 def defaultRepos = readYaml text: defaultExtraReposYaml
99 if (extraRepoMergeStrategy == 'merge') {
100 extraReposConfig = common.mergeMaps(defaultRepos, extraRepos)
101 } else {
102 extraReposConfig = extraRepos ? extraRepos : defaultRepos
103 }
104 }
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400105 def img = docker.image(dockerImageName)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000106
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400107 img.pull()
108
109 try {
110 img.inside(dockerOptsFinal) {
111 withEnv(envOpts) {
112 try {
113 // Currently, we don't have any other point to install
114 // runtime dependencies for tests.
115 if (baseRepoPreConfig) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000116 // Warning! POssible point of 'allow-downgrades' issue
117 // Probably, need to add such flag into apt.prefs
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400118 sh("""#!/bin/bash -xe
119 echo "Installing extra-deb dependencies inside docker:"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000120 echo > /etc/apt/sources.list
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400121 rm -vf /etc/apt/sources.list.d/* || true
Denis Egorenkoe02a1b22018-10-19 17:47:53 +0400122 rm -vf /etc/apt/preferences.d/* || true
Aleksey Zvyagintsevc5453342018-10-05 15:03:59 +0000123 """)
Denis Egorenko5cea1412018-10-18 16:40:11 +0400124 common.debianExtraRepos(extraReposConfig)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000125 sh('''#!/bin/bash -xe
126 apt-get update
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400127 apt-get install -y python-netaddr
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000128 ''')
129
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400130 }
131 runCommands.sort().each { command, body ->
132 common.warningMsg("Running command: ${command}")
133 // doCall is the closure implementation in groovy, allow to pass arguments to closure
134 body.call()
135 }
136 // If we didn't dropped for now - test has been passed.
137 TestMarkerResult = true
138 }
139 finally {
140 runFinally.sort().each { command, body ->
141 common.warningMsg("Running ${command} command.")
142 // doCall is the closure implementation in groovy, allow to pass arguments to closure
143 body.call()
144 }
145 }
146 }
147 }
148 }
149 catch (Exception er) {
150 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
151 }
152
153 try {
154 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
155 timeout(time: 10, unit: 'SECONDS') {
156 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
157 }
158 timeout(time: 10, unit: 'SECONDS') {
159 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
160 }
161 }
162 catch (Exception er) {
163 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
164 }
165
166 if (TestMarkerResult) {
167 common.infoMsg("Test finished: SUCCESS")
168 } else {
169 common.warningMsg("Test finished: FAILURE")
170 }
171 return TestMarkerResult
172}
173
174/**
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000175 * Wrapper around setupDockerAndTest, to run checks against new Reclass version
176 * that current model is compatible with new Reclass.
177 *
178 * @param config - LinkedHashMap with configuration params:
179 * dockerHostname - (required) Hostname to use for Docker container.
180 * distribRevision - (optional) Revision of packages to use (default proposed).
181 * extraRepo - (optional) Extra repo to use to install new Reclass version. Has
182 * high priority on distribRevision
183 * targetNodes - (required) List nodes to check pillar data.
Denis Egorenkob090a762018-09-12 19:25:41 +0400184 */
185def compareReclassVersions(config) {
186 def common = new com.mirantis.mk.Common()
187 def salt = new com.mirantis.mk.Salt()
188 common.infoMsg("Going to test new reclass for CFG node")
189 def distribRevision = config.get('distribRevision', 'proposed')
190 def venv = config.get('venv')
191 def extraRepo = config.get('extraRepo', '')
192 def extraRepoKey = config.get('extraRepoKey', '')
193 def targetNodes = config.get('targetNodes')
194 sh "rm -rf ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
195 sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
196 def configRun = [
azvyagintsevabcf42e2018-10-05 20:40:27 +0300197 'distribRevision': distribRevision,
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000198 'dockerExtraOpts' : [
Denis Egorenkob090a762018-09-12 19:25:41 +0400199 "-v /srv/salt/reclass:/srv/salt/reclass:ro",
200 "-v /etc/salt:/etc/salt:ro",
201 "-v /usr/share/salt-formulas/:/usr/share/salt-formulas/:ro"
202 ],
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000203 'envOpts' : [
Denis Egorenkob090a762018-09-12 19:25:41 +0400204 "WORKSPACE=${env.WORKSPACE}",
205 "NODES_LIST=${targetNodes.join(' ')}"
206 ],
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000207 'runCommands' : [
208 '001_Update_Reclass_package' : {
209 sh('apt-get update && apt-get install -y reclass')
Denis Egorenkob090a762018-09-12 19:25:41 +0400210 },
211 '002_Test_Reclass_Compatibility': {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000212 sh('''
Denis Egorenkob090a762018-09-12 19:25:41 +0400213 reclass-salt -b /srv/salt/reclass -t > ${WORKSPACE}/new/inventory || exit 1
214 for node in $NODES_LIST; do
215 reclass-salt -b /srv/salt/reclass -p $node > ${WORKSPACE}/new/$node || exit 1
216 done
217 ''')
218 }
219 ]
220 ]
221 if (extraRepo) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000222 // FIXME
Denis Egorenkob090a762018-09-12 19:25:41 +0400223 configRun['runCommands']['0001_Additional_Extra_Repo_Passed'] = {
224 sh("""
225 echo "${extraRepo}" > /etc/apt/sources.list.d/mcp_extra.list
226 [ "${extraRepoKey}" ] && wget -O - ${extraRepoKey} | apt-key add -
227 """)
228 }
Denis Egorenkob090a762018-09-12 19:25:41 +0400229 }
230 if (setupDockerAndTest(configRun)) {
231 common.infoMsg("New reclass version is compatible with current model: SUCCESS")
232 def inventoryOld = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -t", true, null, true).get("return")[0].values()[0]
233 // [0..-31] to exclude 'echo Salt command execution success' from output
234 writeFile(file: "${env.WORKSPACE}/old/inventory", text: inventoryOld[0..-31])
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000235 for (String node in targetNodes) {
Denis Egorenkob090a762018-09-12 19:25:41 +0400236 def nodeOut = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -p ${node}", true, null, true).get("return")[0].values()[0]
237 writeFile(file: "${env.WORKSPACE}/old/${node}", text: nodeOut[0..-31])
238 }
239 def reclassDiff = common.comparePillars(env.WORKSPACE, env.BUILD_URL, '')
240 currentBuild.description = reclassDiff
241 if (reclassDiff != '<b>No job changes</b>') {
242 throw new RuntimeException("Pillars with new reclass version has been changed: FAILED")
243 } else {
244 common.infoMsg("Pillars not changed with new reclass version: SUCCESS")
245 }
246 } else {
247 throw new RuntimeException("New reclass version is not compatible with current model: FAILED")
248 }
249}
250
251/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400252 * Wrapper over setupDockerAndTest, to test CC model.
253 *
254 * @param config - dict with params:
255 * dockerHostname - (required) salt master's name
256 * clusterName - (optional) model cluster name
257 * extraFormulas - (optional) extraFormulas to install. DEPRECATED
258 * formulasSource - (optional) formulas source (git or pkg, default pkg)
259 * reclassVersion - (optional) Version of used reclass (branch, tag, ...) (optional, default master)
260 * reclassEnv - (require) directory of model
261 * ignoreClassNotfound - (optional) Ignore missing classes for reclass model (default false)
262 * aptRepoUrl - (optional) package repository with salt formulas
263 * aptRepoGPG - (optional) GPG key for apt repository with formulas
264 * testContext - (optional) Description of test
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000265 Return: true\exception
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400266 */
267
268def testNode(LinkedHashMap config) {
269 def common = new com.mirantis.mk.Common()
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400270 def dockerHostname = config.get('dockerHostname')
271 def reclassEnv = config.get('reclassEnv')
272 def clusterName = config.get('clusterName', "")
273 def formulasSource = config.get('formulasSource', 'pkg')
274 def extraFormulas = config.get('extraFormulas', 'linux')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400275 def ignoreClassNotfound = config.get('ignoreClassNotfound', false)
276 def aptRepoUrl = config.get('aptRepoUrl', "")
277 def aptRepoGPG = config.get('aptRepoGPG', "")
278 def testContext = config.get('testContext', 'test')
279 config['envOpts'] = [
280 "RECLASS_ENV=${reclassEnv}", "SALT_STOPSTART_WAIT=5",
281 "MASTER_HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}",
282 "MINION_ID=${dockerHostname}", "FORMULAS_SOURCE=${formulasSource}",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400283 "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400284 "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "DEBUG=1",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400285 "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}"
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400286 ]
287
288 config['runCommands'] = [
289 '001_Clone_salt_formulas_scripts': {
azvyagintsev62c97742019-01-02 17:53:54 +0200290 sh(script: 'git clone http://gerrit.mcp.mirantis.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400291 },
292
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000293 '002_Prepare_something' : {
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400294 sh('''rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
Alexander Evseev4983dec2018-10-25 15:49:39 +0200295 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
296 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400297 ''')
298 },
299
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400300 '003_Install_Reclass_package' : {
301 sh('apt-get install -y reclass')
302 },
303
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000304 '004_Run_tests' : {
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400305 def testTimeout = 40 * 60
306 timeout(time: testTimeout, unit: 'SECONDS') {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000307 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400308 source /srv/salt/scripts/bootstrap.sh
309 cd /srv/salt/scripts
310 source_local_envs
311 configure_salt_master
312 configure_salt_minion
313 install_salt_formula_pkg
314 source /srv/salt/scripts/bootstrap.sh
315 cd /srv/salt/scripts
316 saltservice_restart''')
317
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000318 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400319 source /srv/salt/scripts/bootstrap.sh
320 cd /srv/salt/scripts
321 source_local_envs
322 saltmaster_init''')
323
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000324 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400325 source /srv/salt/scripts/bootstrap.sh
326 cd /srv/salt/scripts
327 verify_salt_minions''')
328 }
329 }
330 ]
331 config['runFinally'] = [
332 '001_Archive_artefacts': {
333 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
334 archiveArtifacts artifacts: "nodesinfo.tar.gz"
335 }
336 ]
337 testResult = setupDockerAndTest(config)
338 if (testResult) {
339 common.infoMsg("Node test for context: ${testContext} model: ${reclassEnv} finished: SUCCESS")
340 } else {
341 throw new RuntimeException("Node test for context: ${testContext} model: ${reclassEnv} finished: FAILURE")
342 }
343 return testResult
344}
345
346/**
chnydaf14ea2a2017-05-26 15:07:47 +0200347 * setup and test salt-master
348 *
azvyagintsevb4e0c442018-09-12 17:00:04 +0300349 * @param masterName salt master's name
350 * @param clusterName model cluster name
351 * @param extraFormulas extraFormulas to install. DEPRECATED
352 * @param formulasSource formulas source (git or pkg)
353 * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master)
354 * @param testDir directory of model
355 * @param formulasSource Salt formulas source type (optional, default pkg)
356 * @param formulasRevision APT revision for formulas (optional default stable)
Petr Michalec6414aa52017-08-17 14:32:52 +0200357 * @param ignoreClassNotfound Ignore missing classes for reclass model
azvyagintsevb4e0c442018-09-12 17:00:04 +0300358 * @param dockerMaxCpus max cpus passed to docker (default 0, disabled)
359 * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models)
360 * @param aptRepoUrl package repository with salt formulas
361 * @param aptRepoGPG GPG key for apt repository with formulas
azvyagintsev28fa9d92018-06-26 14:31:49 +0300362 * Return true | false
chnydaf14ea2a2017-05-26 15:07:47 +0200363 */
364
azvyagintsevb4e0c442018-09-12 17:00:04 +0300365def setupAndTestNode(masterName, clusterName, extraFormulas = '*', testDir, formulasSource = 'pkg',
Vasyl Saienko369ed902018-07-23 11:49:32 +0000366 formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0,
azvyagintsev28fa9d92018-06-26 14:31:49 +0300367 ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300368 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300369 // TODO
370 common.errorMsg('You are using deprecated function!Please migrate to "setupDockerAndTest".' +
371 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
372 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300373 // timeout for test execution (40min)
374 def testTimeout = 40 * 60
375 def TestMarkerResult = false
376 def saltOpts = "--retcode-passthrough --force-color"
377 def workspace = common.getWorkspace()
378 def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
379 img.pull()
chnydaf14ea2a2017-05-26 15:07:47 +0200380
azvyagintsev635affb2018-09-13 13:02:54 +0300381 if (formulasSource == 'pkg') {
382 if (extraFormulas) {
383 common.warningMsg("You have passed deprecated variable:extraFormulas=${extraFormulas}. " +
384 "\n It would be ignored, and all formulas would be installed anyway")
385 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300386 }
387 if (!dockerContainerName) {
388 dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString()
389 }
390 def dockerMaxCpusOpt = "--cpus=4"
391 if (dockerMaxCpus > 0) {
392 dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}"
393 }
394 try {
395 img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") {
azvyagintsev635affb2018-09-13 13:02:54 +0300396 withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
azvyagintsevb4e0c442018-09-12 17:00:04 +0300397 "DISTRIB_REVISION=${formulasRevision}",
398 "DEBUG=1", "MASTER_HOSTNAME=${masterName}",
399 "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
400 "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}",
401 "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5",
402 "APT_REPOSITORY_GPG=${aptRepoGPG}"]) {
403 try {
404 // Currently, we don't have any other point to install
405 // runtime dependencies for tests.
406 sh("""#!/bin/bash -xe
azvyagintsev1bfe6842018-08-09 18:40:17 +0200407 echo "Installing extra-deb dependencies inside docker:"
408 echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
409 echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
410 echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
411 rm -vf /etc/apt/sources.list.d/* || true
412 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list
413 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
414 apt-get update
415 apt-get install -y python-netaddr
416 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300417 sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
418 sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
Alexander Evseev4983dec2018-10-25 15:49:39 +0200419 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
420 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
azvyagintsev1bfe6842018-08-09 18:40:17 +0200421 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300422 // FIXME: should be changed to use reclass from mcp_extra_nigtly?
423 sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
azvyagintsev1bfe6842018-08-09 18:40:17 +0200424 sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
425 -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
426 done""")
azvyagintsevb4e0c442018-09-12 17:00:04 +0300427 timeout(time: testTimeout, unit: 'SECONDS') {
428 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200429 source /srv/salt/scripts/bootstrap.sh
430 cd /srv/salt/scripts
431 source_local_envs
432 configure_salt_master
433 configure_salt_minion
434 install_salt_formula_pkg
435 source /srv/salt/scripts/bootstrap.sh
436 cd /srv/salt/scripts
437 saltservice_restart''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300438 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200439 source /srv/salt/scripts/bootstrap.sh
440 cd /srv/salt/scripts
441 source_local_envs
442 saltmaster_init''')
443
azvyagintsevb4e0c442018-09-12 17:00:04 +0300444 if (!legacyTestingMode.toBoolean()) {
445 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200446 source /srv/salt/scripts/bootstrap.sh
447 cd /srv/salt/scripts
448 verify_salt_minions
449 ''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300450 }
451 }
452 // If we didn't dropped for now - test has been passed.
453 TestMarkerResult = true
454 }
455
456 finally {
457 // Collect rendered per-node data.Those info could be simply used
458 // for diff processing. Data was generated via reclass.cli --nodeinfo,
459 /// during verify_salt_minions.
460 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
461 archiveArtifacts artifacts: "nodesinfo.tar.gz"
462 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200463 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200464 }
chnydaf14ea2a2017-05-26 15:07:47 +0200465 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300466 catch (Exception er) {
467 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
468 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300469
azvyagintsevb4e0c442018-09-12 17:00:04 +0300470 if (legacyTestingMode.toBoolean()) {
471 common.infoMsg("Running legacy mode test for master hostname ${masterName}")
472 def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true)
473 for (minion in nodes.tokenize()) {
474 def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true)
475 if (!basename.trim().contains(masterName)) {
476 testMinion(basename.trim())
477 }
478 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300479 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300480
azvyagintsevb4e0c442018-09-12 17:00:04 +0300481 try {
482 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
483 timeout(time: 10, unit: 'SECONDS') {
484 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
485 }
486 timeout(time: 10, unit: 'SECONDS') {
487 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
488 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300489 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300490 catch (Exception er) {
491 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
azvyagintsev28fa9d92018-06-26 14:31:49 +0300492 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300493
azvyagintsevb4e0c442018-09-12 17:00:04 +0300494 if (TestMarkerResult) {
495 common.infoMsg("Test finished: SUCCESS")
496 } else {
497 common.warningMsg("Test finished: FAILURE")
498 }
499 return TestMarkerResult
azvyagintsev28fa9d92018-06-26 14:31:49 +0300500
chnydaf14ea2a2017-05-26 15:07:47 +0200501}
502
503/**
504 * Test salt-minion
505 *
azvyagintsev28fa9d92018-06-26 14:31:49 +0300506 * @param minion salt minion
chnydaf14ea2a2017-05-26 15:07:47 +0200507 */
508
azvyagintsev28fa9d92018-06-26 14:31:49 +0300509def testMinion(minionName) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300510 sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
Jakub Joseffa6ad8d2017-06-26 18:29:55 +0200511}
azvyagintsev2b279d82018-08-07 17:22:54 +0200512
azvyagintsev2b279d82018-08-07 17:22:54 +0200513/**
514 * Wrapper over setupAndTestNode, to test exactly one CC model.
azvyagintsevb4e0c442018-09-12 17:00:04 +0300515 Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
516 Flow: grab all data, and pass to setupAndTestNode function
517 under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
azvyagintsev2b279d82018-08-07 17:22:54 +0200518 *
519 * @param cfg - dict with params:
azvyagintsevb4e0c442018-09-12 17:00:04 +0300520 MODELS_TARGZ http link to arch with (models|contexts|global_reclass)
521 modelFile
522 DockerCName directly passed to setupAndTestNode
523 EXTRA_FORMULAS directly passed to setupAndTestNode
524 DISTRIB_REVISION directly passed to setupAndTestNode
525 reclassVersion directly passed to setupAndTestNode
azvyagintsev2b279d82018-08-07 17:22:54 +0200526
azvyagintsevb4e0c442018-09-12 17:00:04 +0300527 Return: true\exception
azvyagintsev2b279d82018-08-07 17:22:54 +0200528 */
529
530def testCCModel(cfg) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300531 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300532 common.errorMsg('You are using deprecated function!Please migrate to "testNode".' +
533 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
534 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300535 sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
536 sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
537 // unpack data
538 sh(script: "tar -xzf models.tar.gz ")
539 common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
540 content = readFile(file: cfg.modelFile)
541 templateContext = readYaml text: content
542 clusterName = templateContext.default_context.cluster_name
543 clusterDomain = templateContext.default_context.cluster_domain
azvyagintsev2b279d82018-08-07 17:22:54 +0200544
azvyagintsevb4e0c442018-09-12 17:00:04 +0300545 def testResult = false
546 testResult = setupAndTestNode(
547 "cfg01.${clusterDomain}",
548 clusterName,
549 '',
550 cfg.testReclassEnv, // Sync into image exactly one env
551 'pkg',
552 cfg.DISTRIB_REVISION,
553 cfg.reclassVersion,
554 0,
555 false,
556 false,
557 '',
558 '',
559 cfg.DockerCName)
560 if (testResult) {
561 common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
562 } else {
563 throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
564 }
565 return testResult
azvyagintsev2b279d82018-08-07 17:22:54 +0200566}