blob: c4bd4fa8b7d9159726a6f482b9336965f95861e6 [file] [log] [blame]
chnydaf14ea2a2017-05-26 15:07:47 +02001package com.mirantis.mk
2
3/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04004 * Setup Docker to run some tests. Returns true/false based on
5 were tests successful or not.
6 * @param config - LinkedHashMap with configuration params:
7 * dockerHostname - (required) Hostname to use for Docker container.
azvyagintsevabcf42e2018-10-05 20:40:27 +03008 * distribRevision - (optional) Revision of packages to use (default proposed).
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04009 * runCommands - (optional) Dict with closure structure of body required tests. For example:
10 * [ '001_Test': { sh("./run-some-test") }, '002_Test': { sh("./run-another-test") } ]
11 * Before execution runCommands will be sorted by key names. Alpabetical order is preferred.
12 * runFinally - (optional) Dict with closure structure of body required commands, which should be
13 * executed in any case of test results. Same format as for runCommands
14 * updateRepo - (optional) Whether to run common repo update step.
15 * dockerContainerName - (optional) Docker container name.
16 * dockerImageName - (optional) Docker image name
17 * dockerMaxCpus - (optional) Number of CPUS to use in Docker.
18 * dockerExtraOpts - (optional) Array of Docker extra opts for container
19 * envOpts - (optional) Array of variables that should be passed as ENV vars to Docker container.
20 * Return true | false
21 */
22
23def setupDockerAndTest(LinkedHashMap config) {
24 def common = new com.mirantis.mk.Common()
25 def TestMarkerResult = false
26 // setup options
27 def defaultContainerName = 'test-' + UUID.randomUUID().toString()
28 def dockerHostname = config.get('dockerHostname', defaultContainerName)
azvyagintsevabcf42e2018-10-05 20:40:27 +030029 def distribRevision = config.get('distribRevision', 'proposed')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040030 def runCommands = config.get('runCommands', [:])
31 def runFinally = config.get('runFinally', [:])
32 def baseRepoPreConfig = config.get('baseRepoPreConfig', true)
33 def dockerContainerName = config.get('dockerContainerName', defaultContainerName)
azvyagintseve716a452018-11-25 13:41:52 +020034 // def dockerImageName = config.get('image', "mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
35 // FIXME /PROD-25244
36 def dockerImageName = config.get('image', "docker-dev-virtual.docker.mirantis.net/mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040037 def dockerMaxCpus = config.get('dockerMaxCpus', 4)
38 def dockerExtraOpts = config.get('dockerExtraOpts', [])
39 def envOpts = config.get('envOpts', [])
azvyagintsevabcf42e2018-10-05 20:40:27 +030040 envOpts.add("DISTRIB_REVISION=${distribRevision}")
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040041 def dockerBaseOpts = [
42 '-u root:root',
43 "--hostname=${dockerHostname}",
44 '--ulimit nofile=4096:8192',
45 "--name=${dockerContainerName}",
46 "--cpus=${dockerMaxCpus}"
47 ]
Denis Egorenko649cf7d2018-10-18 16:36:33 +040048 def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ')
Denis Egorenko5cea1412018-10-18 16:40:11 +040049 def extraReposConfig = null
50 if (baseRepoPreConfig) {
51 // extra repo on mirror.mirantis.net, which is not supported before 2018.11.0 release
52 def extraRepoSource = "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/extra/xenial xenial main"
53 try {
54 def releaseNaming = 'yyyy.MM.dd'
55 def repoDateUsed = new Date().parse(releaseNaming, distribRevision)
56 def extraAvailableFrom = new Date().parse(releaseNaming, '2018.11.0')
57 if (repoDateUsed < extraAvailableFrom) {
azvyagintsevb9afd232019-02-11 14:22:31 +020058 extraRepoSource = "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
Denis Egorenko5cea1412018-10-18 16:40:11 +040059 }
60 } catch (Exception e) {
61 common.warningMsg(e)
azvyagintsevb9afd232019-02-11 14:22:31 +020062 if (!(distribRevision in ['nightly', 'proposed', 'testing'])) {
azvyagintsev62c97742019-01-02 17:53:54 +020063 extraRepoSource = "deb [arch=amd64] http://apt.mcp.mirantis.net/xenial ${distribRevision} extra"
Denis Egorenko5cea1412018-10-18 16:40:11 +040064 }
65 }
66
67 def defaultExtraReposYaml = """
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000068---
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000069aprConfD: |-
70 APT::Get::AllowUnauthenticated 'true';
71 APT::Get::Install-Suggests 'false';
72 APT::Get::Install-Recommends 'false';
73repo:
74 mcp_saltstack:
Denis Egorenko395aa212018-10-11 15:11:28 +040075 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/saltstack-2017.7/xenial xenial main"
Denis Egorenkoe02a1b22018-10-19 17:47:53 +040076 pin:
77 - package: "libsodium18"
78 pin: "release o=SaltStack"
79 priority: 50
80 - package: "*"
81 pin: "release o=SaltStack"
82 priority: "1100"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000083 mcp_extra:
Denis Egorenko3c752a52018-10-12 12:21:29 +040084 source: "${extraRepoSource}"
Denis Egorenko395aa212018-10-11 15:11:28 +040085 mcp_saltformulas:
azvyagintsev058c1f42018-12-14 18:32:33 +020086 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial xenial main"
87 repo_key: "http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial/archive-salt-formulas.key"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000088 ubuntu:
Denis Egorenko395aa212018-10-11 15:11:28 +040089 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000090 ubuntu-upd:
Denis Egorenko395aa212018-10-11 15:11:28 +040091 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-updates main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000092 ubuntu-sec:
Denis Egorenko395aa212018-10-11 15:11:28 +040093 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-security main restricted universe"
94"""
Denis Egorenko5cea1412018-10-18 16:40:11 +040095 // override for now
96 def extraRepoMergeStrategy = config.get('extraRepoMergeStrategy', 'override')
97 def extraRepos = config.get('extraRepos', [:])
98 def defaultRepos = readYaml text: defaultExtraReposYaml
99 if (extraRepoMergeStrategy == 'merge') {
100 extraReposConfig = common.mergeMaps(defaultRepos, extraRepos)
101 } else {
102 extraReposConfig = extraRepos ? extraRepos : defaultRepos
103 }
104 }
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400105 def img = docker.image(dockerImageName)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000106
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400107 img.pull()
108
109 try {
110 img.inside(dockerOptsFinal) {
111 withEnv(envOpts) {
112 try {
113 // Currently, we don't have any other point to install
114 // runtime dependencies for tests.
115 if (baseRepoPreConfig) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000116 // Warning! POssible point of 'allow-downgrades' issue
117 // Probably, need to add such flag into apt.prefs
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400118 sh("""#!/bin/bash -xe
119 echo "Installing extra-deb dependencies inside docker:"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000120 echo > /etc/apt/sources.list
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400121 rm -vf /etc/apt/sources.list.d/* || true
Denis Egorenkoe02a1b22018-10-19 17:47:53 +0400122 rm -vf /etc/apt/preferences.d/* || true
Aleksey Zvyagintsevc5453342018-10-05 15:03:59 +0000123 """)
Denis Egorenko5cea1412018-10-18 16:40:11 +0400124 common.debianExtraRepos(extraReposConfig)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000125 sh('''#!/bin/bash -xe
126 apt-get update
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400127 apt-get install -y python-netaddr
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000128 ''')
129
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400130 }
131 runCommands.sort().each { command, body ->
132 common.warningMsg("Running command: ${command}")
133 // doCall is the closure implementation in groovy, allow to pass arguments to closure
134 body.call()
135 }
136 // If we didn't dropped for now - test has been passed.
137 TestMarkerResult = true
138 }
139 finally {
140 runFinally.sort().each { command, body ->
141 common.warningMsg("Running ${command} command.")
142 // doCall is the closure implementation in groovy, allow to pass arguments to closure
143 body.call()
144 }
145 }
146 }
147 }
148 }
149 catch (Exception er) {
150 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
151 }
152
153 try {
154 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
155 timeout(time: 10, unit: 'SECONDS') {
156 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
157 }
158 timeout(time: 10, unit: 'SECONDS') {
159 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
160 }
161 }
162 catch (Exception er) {
163 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
164 }
165
166 if (TestMarkerResult) {
167 common.infoMsg("Test finished: SUCCESS")
168 } else {
169 common.warningMsg("Test finished: FAILURE")
170 }
171 return TestMarkerResult
172}
173
174/**
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000175 * Wrapper around setupDockerAndTest, to run checks against new Reclass version
176 * that current model is compatible with new Reclass.
177 *
178 * @param config - LinkedHashMap with configuration params:
179 * dockerHostname - (required) Hostname to use for Docker container.
180 * distribRevision - (optional) Revision of packages to use (default proposed).
181 * extraRepo - (optional) Extra repo to use to install new Reclass version. Has
182 * high priority on distribRevision
183 * targetNodes - (required) List nodes to check pillar data.
Denis Egorenkob090a762018-09-12 19:25:41 +0400184 */
185def compareReclassVersions(config) {
186 def common = new com.mirantis.mk.Common()
187 def salt = new com.mirantis.mk.Salt()
188 common.infoMsg("Going to test new reclass for CFG node")
189 def distribRevision = config.get('distribRevision', 'proposed')
190 def venv = config.get('venv')
191 def extraRepo = config.get('extraRepo', '')
192 def extraRepoKey = config.get('extraRepoKey', '')
193 def targetNodes = config.get('targetNodes')
194 sh "rm -rf ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
195 sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
196 def configRun = [
azvyagintsevabcf42e2018-10-05 20:40:27 +0300197 'distribRevision': distribRevision,
azvyagintsevb9afd232019-02-11 14:22:31 +0200198 'dockerExtraOpts': [
Denis Egorenkob090a762018-09-12 19:25:41 +0400199 "-v /srv/salt/reclass:/srv/salt/reclass:ro",
200 "-v /etc/salt:/etc/salt:ro",
201 "-v /usr/share/salt-formulas/:/usr/share/salt-formulas/:ro"
202 ],
azvyagintsevb9afd232019-02-11 14:22:31 +0200203 'envOpts' : [
Denis Egorenkob090a762018-09-12 19:25:41 +0400204 "WORKSPACE=${env.WORKSPACE}",
205 "NODES_LIST=${targetNodes.join(' ')}"
206 ],
azvyagintsevb9afd232019-02-11 14:22:31 +0200207 'runCommands' : [
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000208 '001_Update_Reclass_package' : {
209 sh('apt-get update && apt-get install -y reclass')
Denis Egorenkob090a762018-09-12 19:25:41 +0400210 },
211 '002_Test_Reclass_Compatibility': {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000212 sh('''
Denis Egorenkob090a762018-09-12 19:25:41 +0400213 reclass-salt -b /srv/salt/reclass -t > ${WORKSPACE}/new/inventory || exit 1
214 for node in $NODES_LIST; do
215 reclass-salt -b /srv/salt/reclass -p $node > ${WORKSPACE}/new/$node || exit 1
216 done
217 ''')
218 }
219 ]
220 ]
221 if (extraRepo) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000222 // FIXME
Denis Egorenkob090a762018-09-12 19:25:41 +0400223 configRun['runCommands']['0001_Additional_Extra_Repo_Passed'] = {
224 sh("""
225 echo "${extraRepo}" > /etc/apt/sources.list.d/mcp_extra.list
226 [ "${extraRepoKey}" ] && wget -O - ${extraRepoKey} | apt-key add -
227 """)
228 }
Denis Egorenkob090a762018-09-12 19:25:41 +0400229 }
230 if (setupDockerAndTest(configRun)) {
231 common.infoMsg("New reclass version is compatible with current model: SUCCESS")
232 def inventoryOld = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -t", true, null, true).get("return")[0].values()[0]
233 // [0..-31] to exclude 'echo Salt command execution success' from output
234 writeFile(file: "${env.WORKSPACE}/old/inventory", text: inventoryOld[0..-31])
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000235 for (String node in targetNodes) {
Denis Egorenkob090a762018-09-12 19:25:41 +0400236 def nodeOut = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -p ${node}", true, null, true).get("return")[0].values()[0]
237 writeFile(file: "${env.WORKSPACE}/old/${node}", text: nodeOut[0..-31])
238 }
239 def reclassDiff = common.comparePillars(env.WORKSPACE, env.BUILD_URL, '')
240 currentBuild.description = reclassDiff
241 if (reclassDiff != '<b>No job changes</b>') {
242 throw new RuntimeException("Pillars with new reclass version has been changed: FAILED")
243 } else {
244 common.infoMsg("Pillars not changed with new reclass version: SUCCESS")
245 }
246 } else {
247 throw new RuntimeException("New reclass version is not compatible with current model: FAILED")
248 }
249}
250
251/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400252 * Wrapper over setupDockerAndTest, to test CC model.
253 *
254 * @param config - dict with params:
255 * dockerHostname - (required) salt master's name
256 * clusterName - (optional) model cluster name
257 * extraFormulas - (optional) extraFormulas to install. DEPRECATED
258 * formulasSource - (optional) formulas source (git or pkg, default pkg)
259 * reclassVersion - (optional) Version of used reclass (branch, tag, ...) (optional, default master)
260 * reclassEnv - (require) directory of model
261 * ignoreClassNotfound - (optional) Ignore missing classes for reclass model (default false)
262 * aptRepoUrl - (optional) package repository with salt formulas
263 * aptRepoGPG - (optional) GPG key for apt repository with formulas
264 * testContext - (optional) Description of test
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000265 Return: true\exception
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400266 */
267
268def testNode(LinkedHashMap config) {
269 def common = new com.mirantis.mk.Common()
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400270 def dockerHostname = config.get('dockerHostname')
Denis Egorenkoe3509c12019-02-19 14:06:26 +0400271 def domain = config.get('domain')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400272 def reclassEnv = config.get('reclassEnv')
273 def clusterName = config.get('clusterName', "")
274 def formulasSource = config.get('formulasSource', 'pkg')
275 def extraFormulas = config.get('extraFormulas', 'linux')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400276 def ignoreClassNotfound = config.get('ignoreClassNotfound', false)
277 def aptRepoUrl = config.get('aptRepoUrl', "")
278 def aptRepoGPG = config.get('aptRepoGPG', "")
279 def testContext = config.get('testContext', 'test')
280 config['envOpts'] = [
281 "RECLASS_ENV=${reclassEnv}", "SALT_STOPSTART_WAIT=5",
Denis Egorenkoe3509c12019-02-19 14:06:26 +0400282 "HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}",
283 "DOMAIN=${domain}", "FORMULAS_SOURCE=${formulasSource}",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400284 "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400285 "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "DEBUG=1",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400286 "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}"
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400287 ]
288
289 config['runCommands'] = [
290 '001_Clone_salt_formulas_scripts': {
azvyagintsev62c97742019-01-02 17:53:54 +0200291 sh(script: 'git clone http://gerrit.mcp.mirantis.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400292 },
293
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000294 '002_Prepare_something' : {
azvyagintsevb9afd232019-02-11 14:22:31 +0200295 sh('''#!/bin/bash -x
296 rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
Denis Egorenkoe3509c12019-02-19 14:06:26 +0400297 echo "127.0.0.1 ${HOSTNAME}.${DOMAIN}" >> /etc/hosts
azvyagintsevb9afd232019-02-11 14:22:31 +0200298 if [ -f '/srv/salt/reclass/salt_master_pillar.asc' ] ; then
299 mkdir -p /etc/salt/gpgkeys
300 chmod 700 /etc/salt/gpgkeys
301 GNUPGHOME=/etc/salt/gpgkeys gpg --import /srv/salt/reclass/salt_master_pillar.asc
302 fi
Alexander Evseev4983dec2018-10-25 15:49:39 +0200303 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
304 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400305 ''')
306 },
307
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400308 '003_Install_Reclass_package' : {
309 sh('apt-get install -y reclass')
310 },
311
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000312 '004_Run_tests' : {
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400313 def testTimeout = 40 * 60
314 timeout(time: testTimeout, unit: 'SECONDS') {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000315 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400316 source /srv/salt/scripts/bootstrap.sh
317 cd /srv/salt/scripts
318 source_local_envs
319 configure_salt_master
320 configure_salt_minion
321 install_salt_formula_pkg
322 source /srv/salt/scripts/bootstrap.sh
323 cd /srv/salt/scripts
324 saltservice_restart''')
325
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000326 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400327 source /srv/salt/scripts/bootstrap.sh
328 cd /srv/salt/scripts
329 source_local_envs
330 saltmaster_init''')
331
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000332 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400333 source /srv/salt/scripts/bootstrap.sh
334 cd /srv/salt/scripts
335 verify_salt_minions''')
336 }
337 }
338 ]
339 config['runFinally'] = [
340 '001_Archive_artefacts': {
341 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
342 archiveArtifacts artifacts: "nodesinfo.tar.gz"
343 }
344 ]
345 testResult = setupDockerAndTest(config)
346 if (testResult) {
347 common.infoMsg("Node test for context: ${testContext} model: ${reclassEnv} finished: SUCCESS")
348 } else {
349 throw new RuntimeException("Node test for context: ${testContext} model: ${reclassEnv} finished: FAILURE")
350 }
351 return testResult
352}
353
354/**
chnydaf14ea2a2017-05-26 15:07:47 +0200355 * setup and test salt-master
356 *
azvyagintsevb4e0c442018-09-12 17:00:04 +0300357 * @param masterName salt master's name
358 * @param clusterName model cluster name
359 * @param extraFormulas extraFormulas to install. DEPRECATED
360 * @param formulasSource formulas source (git or pkg)
361 * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master)
362 * @param testDir directory of model
363 * @param formulasSource Salt formulas source type (optional, default pkg)
364 * @param formulasRevision APT revision for formulas (optional default stable)
Petr Michalec6414aa52017-08-17 14:32:52 +0200365 * @param ignoreClassNotfound Ignore missing classes for reclass model
azvyagintsevb4e0c442018-09-12 17:00:04 +0300366 * @param dockerMaxCpus max cpus passed to docker (default 0, disabled)
367 * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models)
368 * @param aptRepoUrl package repository with salt formulas
369 * @param aptRepoGPG GPG key for apt repository with formulas
azvyagintsev28fa9d92018-06-26 14:31:49 +0300370 * Return true | false
chnydaf14ea2a2017-05-26 15:07:47 +0200371 */
372
azvyagintsevb4e0c442018-09-12 17:00:04 +0300373def setupAndTestNode(masterName, clusterName, extraFormulas = '*', testDir, formulasSource = 'pkg',
Vasyl Saienko369ed902018-07-23 11:49:32 +0000374 formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0,
azvyagintsev28fa9d92018-06-26 14:31:49 +0300375 ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300376 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300377 // TODO
378 common.errorMsg('You are using deprecated function!Please migrate to "setupDockerAndTest".' +
379 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
380 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300381 // timeout for test execution (40min)
382 def testTimeout = 40 * 60
383 def TestMarkerResult = false
384 def saltOpts = "--retcode-passthrough --force-color"
385 def workspace = common.getWorkspace()
386 def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
387 img.pull()
chnydaf14ea2a2017-05-26 15:07:47 +0200388
azvyagintsev635affb2018-09-13 13:02:54 +0300389 if (formulasSource == 'pkg') {
390 if (extraFormulas) {
391 common.warningMsg("You have passed deprecated variable:extraFormulas=${extraFormulas}. " +
392 "\n It would be ignored, and all formulas would be installed anyway")
393 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300394 }
395 if (!dockerContainerName) {
396 dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString()
397 }
398 def dockerMaxCpusOpt = "--cpus=4"
399 if (dockerMaxCpus > 0) {
400 dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}"
401 }
402 try {
403 img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") {
azvyagintsev635affb2018-09-13 13:02:54 +0300404 withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
azvyagintsevb4e0c442018-09-12 17:00:04 +0300405 "DISTRIB_REVISION=${formulasRevision}",
406 "DEBUG=1", "MASTER_HOSTNAME=${masterName}",
407 "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
408 "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}",
409 "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5",
410 "APT_REPOSITORY_GPG=${aptRepoGPG}"]) {
411 try {
412 // Currently, we don't have any other point to install
413 // runtime dependencies for tests.
414 sh("""#!/bin/bash -xe
azvyagintsev1bfe6842018-08-09 18:40:17 +0200415 echo "Installing extra-deb dependencies inside docker:"
416 echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
417 echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
418 echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
419 rm -vf /etc/apt/sources.list.d/* || true
420 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list
421 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
422 apt-get update
423 apt-get install -y python-netaddr
424 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300425 sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
426 sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
Alexander Evseev4983dec2018-10-25 15:49:39 +0200427 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
428 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;
azvyagintsev1bfe6842018-08-09 18:40:17 +0200429 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300430 // FIXME: should be changed to use reclass from mcp_extra_nigtly?
431 sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
azvyagintsev1bfe6842018-08-09 18:40:17 +0200432 sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
433 -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
434 done""")
azvyagintsevb4e0c442018-09-12 17:00:04 +0300435 timeout(time: testTimeout, unit: 'SECONDS') {
436 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200437 source /srv/salt/scripts/bootstrap.sh
438 cd /srv/salt/scripts
439 source_local_envs
440 configure_salt_master
441 configure_salt_minion
442 install_salt_formula_pkg
443 source /srv/salt/scripts/bootstrap.sh
444 cd /srv/salt/scripts
445 saltservice_restart''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300446 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200447 source /srv/salt/scripts/bootstrap.sh
448 cd /srv/salt/scripts
449 source_local_envs
450 saltmaster_init''')
451
azvyagintsevb4e0c442018-09-12 17:00:04 +0300452 if (!legacyTestingMode.toBoolean()) {
453 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200454 source /srv/salt/scripts/bootstrap.sh
455 cd /srv/salt/scripts
456 verify_salt_minions
457 ''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300458 }
459 }
460 // If we didn't dropped for now - test has been passed.
461 TestMarkerResult = true
462 }
463
464 finally {
465 // Collect rendered per-node data.Those info could be simply used
466 // for diff processing. Data was generated via reclass.cli --nodeinfo,
467 /// during verify_salt_minions.
468 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
469 archiveArtifacts artifacts: "nodesinfo.tar.gz"
470 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200471 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200472 }
chnydaf14ea2a2017-05-26 15:07:47 +0200473 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300474 catch (Exception er) {
475 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
476 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300477
azvyagintsevb4e0c442018-09-12 17:00:04 +0300478 if (legacyTestingMode.toBoolean()) {
479 common.infoMsg("Running legacy mode test for master hostname ${masterName}")
480 def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true)
481 for (minion in nodes.tokenize()) {
482 def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true)
483 if (!basename.trim().contains(masterName)) {
484 testMinion(basename.trim())
485 }
486 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300487 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300488
azvyagintsevb4e0c442018-09-12 17:00:04 +0300489 try {
490 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
491 timeout(time: 10, unit: 'SECONDS') {
492 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
493 }
494 timeout(time: 10, unit: 'SECONDS') {
495 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
496 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300497 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300498 catch (Exception er) {
499 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
azvyagintsev28fa9d92018-06-26 14:31:49 +0300500 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300501
azvyagintsevb4e0c442018-09-12 17:00:04 +0300502 if (TestMarkerResult) {
503 common.infoMsg("Test finished: SUCCESS")
504 } else {
505 common.warningMsg("Test finished: FAILURE")
506 }
507 return TestMarkerResult
azvyagintsev28fa9d92018-06-26 14:31:49 +0300508
chnydaf14ea2a2017-05-26 15:07:47 +0200509}
510
511/**
512 * Test salt-minion
513 *
azvyagintsev28fa9d92018-06-26 14:31:49 +0300514 * @param minion salt minion
chnydaf14ea2a2017-05-26 15:07:47 +0200515 */
516
azvyagintsev28fa9d92018-06-26 14:31:49 +0300517def testMinion(minionName) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300518 sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
Jakub Joseffa6ad8d2017-06-26 18:29:55 +0200519}
azvyagintsev2b279d82018-08-07 17:22:54 +0200520
azvyagintsev2b279d82018-08-07 17:22:54 +0200521/**
522 * Wrapper over setupAndTestNode, to test exactly one CC model.
azvyagintsevb4e0c442018-09-12 17:00:04 +0300523 Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
524 Flow: grab all data, and pass to setupAndTestNode function
525 under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
azvyagintsev2b279d82018-08-07 17:22:54 +0200526 *
527 * @param cfg - dict with params:
azvyagintsevb4e0c442018-09-12 17:00:04 +0300528 MODELS_TARGZ http link to arch with (models|contexts|global_reclass)
529 modelFile
530 DockerCName directly passed to setupAndTestNode
531 EXTRA_FORMULAS directly passed to setupAndTestNode
532 DISTRIB_REVISION directly passed to setupAndTestNode
533 reclassVersion directly passed to setupAndTestNode
azvyagintsev2b279d82018-08-07 17:22:54 +0200534
azvyagintsevb4e0c442018-09-12 17:00:04 +0300535 Return: true\exception
azvyagintsev2b279d82018-08-07 17:22:54 +0200536 */
537
538def testCCModel(cfg) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300539 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300540 common.errorMsg('You are using deprecated function!Please migrate to "testNode".' +
541 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
542 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300543 sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
544 sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
545 // unpack data
546 sh(script: "tar -xzf models.tar.gz ")
547 common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
548 content = readFile(file: cfg.modelFile)
549 templateContext = readYaml text: content
550 clusterName = templateContext.default_context.cluster_name
551 clusterDomain = templateContext.default_context.cluster_domain
azvyagintsev2b279d82018-08-07 17:22:54 +0200552
azvyagintsevb4e0c442018-09-12 17:00:04 +0300553 def testResult = false
554 testResult = setupAndTestNode(
555 "cfg01.${clusterDomain}",
556 clusterName,
557 '',
558 cfg.testReclassEnv, // Sync into image exactly one env
559 'pkg',
560 cfg.DISTRIB_REVISION,
561 cfg.reclassVersion,
562 0,
563 false,
564 false,
565 '',
566 '',
567 cfg.DockerCName)
568 if (testResult) {
569 common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
570 } else {
571 throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
572 }
573 return testResult
azvyagintsev2b279d82018-08-07 17:22:54 +0200574}