blob: 0016d708ad4b83c0ad85f732fbf169ed106d57bb [file] [log] [blame]
chnydaf14ea2a2017-05-26 15:07:47 +02001package com.mirantis.mk
2
3/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04004 * Setup Docker to run some tests. Returns true/false based on
5 were tests successful or not.
6 * @param config - LinkedHashMap with configuration params:
7 * dockerHostname - (required) Hostname to use for Docker container.
azvyagintsevabcf42e2018-10-05 20:40:27 +03008 * distribRevision - (optional) Revision of packages to use (default proposed).
Denis Egorenko6fd79ac2018-09-12 13:28:21 +04009 * runCommands - (optional) Dict with closure structure of body required tests. For example:
10 * [ '001_Test': { sh("./run-some-test") }, '002_Test': { sh("./run-another-test") } ]
11 * Before execution runCommands will be sorted by key names. Alpabetical order is preferred.
12 * runFinally - (optional) Dict with closure structure of body required commands, which should be
13 * executed in any case of test results. Same format as for runCommands
14 * updateRepo - (optional) Whether to run common repo update step.
15 * dockerContainerName - (optional) Docker container name.
16 * dockerImageName - (optional) Docker image name
17 * dockerMaxCpus - (optional) Number of CPUS to use in Docker.
18 * dockerExtraOpts - (optional) Array of Docker extra opts for container
19 * envOpts - (optional) Array of variables that should be passed as ENV vars to Docker container.
20 * Return true | false
21 */
22
23def setupDockerAndTest(LinkedHashMap config) {
24 def common = new com.mirantis.mk.Common()
25 def TestMarkerResult = false
26 // setup options
27 def defaultContainerName = 'test-' + UUID.randomUUID().toString()
28 def dockerHostname = config.get('dockerHostname', defaultContainerName)
azvyagintsevabcf42e2018-10-05 20:40:27 +030029 def distribRevision = config.get('distribRevision', 'proposed')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040030 def runCommands = config.get('runCommands', [:])
31 def runFinally = config.get('runFinally', [:])
32 def baseRepoPreConfig = config.get('baseRepoPreConfig', true)
33 def dockerContainerName = config.get('dockerContainerName', defaultContainerName)
34 def dockerImageName = config.get('image', "mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
35 def dockerMaxCpus = config.get('dockerMaxCpus', 4)
36 def dockerExtraOpts = config.get('dockerExtraOpts', [])
37 def envOpts = config.get('envOpts', [])
azvyagintsevabcf42e2018-10-05 20:40:27 +030038 envOpts.add("DISTRIB_REVISION=${distribRevision}")
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040039 def dockerBaseOpts = [
40 '-u root:root',
41 "--hostname=${dockerHostname}",
42 '--ulimit nofile=4096:8192',
43 "--name=${dockerContainerName}",
44 "--cpus=${dockerMaxCpus}"
45 ]
Denis Egorenkod54f60f2018-10-10 19:38:03 +040046 if (baseRepoPreConfig) {
47 // extra repo on mirror.mirantis.net, which is not supported before 2018.11.0 release
48 def extraRepoSource = "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/extra/xenial xenial main"
49 try {
50 def releaseNaming = 'yyyy.MM.dd'
51 def repoDateUsed = new Date().parse(releaseNaming, distribRevision)
52 def extraAvailableFrom = new Date().parse(releaseNaming, '2018.11.0')
53 if (repoDateUsed < extraAvailableFrom) {
54 extraRepoSource = "deb http://apt.mcp.mirantis.net:8085/xenial ${distribRevision} extra"
55 }
56 } catch (Exception e) {
57 common.warningMsg(e)
58 if ( !(distribRevision in [ 'nightly', 'proposed', 'testing' ] )) {
59 extraRepoSource = "deb http://apt.mcp.mirantis.net:8085/xenial ${distribRevision} extra"
60 }
Denis Egorenko3c752a52018-10-12 12:21:29 +040061 }
Denis Egorenko6fd79ac2018-09-12 13:28:21 +040062
Denis Egorenkod54f60f2018-10-10 19:38:03 +040063 def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ')
64 def defaultExtraReposYaml = """
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000065---
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000066aprConfD: |-
67 APT::Get::AllowUnauthenticated 'true';
68 APT::Get::Install-Suggests 'false';
69 APT::Get::Install-Recommends 'false';
70repo:
71 mcp_saltstack:
Denis Egorenko395aa212018-10-11 15:11:28 +040072 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/saltstack-2017.7/xenial xenial main"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000073 pinning: |-
74 Package: libsodium18
75 Pin: release o=SaltStack
76 Pin-Priority: 50
77
78 Package: *
79 Pin: release o=SaltStack
80 Pin-Priority: 1100
81 mcp_extra:
Denis Egorenko3c752a52018-10-12 12:21:29 +040082 source: "${extraRepoSource}"
Denis Egorenko395aa212018-10-11 15:11:28 +040083 mcp_saltformulas:
84 source: "deb http://apt.mcp.mirantis.net:8085/xenial ${distribRevision} salt salt-latest"
85 repo_key: "http://apt.mcp.mirantis.net:8085/public.gpg"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000086 ubuntu:
Denis Egorenko395aa212018-10-11 15:11:28 +040087 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000088 ubuntu-upd:
Denis Egorenko395aa212018-10-11 15:11:28 +040089 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-updates main restricted universe"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +000090 ubuntu-sec:
Denis Egorenko395aa212018-10-11 15:11:28 +040091 source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-security main restricted universe"
92"""
Denis Egorenkod54f60f2018-10-10 19:38:03 +040093 def extraRepoMergeStrategy = config.get('extraRepoMergeStrategy', 'merge')
94 def extraReposYaml = null
95 if (defaultMergeStrategy == 'merge') {
96 def extraReposYamlConfig = config.get('extraReposYaml', '')
97 def extraRepos = readYaml text: extraReposYamlConfig
98 def defaultRepos = readYaml text: defaultExtraReposYaml
99
100 Map.metaClass.mergeNested = { Map rhs ->
101 def lhs = delegate
102 rhs.each { k, v -> lhs[k] = lhs[k] in Map ? lhs[k].addNested(v) : v }
103 lhs
104 }
105
106 extraReposYaml = defaultRepos.mergeNested(extraRepos)
107 } else {
108 extraReposYaml = config.get('extraReposYaml', defaultExtraReposYaml)
109 }
110 }
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400111 def img = docker.image(dockerImageName)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000112
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400113 img.pull()
114
115 try {
116 img.inside(dockerOptsFinal) {
117 withEnv(envOpts) {
118 try {
119 // Currently, we don't have any other point to install
120 // runtime dependencies for tests.
121 if (baseRepoPreConfig) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000122 // Warning! POssible point of 'allow-downgrades' issue
123 // Probably, need to add such flag into apt.prefs
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400124 sh("""#!/bin/bash -xe
125 echo "Installing extra-deb dependencies inside docker:"
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000126 echo > /etc/apt/sources.list
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400127 rm -vf /etc/apt/sources.list.d/* || true
Aleksey Zvyagintsevc5453342018-10-05 15:03:59 +0000128 """)
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000129 common.debianExtraRepos(extraReposYaml)
130 sh('''#!/bin/bash -xe
131 apt-get update
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400132 apt-get install -y python-netaddr
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000133 ''')
134
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400135 }
136 runCommands.sort().each { command, body ->
137 common.warningMsg("Running command: ${command}")
138 // doCall is the closure implementation in groovy, allow to pass arguments to closure
139 body.call()
140 }
141 // If we didn't dropped for now - test has been passed.
142 TestMarkerResult = true
143 }
144 finally {
145 runFinally.sort().each { command, body ->
146 common.warningMsg("Running ${command} command.")
147 // doCall is the closure implementation in groovy, allow to pass arguments to closure
148 body.call()
149 }
150 }
151 }
152 }
153 }
154 catch (Exception er) {
155 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
156 }
157
158 try {
159 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
160 timeout(time: 10, unit: 'SECONDS') {
161 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
162 }
163 timeout(time: 10, unit: 'SECONDS') {
164 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
165 }
166 }
167 catch (Exception er) {
168 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
169 }
170
171 if (TestMarkerResult) {
172 common.infoMsg("Test finished: SUCCESS")
173 } else {
174 common.warningMsg("Test finished: FAILURE")
175 }
176 return TestMarkerResult
177}
178
179/**
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000180 * Wrapper around setupDockerAndTest, to run checks against new Reclass version
181 * that current model is compatible with new Reclass.
182 *
183 * @param config - LinkedHashMap with configuration params:
184 * dockerHostname - (required) Hostname to use for Docker container.
185 * distribRevision - (optional) Revision of packages to use (default proposed).
186 * extraRepo - (optional) Extra repo to use to install new Reclass version. Has
187 * high priority on distribRevision
188 * targetNodes - (required) List nodes to check pillar data.
Denis Egorenkob090a762018-09-12 19:25:41 +0400189 */
190def compareReclassVersions(config) {
191 def common = new com.mirantis.mk.Common()
192 def salt = new com.mirantis.mk.Salt()
193 common.infoMsg("Going to test new reclass for CFG node")
194 def distribRevision = config.get('distribRevision', 'proposed')
195 def venv = config.get('venv')
196 def extraRepo = config.get('extraRepo', '')
197 def extraRepoKey = config.get('extraRepoKey', '')
198 def targetNodes = config.get('targetNodes')
199 sh "rm -rf ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
200 sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
201 def configRun = [
azvyagintsevabcf42e2018-10-05 20:40:27 +0300202 'distribRevision': distribRevision,
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000203 'dockerExtraOpts' : [
Denis Egorenkob090a762018-09-12 19:25:41 +0400204 "-v /srv/salt/reclass:/srv/salt/reclass:ro",
205 "-v /etc/salt:/etc/salt:ro",
206 "-v /usr/share/salt-formulas/:/usr/share/salt-formulas/:ro"
207 ],
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000208 'envOpts' : [
Denis Egorenkob090a762018-09-12 19:25:41 +0400209 "WORKSPACE=${env.WORKSPACE}",
210 "NODES_LIST=${targetNodes.join(' ')}"
211 ],
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000212 'runCommands' : [
213 '001_Update_Reclass_package' : {
214 sh('apt-get update && apt-get install -y reclass')
Denis Egorenkob090a762018-09-12 19:25:41 +0400215 },
216 '002_Test_Reclass_Compatibility': {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000217 sh('''
Denis Egorenkob090a762018-09-12 19:25:41 +0400218 reclass-salt -b /srv/salt/reclass -t > ${WORKSPACE}/new/inventory || exit 1
219 for node in $NODES_LIST; do
220 reclass-salt -b /srv/salt/reclass -p $node > ${WORKSPACE}/new/$node || exit 1
221 done
222 ''')
223 }
224 ]
225 ]
226 if (extraRepo) {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000227 // FIXME
Denis Egorenkob090a762018-09-12 19:25:41 +0400228 configRun['runCommands']['0001_Additional_Extra_Repo_Passed'] = {
229 sh("""
230 echo "${extraRepo}" > /etc/apt/sources.list.d/mcp_extra.list
231 [ "${extraRepoKey}" ] && wget -O - ${extraRepoKey} | apt-key add -
232 """)
233 }
Denis Egorenkob090a762018-09-12 19:25:41 +0400234 }
235 if (setupDockerAndTest(configRun)) {
236 common.infoMsg("New reclass version is compatible with current model: SUCCESS")
237 def inventoryOld = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -t", true, null, true).get("return")[0].values()[0]
238 // [0..-31] to exclude 'echo Salt command execution success' from output
239 writeFile(file: "${env.WORKSPACE}/old/inventory", text: inventoryOld[0..-31])
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000240 for (String node in targetNodes) {
Denis Egorenkob090a762018-09-12 19:25:41 +0400241 def nodeOut = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -p ${node}", true, null, true).get("return")[0].values()[0]
242 writeFile(file: "${env.WORKSPACE}/old/${node}", text: nodeOut[0..-31])
243 }
244 def reclassDiff = common.comparePillars(env.WORKSPACE, env.BUILD_URL, '')
245 currentBuild.description = reclassDiff
246 if (reclassDiff != '<b>No job changes</b>') {
247 throw new RuntimeException("Pillars with new reclass version has been changed: FAILED")
248 } else {
249 common.infoMsg("Pillars not changed with new reclass version: SUCCESS")
250 }
251 } else {
252 throw new RuntimeException("New reclass version is not compatible with current model: FAILED")
253 }
254}
255
256/**
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400257 * Wrapper over setupDockerAndTest, to test CC model.
258 *
259 * @param config - dict with params:
260 * dockerHostname - (required) salt master's name
261 * clusterName - (optional) model cluster name
262 * extraFormulas - (optional) extraFormulas to install. DEPRECATED
263 * formulasSource - (optional) formulas source (git or pkg, default pkg)
264 * reclassVersion - (optional) Version of used reclass (branch, tag, ...) (optional, default master)
265 * reclassEnv - (require) directory of model
266 * ignoreClassNotfound - (optional) Ignore missing classes for reclass model (default false)
267 * aptRepoUrl - (optional) package repository with salt formulas
268 * aptRepoGPG - (optional) GPG key for apt repository with formulas
269 * testContext - (optional) Description of test
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000270 Return: true\exception
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400271 */
272
273def testNode(LinkedHashMap config) {
274 def common = new com.mirantis.mk.Common()
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400275 def dockerHostname = config.get('dockerHostname')
276 def reclassEnv = config.get('reclassEnv')
277 def clusterName = config.get('clusterName', "")
278 def formulasSource = config.get('formulasSource', 'pkg')
279 def extraFormulas = config.get('extraFormulas', 'linux')
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400280 def ignoreClassNotfound = config.get('ignoreClassNotfound', false)
281 def aptRepoUrl = config.get('aptRepoUrl', "")
282 def aptRepoGPG = config.get('aptRepoGPG', "")
283 def testContext = config.get('testContext', 'test')
284 config['envOpts'] = [
285 "RECLASS_ENV=${reclassEnv}", "SALT_STOPSTART_WAIT=5",
286 "MASTER_HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}",
287 "MINION_ID=${dockerHostname}", "FORMULAS_SOURCE=${formulasSource}",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400288 "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400289 "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "DEBUG=1",
Denis Egorenkod54f60f2018-10-10 19:38:03 +0400290 "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}"
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400291 ]
292
293 config['runCommands'] = [
294 '001_Clone_salt_formulas_scripts': {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000295 sh(script: 'git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400296 },
297
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000298 '002_Prepare_something' : {
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400299 sh('''rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
300 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
301 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
302 ''')
303 },
304
Denis Egorenkoc6b24be2018-10-10 17:36:04 +0400305 '003_Install_Reclass_package' : {
306 sh('apt-get install -y reclass')
307 },
308
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000309 '004_Run_tests' : {
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400310 def testTimeout = 40 * 60
311 timeout(time: testTimeout, unit: 'SECONDS') {
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000312 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400313 source /srv/salt/scripts/bootstrap.sh
314 cd /srv/salt/scripts
315 source_local_envs
316 configure_salt_master
317 configure_salt_minion
318 install_salt_formula_pkg
319 source /srv/salt/scripts/bootstrap.sh
320 cd /srv/salt/scripts
321 saltservice_restart''')
322
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000323 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400324 source /srv/salt/scripts/bootstrap.sh
325 cd /srv/salt/scripts
326 source_local_envs
327 saltmaster_init''')
328
Aleksey Zvyagintsevb20bd262018-10-05 15:09:06 +0000329 sh('''#!/bin/bash
Denis Egorenko6fd79ac2018-09-12 13:28:21 +0400330 source /srv/salt/scripts/bootstrap.sh
331 cd /srv/salt/scripts
332 verify_salt_minions''')
333 }
334 }
335 ]
336 config['runFinally'] = [
337 '001_Archive_artefacts': {
338 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
339 archiveArtifacts artifacts: "nodesinfo.tar.gz"
340 }
341 ]
342 testResult = setupDockerAndTest(config)
343 if (testResult) {
344 common.infoMsg("Node test for context: ${testContext} model: ${reclassEnv} finished: SUCCESS")
345 } else {
346 throw new RuntimeException("Node test for context: ${testContext} model: ${reclassEnv} finished: FAILURE")
347 }
348 return testResult
349}
350
351/**
chnydaf14ea2a2017-05-26 15:07:47 +0200352 * setup and test salt-master
353 *
azvyagintsevb4e0c442018-09-12 17:00:04 +0300354 * @param masterName salt master's name
355 * @param clusterName model cluster name
356 * @param extraFormulas extraFormulas to install. DEPRECATED
357 * @param formulasSource formulas source (git or pkg)
358 * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master)
359 * @param testDir directory of model
360 * @param formulasSource Salt formulas source type (optional, default pkg)
361 * @param formulasRevision APT revision for formulas (optional default stable)
Petr Michalec6414aa52017-08-17 14:32:52 +0200362 * @param ignoreClassNotfound Ignore missing classes for reclass model
azvyagintsevb4e0c442018-09-12 17:00:04 +0300363 * @param dockerMaxCpus max cpus passed to docker (default 0, disabled)
364 * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models)
365 * @param aptRepoUrl package repository with salt formulas
366 * @param aptRepoGPG GPG key for apt repository with formulas
azvyagintsev28fa9d92018-06-26 14:31:49 +0300367 * Return true | false
chnydaf14ea2a2017-05-26 15:07:47 +0200368 */
369
azvyagintsevb4e0c442018-09-12 17:00:04 +0300370def setupAndTestNode(masterName, clusterName, extraFormulas = '*', testDir, formulasSource = 'pkg',
Vasyl Saienko369ed902018-07-23 11:49:32 +0000371 formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0,
azvyagintsev28fa9d92018-06-26 14:31:49 +0300372 ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300373 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300374 // TODO
375 common.errorMsg('You are using deprecated function!Please migrate to "setupDockerAndTest".' +
376 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
377 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300378 // timeout for test execution (40min)
379 def testTimeout = 40 * 60
380 def TestMarkerResult = false
381 def saltOpts = "--retcode-passthrough --force-color"
382 def workspace = common.getWorkspace()
383 def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
384 img.pull()
chnydaf14ea2a2017-05-26 15:07:47 +0200385
azvyagintsev635affb2018-09-13 13:02:54 +0300386 if (formulasSource == 'pkg') {
387 if (extraFormulas) {
388 common.warningMsg("You have passed deprecated variable:extraFormulas=${extraFormulas}. " +
389 "\n It would be ignored, and all formulas would be installed anyway")
390 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300391 }
392 if (!dockerContainerName) {
393 dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString()
394 }
395 def dockerMaxCpusOpt = "--cpus=4"
396 if (dockerMaxCpus > 0) {
397 dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}"
398 }
399 try {
400 img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") {
azvyagintsev635affb2018-09-13 13:02:54 +0300401 withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true",
azvyagintsevb4e0c442018-09-12 17:00:04 +0300402 "DISTRIB_REVISION=${formulasRevision}",
403 "DEBUG=1", "MASTER_HOSTNAME=${masterName}",
404 "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
405 "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}",
406 "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5",
407 "APT_REPOSITORY_GPG=${aptRepoGPG}"]) {
408 try {
409 // Currently, we don't have any other point to install
410 // runtime dependencies for tests.
411 sh("""#!/bin/bash -xe
azvyagintsev1bfe6842018-08-09 18:40:17 +0200412 echo "Installing extra-deb dependencies inside docker:"
413 echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
414 echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
415 echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
416 rm -vf /etc/apt/sources.list.d/* || true
417 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list
418 echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
419 apt-get update
420 apt-get install -y python-netaddr
421 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300422 sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
423 sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
azvyagintsev1bfe6842018-08-09 18:40:17 +0200424 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
425 cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
426 """)
azvyagintsevb4e0c442018-09-12 17:00:04 +0300427 // FIXME: should be changed to use reclass from mcp_extra_nigtly?
428 sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
azvyagintsev1bfe6842018-08-09 18:40:17 +0200429 sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
430 -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
431 done""")
azvyagintsevb4e0c442018-09-12 17:00:04 +0300432 timeout(time: testTimeout, unit: 'SECONDS') {
433 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200434 source /srv/salt/scripts/bootstrap.sh
435 cd /srv/salt/scripts
436 source_local_envs
437 configure_salt_master
438 configure_salt_minion
439 install_salt_formula_pkg
440 source /srv/salt/scripts/bootstrap.sh
441 cd /srv/salt/scripts
442 saltservice_restart''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300443 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200444 source /srv/salt/scripts/bootstrap.sh
445 cd /srv/salt/scripts
446 source_local_envs
447 saltmaster_init''')
448
azvyagintsevb4e0c442018-09-12 17:00:04 +0300449 if (!legacyTestingMode.toBoolean()) {
450 sh('''#!/bin/bash
azvyagintsev1bfe6842018-08-09 18:40:17 +0200451 source /srv/salt/scripts/bootstrap.sh
452 cd /srv/salt/scripts
453 verify_salt_minions
454 ''')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300455 }
456 }
457 // If we didn't dropped for now - test has been passed.
458 TestMarkerResult = true
459 }
460
461 finally {
462 // Collect rendered per-node data.Those info could be simply used
463 // for diff processing. Data was generated via reclass.cli --nodeinfo,
464 /// during verify_salt_minions.
465 sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
466 archiveArtifacts artifacts: "nodesinfo.tar.gz"
467 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200468 }
azvyagintsev1bfe6842018-08-09 18:40:17 +0200469 }
chnydaf14ea2a2017-05-26 15:07:47 +0200470 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300471 catch (Exception er) {
472 common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
473 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300474
azvyagintsevb4e0c442018-09-12 17:00:04 +0300475 if (legacyTestingMode.toBoolean()) {
476 common.infoMsg("Running legacy mode test for master hostname ${masterName}")
477 def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true)
478 for (minion in nodes.tokenize()) {
479 def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true)
480 if (!basename.trim().contains(masterName)) {
481 testMinion(basename.trim())
482 }
483 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300484 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300485
azvyagintsevb4e0c442018-09-12 17:00:04 +0300486 try {
487 common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
488 timeout(time: 10, unit: 'SECONDS') {
489 sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
490 }
491 timeout(time: 10, unit: 'SECONDS') {
492 sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
493 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300494 }
azvyagintsevb4e0c442018-09-12 17:00:04 +0300495 catch (Exception er) {
496 common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
azvyagintsev28fa9d92018-06-26 14:31:49 +0300497 }
azvyagintsev28fa9d92018-06-26 14:31:49 +0300498
azvyagintsevb4e0c442018-09-12 17:00:04 +0300499 if (TestMarkerResult) {
500 common.infoMsg("Test finished: SUCCESS")
501 } else {
502 common.warningMsg("Test finished: FAILURE")
503 }
504 return TestMarkerResult
azvyagintsev28fa9d92018-06-26 14:31:49 +0300505
chnydaf14ea2a2017-05-26 15:07:47 +0200506}
507
508/**
509 * Test salt-minion
510 *
azvyagintsev28fa9d92018-06-26 14:31:49 +0300511 * @param minion salt minion
chnydaf14ea2a2017-05-26 15:07:47 +0200512 */
513
azvyagintsev28fa9d92018-06-26 14:31:49 +0300514def testMinion(minionName) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300515 sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
Jakub Joseffa6ad8d2017-06-26 18:29:55 +0200516}
azvyagintsev2b279d82018-08-07 17:22:54 +0200517
azvyagintsev2b279d82018-08-07 17:22:54 +0200518/**
519 * Wrapper over setupAndTestNode, to test exactly one CC model.
azvyagintsevb4e0c442018-09-12 17:00:04 +0300520 Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
521 Flow: grab all data, and pass to setupAndTestNode function
522 under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
azvyagintsev2b279d82018-08-07 17:22:54 +0200523 *
524 * @param cfg - dict with params:
azvyagintsevb4e0c442018-09-12 17:00:04 +0300525 MODELS_TARGZ http link to arch with (models|contexts|global_reclass)
526 modelFile
527 DockerCName directly passed to setupAndTestNode
528 EXTRA_FORMULAS directly passed to setupAndTestNode
529 DISTRIB_REVISION directly passed to setupAndTestNode
530 reclassVersion directly passed to setupAndTestNode
azvyagintsev2b279d82018-08-07 17:22:54 +0200531
azvyagintsevb4e0c442018-09-12 17:00:04 +0300532 Return: true\exception
azvyagintsev2b279d82018-08-07 17:22:54 +0200533 */
534
535def testCCModel(cfg) {
azvyagintsevb4e0c442018-09-12 17:00:04 +0300536 def common = new com.mirantis.mk.Common()
azvyagintsev1cecc092018-09-14 13:19:16 +0300537 common.errorMsg('You are using deprecated function!Please migrate to "testNode".' +
538 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..')
539 sh('sleep 60')
azvyagintsevb4e0c442018-09-12 17:00:04 +0300540 sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
541 sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
542 // unpack data
543 sh(script: "tar -xzf models.tar.gz ")
544 common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
545 content = readFile(file: cfg.modelFile)
546 templateContext = readYaml text: content
547 clusterName = templateContext.default_context.cluster_name
548 clusterDomain = templateContext.default_context.cluster_domain
azvyagintsev2b279d82018-08-07 17:22:54 +0200549
azvyagintsevb4e0c442018-09-12 17:00:04 +0300550 def testResult = false
551 testResult = setupAndTestNode(
552 "cfg01.${clusterDomain}",
553 clusterName,
554 '',
555 cfg.testReclassEnv, // Sync into image exactly one env
556 'pkg',
557 cfg.DISTRIB_REVISION,
558 cfg.reclassVersion,
559 0,
560 false,
561 false,
562 '',
563 '',
564 cfg.DockerCName)
565 if (testResult) {
566 common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
567 } else {
568 throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
569 }
570 return testResult
azvyagintsev2b279d82018-08-07 17:22:54 +0200571}