chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 1 | package com.mirantis.mk |
| 2 | |
| 3 | /** |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 4 | * Setup Docker to run some tests. Returns true/false based on |
| 5 | were tests successful or not. |
| 6 | * @param config - LinkedHashMap with configuration params: |
| 7 | * dockerHostname - (required) Hostname to use for Docker container. |
azvyagintsev | abcf42e | 2018-10-05 20:40:27 +0300 | [diff] [blame] | 8 | * distribRevision - (optional) Revision of packages to use (default proposed). |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 9 | * runCommands - (optional) Dict with closure structure of body required tests. For example: |
| 10 | * [ '001_Test': { sh("./run-some-test") }, '002_Test': { sh("./run-another-test") } ] |
| 11 | * Before execution runCommands will be sorted by key names. Alpabetical order is preferred. |
| 12 | * runFinally - (optional) Dict with closure structure of body required commands, which should be |
| 13 | * executed in any case of test results. Same format as for runCommands |
| 14 | * updateRepo - (optional) Whether to run common repo update step. |
| 15 | * dockerContainerName - (optional) Docker container name. |
| 16 | * dockerImageName - (optional) Docker image name |
| 17 | * dockerMaxCpus - (optional) Number of CPUS to use in Docker. |
| 18 | * dockerExtraOpts - (optional) Array of Docker extra opts for container |
| 19 | * envOpts - (optional) Array of variables that should be passed as ENV vars to Docker container. |
| 20 | * Return true | false |
| 21 | */ |
| 22 | |
| 23 | def setupDockerAndTest(LinkedHashMap config) { |
| 24 | def common = new com.mirantis.mk.Common() |
| 25 | def TestMarkerResult = false |
| 26 | // setup options |
| 27 | def defaultContainerName = 'test-' + UUID.randomUUID().toString() |
| 28 | def dockerHostname = config.get('dockerHostname', defaultContainerName) |
azvyagintsev | abcf42e | 2018-10-05 20:40:27 +0300 | [diff] [blame] | 29 | def distribRevision = config.get('distribRevision', 'proposed') |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 30 | def runCommands = config.get('runCommands', [:]) |
| 31 | def runFinally = config.get('runFinally', [:]) |
| 32 | def baseRepoPreConfig = config.get('baseRepoPreConfig', true) |
| 33 | def dockerContainerName = config.get('dockerContainerName', defaultContainerName) |
| 34 | def dockerImageName = config.get('image', "mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7") |
| 35 | def dockerMaxCpus = config.get('dockerMaxCpus', 4) |
| 36 | def dockerExtraOpts = config.get('dockerExtraOpts', []) |
| 37 | def envOpts = config.get('envOpts', []) |
azvyagintsev | abcf42e | 2018-10-05 20:40:27 +0300 | [diff] [blame] | 38 | envOpts.add("DISTRIB_REVISION=${distribRevision}") |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 39 | def dockerBaseOpts = [ |
| 40 | '-u root:root', |
| 41 | "--hostname=${dockerHostname}", |
| 42 | '--ulimit nofile=4096:8192', |
| 43 | "--name=${dockerContainerName}", |
| 44 | "--cpus=${dockerMaxCpus}" |
| 45 | ] |
Denis Egorenko | d54f60f | 2018-10-10 19:38:03 +0400 | [diff] [blame^] | 46 | if (baseRepoPreConfig) { |
| 47 | // extra repo on mirror.mirantis.net, which is not supported before 2018.11.0 release |
| 48 | def extraRepoSource = "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/extra/xenial xenial main" |
| 49 | try { |
| 50 | def releaseNaming = 'yyyy.MM.dd' |
| 51 | def repoDateUsed = new Date().parse(releaseNaming, distribRevision) |
| 52 | def extraAvailableFrom = new Date().parse(releaseNaming, '2018.11.0') |
| 53 | if (repoDateUsed < extraAvailableFrom) { |
| 54 | extraRepoSource = "deb http://apt.mcp.mirantis.net:8085/xenial ${distribRevision} extra" |
| 55 | } |
| 56 | } catch (Exception e) { |
| 57 | common.warningMsg(e) |
| 58 | if ( !(distribRevision in [ 'nightly', 'proposed', 'testing' ] )) { |
| 59 | extraRepoSource = "deb http://apt.mcp.mirantis.net:8085/xenial ${distribRevision} extra" |
| 60 | } |
Denis Egorenko | 3c752a5 | 2018-10-12 12:21:29 +0400 | [diff] [blame] | 61 | } |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 62 | |
Denis Egorenko | d54f60f | 2018-10-10 19:38:03 +0400 | [diff] [blame^] | 63 | def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ') |
| 64 | def defaultExtraReposYaml = """ |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 65 | --- |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 66 | aprConfD: |- |
| 67 | APT::Get::AllowUnauthenticated 'true'; |
| 68 | APT::Get::Install-Suggests 'false'; |
| 69 | APT::Get::Install-Recommends 'false'; |
| 70 | repo: |
| 71 | mcp_saltstack: |
Denis Egorenko | 395aa21 | 2018-10-11 15:11:28 +0400 | [diff] [blame] | 72 | source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/saltstack-2017.7/xenial xenial main" |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 73 | pinning: |- |
| 74 | Package: libsodium18 |
| 75 | Pin: release o=SaltStack |
| 76 | Pin-Priority: 50 |
| 77 | |
| 78 | Package: * |
| 79 | Pin: release o=SaltStack |
| 80 | Pin-Priority: 1100 |
| 81 | mcp_extra: |
Denis Egorenko | 3c752a5 | 2018-10-12 12:21:29 +0400 | [diff] [blame] | 82 | source: "${extraRepoSource}" |
Denis Egorenko | 395aa21 | 2018-10-11 15:11:28 +0400 | [diff] [blame] | 83 | mcp_saltformulas: |
| 84 | source: "deb http://apt.mcp.mirantis.net:8085/xenial ${distribRevision} salt salt-latest" |
| 85 | repo_key: "http://apt.mcp.mirantis.net:8085/public.gpg" |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 86 | ubuntu: |
Denis Egorenko | 395aa21 | 2018-10-11 15:11:28 +0400 | [diff] [blame] | 87 | source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial main restricted universe" |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 88 | ubuntu-upd: |
Denis Egorenko | 395aa21 | 2018-10-11 15:11:28 +0400 | [diff] [blame] | 89 | source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-updates main restricted universe" |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 90 | ubuntu-sec: |
Denis Egorenko | 395aa21 | 2018-10-11 15:11:28 +0400 | [diff] [blame] | 91 | source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial-security main restricted universe" |
| 92 | """ |
Denis Egorenko | d54f60f | 2018-10-10 19:38:03 +0400 | [diff] [blame^] | 93 | def extraRepoMergeStrategy = config.get('extraRepoMergeStrategy', 'merge') |
| 94 | def extraReposYaml = null |
| 95 | if (defaultMergeStrategy == 'merge') { |
| 96 | def extraReposYamlConfig = config.get('extraReposYaml', '') |
| 97 | def extraRepos = readYaml text: extraReposYamlConfig |
| 98 | def defaultRepos = readYaml text: defaultExtraReposYaml |
| 99 | |
| 100 | Map.metaClass.mergeNested = { Map rhs -> |
| 101 | def lhs = delegate |
| 102 | rhs.each { k, v -> lhs[k] = lhs[k] in Map ? lhs[k].addNested(v) : v } |
| 103 | lhs |
| 104 | } |
| 105 | |
| 106 | extraReposYaml = defaultRepos.mergeNested(extraRepos) |
| 107 | } else { |
| 108 | extraReposYaml = config.get('extraReposYaml', defaultExtraReposYaml) |
| 109 | } |
| 110 | } |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 111 | def img = docker.image(dockerImageName) |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 112 | |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 113 | img.pull() |
| 114 | |
| 115 | try { |
| 116 | img.inside(dockerOptsFinal) { |
| 117 | withEnv(envOpts) { |
| 118 | try { |
| 119 | // Currently, we don't have any other point to install |
| 120 | // runtime dependencies for tests. |
| 121 | if (baseRepoPreConfig) { |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 122 | // Warning! POssible point of 'allow-downgrades' issue |
| 123 | // Probably, need to add such flag into apt.prefs |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 124 | sh("""#!/bin/bash -xe |
| 125 | echo "Installing extra-deb dependencies inside docker:" |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 126 | echo > /etc/apt/sources.list |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 127 | rm -vf /etc/apt/sources.list.d/* || true |
Aleksey Zvyagintsev | c545334 | 2018-10-05 15:03:59 +0000 | [diff] [blame] | 128 | """) |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 129 | common.debianExtraRepos(extraReposYaml) |
| 130 | sh('''#!/bin/bash -xe |
| 131 | apt-get update |
Denis Egorenko | c6b24be | 2018-10-10 17:36:04 +0400 | [diff] [blame] | 132 | apt-get install -y python-netaddr |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 133 | ''') |
| 134 | |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 135 | } |
| 136 | runCommands.sort().each { command, body -> |
| 137 | common.warningMsg("Running command: ${command}") |
| 138 | // doCall is the closure implementation in groovy, allow to pass arguments to closure |
| 139 | body.call() |
| 140 | } |
| 141 | // If we didn't dropped for now - test has been passed. |
| 142 | TestMarkerResult = true |
| 143 | } |
| 144 | finally { |
| 145 | runFinally.sort().each { command, body -> |
| 146 | common.warningMsg("Running ${command} command.") |
| 147 | // doCall is the closure implementation in groovy, allow to pass arguments to closure |
| 148 | body.call() |
| 149 | } |
| 150 | } |
| 151 | } |
| 152 | } |
| 153 | } |
| 154 | catch (Exception er) { |
| 155 | common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString()) |
| 156 | } |
| 157 | |
| 158 | try { |
| 159 | common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors") |
| 160 | timeout(time: 10, unit: 'SECONDS') { |
| 161 | sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true) |
| 162 | } |
| 163 | timeout(time: 10, unit: 'SECONDS') { |
| 164 | sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true) |
| 165 | } |
| 166 | } |
| 167 | catch (Exception er) { |
| 168 | common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString()) |
| 169 | } |
| 170 | |
| 171 | if (TestMarkerResult) { |
| 172 | common.infoMsg("Test finished: SUCCESS") |
| 173 | } else { |
| 174 | common.warningMsg("Test finished: FAILURE") |
| 175 | } |
| 176 | return TestMarkerResult |
| 177 | } |
| 178 | |
| 179 | /** |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 180 | * Wrapper around setupDockerAndTest, to run checks against new Reclass version |
| 181 | * that current model is compatible with new Reclass. |
| 182 | * |
| 183 | * @param config - LinkedHashMap with configuration params: |
| 184 | * dockerHostname - (required) Hostname to use for Docker container. |
| 185 | * distribRevision - (optional) Revision of packages to use (default proposed). |
| 186 | * extraRepo - (optional) Extra repo to use to install new Reclass version. Has |
| 187 | * high priority on distribRevision |
| 188 | * targetNodes - (required) List nodes to check pillar data. |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 189 | */ |
| 190 | def compareReclassVersions(config) { |
| 191 | def common = new com.mirantis.mk.Common() |
| 192 | def salt = new com.mirantis.mk.Salt() |
| 193 | common.infoMsg("Going to test new reclass for CFG node") |
| 194 | def distribRevision = config.get('distribRevision', 'proposed') |
| 195 | def venv = config.get('venv') |
| 196 | def extraRepo = config.get('extraRepo', '') |
| 197 | def extraRepoKey = config.get('extraRepoKey', '') |
| 198 | def targetNodes = config.get('targetNodes') |
| 199 | sh "rm -rf ${env.WORKSPACE}/old ${env.WORKSPACE}/new" |
| 200 | sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new" |
| 201 | def configRun = [ |
azvyagintsev | abcf42e | 2018-10-05 20:40:27 +0300 | [diff] [blame] | 202 | 'distribRevision': distribRevision, |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 203 | 'dockerExtraOpts' : [ |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 204 | "-v /srv/salt/reclass:/srv/salt/reclass:ro", |
| 205 | "-v /etc/salt:/etc/salt:ro", |
| 206 | "-v /usr/share/salt-formulas/:/usr/share/salt-formulas/:ro" |
| 207 | ], |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 208 | 'envOpts' : [ |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 209 | "WORKSPACE=${env.WORKSPACE}", |
| 210 | "NODES_LIST=${targetNodes.join(' ')}" |
| 211 | ], |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 212 | 'runCommands' : [ |
| 213 | '001_Update_Reclass_package' : { |
| 214 | sh('apt-get update && apt-get install -y reclass') |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 215 | }, |
| 216 | '002_Test_Reclass_Compatibility': { |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 217 | sh(''' |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 218 | reclass-salt -b /srv/salt/reclass -t > ${WORKSPACE}/new/inventory || exit 1 |
| 219 | for node in $NODES_LIST; do |
| 220 | reclass-salt -b /srv/salt/reclass -p $node > ${WORKSPACE}/new/$node || exit 1 |
| 221 | done |
| 222 | ''') |
| 223 | } |
| 224 | ] |
| 225 | ] |
| 226 | if (extraRepo) { |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 227 | // FIXME |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 228 | configRun['runCommands']['0001_Additional_Extra_Repo_Passed'] = { |
| 229 | sh(""" |
| 230 | echo "${extraRepo}" > /etc/apt/sources.list.d/mcp_extra.list |
| 231 | [ "${extraRepoKey}" ] && wget -O - ${extraRepoKey} | apt-key add - |
| 232 | """) |
| 233 | } |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 234 | } |
| 235 | if (setupDockerAndTest(configRun)) { |
| 236 | common.infoMsg("New reclass version is compatible with current model: SUCCESS") |
| 237 | def inventoryOld = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -t", true, null, true).get("return")[0].values()[0] |
| 238 | // [0..-31] to exclude 'echo Salt command execution success' from output |
| 239 | writeFile(file: "${env.WORKSPACE}/old/inventory", text: inventoryOld[0..-31]) |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 240 | for (String node in targetNodes) { |
Denis Egorenko | b090a76 | 2018-09-12 19:25:41 +0400 | [diff] [blame] | 241 | def nodeOut = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -p ${node}", true, null, true).get("return")[0].values()[0] |
| 242 | writeFile(file: "${env.WORKSPACE}/old/${node}", text: nodeOut[0..-31]) |
| 243 | } |
| 244 | def reclassDiff = common.comparePillars(env.WORKSPACE, env.BUILD_URL, '') |
| 245 | currentBuild.description = reclassDiff |
| 246 | if (reclassDiff != '<b>No job changes</b>') { |
| 247 | throw new RuntimeException("Pillars with new reclass version has been changed: FAILED") |
| 248 | } else { |
| 249 | common.infoMsg("Pillars not changed with new reclass version: SUCCESS") |
| 250 | } |
| 251 | } else { |
| 252 | throw new RuntimeException("New reclass version is not compatible with current model: FAILED") |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | /** |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 257 | * Wrapper over setupDockerAndTest, to test CC model. |
| 258 | * |
| 259 | * @param config - dict with params: |
| 260 | * dockerHostname - (required) salt master's name |
| 261 | * clusterName - (optional) model cluster name |
| 262 | * extraFormulas - (optional) extraFormulas to install. DEPRECATED |
| 263 | * formulasSource - (optional) formulas source (git or pkg, default pkg) |
| 264 | * reclassVersion - (optional) Version of used reclass (branch, tag, ...) (optional, default master) |
| 265 | * reclassEnv - (require) directory of model |
| 266 | * ignoreClassNotfound - (optional) Ignore missing classes for reclass model (default false) |
| 267 | * aptRepoUrl - (optional) package repository with salt formulas |
| 268 | * aptRepoGPG - (optional) GPG key for apt repository with formulas |
| 269 | * testContext - (optional) Description of test |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 270 | Return: true\exception |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 271 | */ |
| 272 | |
| 273 | def testNode(LinkedHashMap config) { |
| 274 | def common = new com.mirantis.mk.Common() |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 275 | def dockerHostname = config.get('dockerHostname') |
| 276 | def reclassEnv = config.get('reclassEnv') |
| 277 | def clusterName = config.get('clusterName', "") |
| 278 | def formulasSource = config.get('formulasSource', 'pkg') |
| 279 | def extraFormulas = config.get('extraFormulas', 'linux') |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 280 | def ignoreClassNotfound = config.get('ignoreClassNotfound', false) |
| 281 | def aptRepoUrl = config.get('aptRepoUrl', "") |
| 282 | def aptRepoGPG = config.get('aptRepoGPG', "") |
| 283 | def testContext = config.get('testContext', 'test') |
| 284 | config['envOpts'] = [ |
| 285 | "RECLASS_ENV=${reclassEnv}", "SALT_STOPSTART_WAIT=5", |
| 286 | "MASTER_HOSTNAME=${dockerHostname}", "CLUSTER_NAME=${clusterName}", |
| 287 | "MINION_ID=${dockerHostname}", "FORMULAS_SOURCE=${formulasSource}", |
Denis Egorenko | d54f60f | 2018-10-10 19:38:03 +0400 | [diff] [blame^] | 288 | "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true", |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 289 | "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "DEBUG=1", |
Denis Egorenko | d54f60f | 2018-10-10 19:38:03 +0400 | [diff] [blame^] | 290 | "APT_REPOSITORY=${aptRepoUrl}", "APT_REPOSITORY_GPG=${aptRepoGPG}" |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 291 | ] |
| 292 | |
| 293 | config['runCommands'] = [ |
| 294 | '001_Clone_salt_formulas_scripts': { |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 295 | sh(script: 'git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true) |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 296 | }, |
| 297 | |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 298 | '002_Prepare_something' : { |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 299 | sh('''rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts |
| 300 | cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\; |
| 301 | cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\; |
| 302 | ''') |
| 303 | }, |
| 304 | |
Denis Egorenko | c6b24be | 2018-10-10 17:36:04 +0400 | [diff] [blame] | 305 | '003_Install_Reclass_package' : { |
| 306 | sh('apt-get install -y reclass') |
| 307 | }, |
| 308 | |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 309 | '004_Run_tests' : { |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 310 | def testTimeout = 40 * 60 |
| 311 | timeout(time: testTimeout, unit: 'SECONDS') { |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 312 | sh('''#!/bin/bash |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 313 | source /srv/salt/scripts/bootstrap.sh |
| 314 | cd /srv/salt/scripts |
| 315 | source_local_envs |
| 316 | configure_salt_master |
| 317 | configure_salt_minion |
| 318 | install_salt_formula_pkg |
| 319 | source /srv/salt/scripts/bootstrap.sh |
| 320 | cd /srv/salt/scripts |
| 321 | saltservice_restart''') |
| 322 | |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 323 | sh('''#!/bin/bash |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 324 | source /srv/salt/scripts/bootstrap.sh |
| 325 | cd /srv/salt/scripts |
| 326 | source_local_envs |
| 327 | saltmaster_init''') |
| 328 | |
Aleksey Zvyagintsev | b20bd26 | 2018-10-05 15:09:06 +0000 | [diff] [blame] | 329 | sh('''#!/bin/bash |
Denis Egorenko | 6fd79ac | 2018-09-12 13:28:21 +0400 | [diff] [blame] | 330 | source /srv/salt/scripts/bootstrap.sh |
| 331 | cd /srv/salt/scripts |
| 332 | verify_salt_minions''') |
| 333 | } |
| 334 | } |
| 335 | ] |
| 336 | config['runFinally'] = [ |
| 337 | '001_Archive_artefacts': { |
| 338 | sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true) |
| 339 | archiveArtifacts artifacts: "nodesinfo.tar.gz" |
| 340 | } |
| 341 | ] |
| 342 | testResult = setupDockerAndTest(config) |
| 343 | if (testResult) { |
| 344 | common.infoMsg("Node test for context: ${testContext} model: ${reclassEnv} finished: SUCCESS") |
| 345 | } else { |
| 346 | throw new RuntimeException("Node test for context: ${testContext} model: ${reclassEnv} finished: FAILURE") |
| 347 | } |
| 348 | return testResult |
| 349 | } |
| 350 | |
| 351 | /** |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 352 | * setup and test salt-master |
| 353 | * |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 354 | * @param masterName salt master's name |
| 355 | * @param clusterName model cluster name |
| 356 | * @param extraFormulas extraFormulas to install. DEPRECATED |
| 357 | * @param formulasSource formulas source (git or pkg) |
| 358 | * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master) |
| 359 | * @param testDir directory of model |
| 360 | * @param formulasSource Salt formulas source type (optional, default pkg) |
| 361 | * @param formulasRevision APT revision for formulas (optional default stable) |
Petr Michalec | 6414aa5 | 2017-08-17 14:32:52 +0200 | [diff] [blame] | 362 | * @param ignoreClassNotfound Ignore missing classes for reclass model |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 363 | * @param dockerMaxCpus max cpus passed to docker (default 0, disabled) |
| 364 | * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models) |
| 365 | * @param aptRepoUrl package repository with salt formulas |
| 366 | * @param aptRepoGPG GPG key for apt repository with formulas |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 367 | * Return true | false |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 368 | */ |
| 369 | |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 370 | def setupAndTestNode(masterName, clusterName, extraFormulas = '*', testDir, formulasSource = 'pkg', |
Vasyl Saienko | 369ed90 | 2018-07-23 11:49:32 +0000 | [diff] [blame] | 371 | formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0, |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 372 | ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) { |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 373 | def common = new com.mirantis.mk.Common() |
azvyagintsev | 1cecc09 | 2018-09-14 13:19:16 +0300 | [diff] [blame] | 374 | // TODO |
| 375 | common.errorMsg('You are using deprecated function!Please migrate to "setupDockerAndTest".' + |
| 376 | 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..') |
| 377 | sh('sleep 60') |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 378 | // timeout for test execution (40min) |
| 379 | def testTimeout = 40 * 60 |
| 380 | def TestMarkerResult = false |
| 381 | def saltOpts = "--retcode-passthrough --force-color" |
| 382 | def workspace = common.getWorkspace() |
| 383 | def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7") |
| 384 | img.pull() |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 385 | |
azvyagintsev | 635affb | 2018-09-13 13:02:54 +0300 | [diff] [blame] | 386 | if (formulasSource == 'pkg') { |
| 387 | if (extraFormulas) { |
| 388 | common.warningMsg("You have passed deprecated variable:extraFormulas=${extraFormulas}. " + |
| 389 | "\n It would be ignored, and all formulas would be installed anyway") |
| 390 | } |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 391 | } |
| 392 | if (!dockerContainerName) { |
| 393 | dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString() |
| 394 | } |
| 395 | def dockerMaxCpusOpt = "--cpus=4" |
| 396 | if (dockerMaxCpus > 0) { |
| 397 | dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}" |
| 398 | } |
| 399 | try { |
| 400 | img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") { |
azvyagintsev | 635affb | 2018-09-13 13:02:54 +0300 | [diff] [blame] | 401 | withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "EXTRA_FORMULAS_PKG_ALL=true", |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 402 | "DISTRIB_REVISION=${formulasRevision}", |
| 403 | "DEBUG=1", "MASTER_HOSTNAME=${masterName}", |
| 404 | "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}", |
| 405 | "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", |
| 406 | "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5", |
| 407 | "APT_REPOSITORY_GPG=${aptRepoGPG}"]) { |
| 408 | try { |
| 409 | // Currently, we don't have any other point to install |
| 410 | // runtime dependencies for tests. |
| 411 | sh("""#!/bin/bash -xe |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 412 | echo "Installing extra-deb dependencies inside docker:" |
| 413 | echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode |
| 414 | echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode |
| 415 | echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode |
| 416 | rm -vf /etc/apt/sources.list.d/* || true |
| 417 | echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list |
| 418 | echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list |
| 419 | apt-get update |
| 420 | apt-get install -y python-netaddr |
| 421 | """) |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 422 | sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true) |
| 423 | sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 424 | cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\; |
| 425 | cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\; |
| 426 | """) |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 427 | // FIXME: should be changed to use reclass from mcp_extra_nigtly? |
| 428 | sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 429 | sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \ |
| 430 | -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion}; |
| 431 | done""") |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 432 | timeout(time: testTimeout, unit: 'SECONDS') { |
| 433 | sh('''#!/bin/bash |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 434 | source /srv/salt/scripts/bootstrap.sh |
| 435 | cd /srv/salt/scripts |
| 436 | source_local_envs |
| 437 | configure_salt_master |
| 438 | configure_salt_minion |
| 439 | install_salt_formula_pkg |
| 440 | source /srv/salt/scripts/bootstrap.sh |
| 441 | cd /srv/salt/scripts |
| 442 | saltservice_restart''') |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 443 | sh('''#!/bin/bash |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 444 | source /srv/salt/scripts/bootstrap.sh |
| 445 | cd /srv/salt/scripts |
| 446 | source_local_envs |
| 447 | saltmaster_init''') |
| 448 | |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 449 | if (!legacyTestingMode.toBoolean()) { |
| 450 | sh('''#!/bin/bash |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 451 | source /srv/salt/scripts/bootstrap.sh |
| 452 | cd /srv/salt/scripts |
| 453 | verify_salt_minions |
| 454 | ''') |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 455 | } |
| 456 | } |
| 457 | // If we didn't dropped for now - test has been passed. |
| 458 | TestMarkerResult = true |
| 459 | } |
| 460 | |
| 461 | finally { |
| 462 | // Collect rendered per-node data.Those info could be simply used |
| 463 | // for diff processing. Data was generated via reclass.cli --nodeinfo, |
| 464 | /// during verify_salt_minions. |
| 465 | sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true) |
| 466 | archiveArtifacts artifacts: "nodesinfo.tar.gz" |
| 467 | } |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 468 | } |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 469 | } |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 470 | } |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 471 | catch (Exception er) { |
| 472 | common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString()) |
| 473 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 474 | |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 475 | if (legacyTestingMode.toBoolean()) { |
| 476 | common.infoMsg("Running legacy mode test for master hostname ${masterName}") |
| 477 | def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true) |
| 478 | for (minion in nodes.tokenize()) { |
| 479 | def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true) |
| 480 | if (!basename.trim().contains(masterName)) { |
| 481 | testMinion(basename.trim()) |
| 482 | } |
| 483 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 484 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 485 | |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 486 | try { |
| 487 | common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors") |
| 488 | timeout(time: 10, unit: 'SECONDS') { |
| 489 | sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true) |
| 490 | } |
| 491 | timeout(time: 10, unit: 'SECONDS') { |
| 492 | sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true) |
| 493 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 494 | } |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 495 | catch (Exception er) { |
| 496 | common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString()) |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 497 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 498 | |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 499 | if (TestMarkerResult) { |
| 500 | common.infoMsg("Test finished: SUCCESS") |
| 501 | } else { |
| 502 | common.warningMsg("Test finished: FAILURE") |
| 503 | } |
| 504 | return TestMarkerResult |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 505 | |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 506 | } |
| 507 | |
| 508 | /** |
| 509 | * Test salt-minion |
| 510 | * |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 511 | * @param minion salt minion |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 512 | */ |
| 513 | |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 514 | def testMinion(minionName) { |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 515 | sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true) |
Jakub Josef | fa6ad8d | 2017-06-26 18:29:55 +0200 | [diff] [blame] | 516 | } |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 517 | |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 518 | /** |
| 519 | * Wrapper over setupAndTestNode, to test exactly one CC model. |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 520 | Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ |
| 521 | Flow: grab all data, and pass to setupAndTestNode function |
| 522 | under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*` |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 523 | * |
| 524 | * @param cfg - dict with params: |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 525 | MODELS_TARGZ http link to arch with (models|contexts|global_reclass) |
| 526 | modelFile |
| 527 | DockerCName directly passed to setupAndTestNode |
| 528 | EXTRA_FORMULAS directly passed to setupAndTestNode |
| 529 | DISTRIB_REVISION directly passed to setupAndTestNode |
| 530 | reclassVersion directly passed to setupAndTestNode |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 531 | |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 532 | Return: true\exception |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 533 | */ |
| 534 | |
| 535 | def testCCModel(cfg) { |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 536 | def common = new com.mirantis.mk.Common() |
azvyagintsev | 1cecc09 | 2018-09-14 13:19:16 +0300 | [diff] [blame] | 537 | common.errorMsg('You are using deprecated function!Please migrate to "testNode".' + |
| 538 | 'It would be removed after 2018.q4 release!Pushing forced 60s sleep..') |
| 539 | sh('sleep 60') |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 540 | sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true) |
| 541 | sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}") |
| 542 | // unpack data |
| 543 | sh(script: "tar -xzf models.tar.gz ") |
| 544 | common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}") |
| 545 | content = readFile(file: cfg.modelFile) |
| 546 | templateContext = readYaml text: content |
| 547 | clusterName = templateContext.default_context.cluster_name |
| 548 | clusterDomain = templateContext.default_context.cluster_domain |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 549 | |
azvyagintsev | b4e0c44 | 2018-09-12 17:00:04 +0300 | [diff] [blame] | 550 | def testResult = false |
| 551 | testResult = setupAndTestNode( |
| 552 | "cfg01.${clusterDomain}", |
| 553 | clusterName, |
| 554 | '', |
| 555 | cfg.testReclassEnv, // Sync into image exactly one env |
| 556 | 'pkg', |
| 557 | cfg.DISTRIB_REVISION, |
| 558 | cfg.reclassVersion, |
| 559 | 0, |
| 560 | false, |
| 561 | false, |
| 562 | '', |
| 563 | '', |
| 564 | cfg.DockerCName) |
| 565 | if (testResult) { |
| 566 | common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS") |
| 567 | } else { |
| 568 | throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE") |
| 569 | } |
| 570 | return testResult |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 571 | } |