chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 1 | package com.mirantis.mk |
| 2 | |
| 3 | /** |
| 4 | * setup and test salt-master |
| 5 | * |
| 6 | * @param masterName salt master's name |
Jakub Josef | 2d01c84 | 2017-08-17 13:37:26 +0200 | [diff] [blame] | 7 | * @param clusterName model cluster name |
chnyda | 85f72e6 | 2017-05-26 15:43:27 +0200 | [diff] [blame] | 8 | * @param extraFormulas extraFormulas to install |
Mykyta Karpin | 1cb8d3f | 2017-06-23 15:42:17 +0300 | [diff] [blame] | 9 | * @param formulasSource formulas source (git or pkg) |
Jakub Josef | f5b7e08 | 2018-03-22 14:40:57 +0100 | [diff] [blame] | 10 | * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master) |
chnyda | f4e5eb4 | 2017-05-26 15:34:38 +0200 | [diff] [blame] | 11 | * @param testDir directory of model |
Ruslan Kamaldinov | 00c1ea4 | 2017-08-07 18:45:27 +0400 | [diff] [blame] | 12 | * @param formulasSource Salt formulas source type (optional, default pkg) |
Jakub Josef | fa6ad8d | 2017-06-26 18:29:55 +0200 | [diff] [blame] | 13 | * @param formulasRevision APT revision for formulas (optional default stable) |
Petr Michalec | 6414aa5 | 2017-08-17 14:32:52 +0200 | [diff] [blame] | 14 | * @param ignoreClassNotfound Ignore missing classes for reclass model |
Jakub Josef | 90d8e57 | 2017-10-03 18:44:44 +0200 | [diff] [blame] | 15 | * @param dockerMaxCpus max cpus passed to docker (default 0, disabled) |
| 16 | * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models) |
Dmitry Ukov | 630b9e1 | 2017-10-23 12:00:19 +0400 | [diff] [blame] | 17 | * @param aptRepoUrl package repository with salt formulas |
| 18 | * @param aptRepoGPG GPG key for apt repository with formulas |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 19 | * Return true | false |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 20 | */ |
| 21 | |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 22 | def setupAndTestNode(masterName, clusterName, extraFormulas, testDir, formulasSource = 'pkg', |
Vasyl Saienko | 369ed90 | 2018-07-23 11:49:32 +0000 | [diff] [blame] | 23 | formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0, |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 24 | ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) { |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 25 | def common = new com.mirantis.mk.Common() |
Jakub Josef | 8fc0fd1 | 2018-01-15 18:36:48 +0100 | [diff] [blame] | 26 | // timeout for test execution (40min) |
| 27 | def testTimeout = 40 * 60 |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 28 | def TestMarkerResult = false |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 29 | def saltOpts = "--retcode-passthrough --force-color" |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 30 | def workspace = common.getWorkspace() |
Jakub Josef | ecf8b30 | 2018-05-11 14:33:08 +0000 | [diff] [blame] | 31 | def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7") |
Jakub Josef | f5b7e08 | 2018-03-22 14:40:57 +0100 | [diff] [blame] | 32 | img.pull() |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 33 | |
chnyda | f6a0c95 | 2017-05-29 10:34:45 +0200 | [diff] [blame] | 34 | if (!extraFormulas || extraFormulas == "") { |
chnyda | 77e4697 | 2017-05-26 16:14:27 +0200 | [diff] [blame] | 35 | extraFormulas = "linux" |
chnyda | 66bfc58 | 2017-05-26 16:24:21 +0200 | [diff] [blame] | 36 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 37 | if (!dockerContainerName) { |
| 38 | dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString() |
Dennis Dmitriev | 3c3ee6d | 2017-07-25 20:02:44 +0300 | [diff] [blame] | 39 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 40 | def dockerMaxCpusOpt = "--cpus=4" |
| 41 | if (dockerMaxCpus > 0) { |
| 42 | dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}" |
| 43 | } |
| 44 | try { |
| 45 | img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") { |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 46 | withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", |
| 47 | "DISTRIB_REVISION=${formulasRevision}", |
| 48 | "DEBUG=1", "MASTER_HOSTNAME=${masterName}", |
| 49 | "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}", |
| 50 | "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", |
| 51 | "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5", |
| 52 | "APT_REPOSITORY_GPG=${aptRepoGPG}"]) { |
| 53 | try { |
| 54 | // Currently, we don't have any other point to install |
| 55 | // runtime dependencies for tests. |
| 56 | sh("""#!/bin/bash -xe |
| 57 | echo "Installing extra-deb dependencies inside docker:" |
| 58 | echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode |
| 59 | echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode |
| 60 | echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode |
| 61 | rm -vf /etc/apt/sources.list.d/* || true |
| 62 | echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list |
| 63 | echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list |
| 64 | apt-get update |
| 65 | apt-get install -y python-netaddr |
| 66 | """) |
| 67 | sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true) |
| 68 | sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts |
| 69 | cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\; |
| 70 | cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\; |
| 71 | """) |
| 72 | // FIXME: should be changed to use reclass from mcp_extra_nigtly? |
| 73 | sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do |
| 74 | sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \ |
| 75 | -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion}; |
| 76 | done""") |
| 77 | timeout(time: testTimeout, unit: 'SECONDS') { |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 78 | sh('''#!/bin/bash |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 79 | source /srv/salt/scripts/bootstrap.sh |
| 80 | cd /srv/salt/scripts |
| 81 | source_local_envs |
| 82 | configure_salt_master |
| 83 | configure_salt_minion |
| 84 | install_salt_formula_pkg |
| 85 | source /srv/salt/scripts/bootstrap.sh |
| 86 | cd /srv/salt/scripts |
| 87 | saltservice_restart''') |
| 88 | sh('''#!/bin/bash |
| 89 | source /srv/salt/scripts/bootstrap.sh |
| 90 | cd /srv/salt/scripts |
| 91 | source_local_envs |
| 92 | saltmaster_init''') |
| 93 | |
| 94 | if (!legacyTestingMode.toBoolean()) { |
| 95 | sh('''#!/bin/bash |
| 96 | source /srv/salt/scripts/bootstrap.sh |
| 97 | cd /srv/salt/scripts |
| 98 | verify_salt_minions |
| 99 | ''') |
| 100 | } |
| 101 | } |
| 102 | // If we didn't dropped for now - test has been passed. |
| 103 | TestMarkerResult = true |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 104 | } |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 105 | |
| 106 | finally { |
azvyagintsev | a1ee5ac | 2018-08-08 16:09:58 +0200 | [diff] [blame] | 107 | // Collect rendered per-node data.Those info could be simply used |
| 108 | // for diff processing. Data was generated via reclass.cli --nodeinfo, |
| 109 | /// during verify_salt_minions. |
| 110 | sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true) |
| 111 | archiveArtifacts artifacts: "nodesinfo.tar.gz" |
azvyagintsev | 1bfe684 | 2018-08-09 18:40:17 +0200 | [diff] [blame] | 112 | } |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 113 | } |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 114 | } |
| 115 | } |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 116 | catch (Exception er) { |
| 117 | common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString()) |
| 118 | } |
| 119 | |
| 120 | if (legacyTestingMode.toBoolean()) { |
| 121 | common.infoMsg("Running legacy mode test for master hostname ${masterName}") |
| 122 | def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true) |
| 123 | for (minion in nodes.tokenize()) { |
| 124 | def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true) |
| 125 | if (!basename.trim().contains(masterName)) { |
| 126 | testMinion(basename.trim()) |
| 127 | } |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | try { |
| 132 | common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors") |
| 133 | timeout(time: 10, unit: 'SECONDS') { |
| 134 | sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true) |
| 135 | } |
| 136 | timeout(time: 10, unit: 'SECONDS') { |
| 137 | sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true) |
| 138 | } |
| 139 | } |
| 140 | catch (Exception er) { |
| 141 | common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString()) |
| 142 | } |
| 143 | |
| 144 | if (TestMarkerResult) { |
| 145 | common.infoMsg("Test finished: SUCCESS") |
| 146 | } else { |
| 147 | common.warningMsg("Test finished: FAILURE") |
| 148 | } |
| 149 | return TestMarkerResult |
| 150 | |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 151 | } |
| 152 | |
| 153 | /** |
| 154 | * Test salt-minion |
| 155 | * |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 156 | * @param minion salt minion |
chnyda | f14ea2a | 2017-05-26 15:07:47 +0200 | [diff] [blame] | 157 | */ |
| 158 | |
azvyagintsev | 28fa9d9 | 2018-06-26 14:31:49 +0300 | [diff] [blame] | 159 | def testMinion(minionName) { |
| 160 | sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true) |
Jakub Josef | fa6ad8d | 2017-06-26 18:29:55 +0200 | [diff] [blame] | 161 | } |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 162 | |
| 163 | |
| 164 | /** |
| 165 | * Wrapper over setupAndTestNode, to test exactly one CC model. |
| 166 | Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ |
| 167 | Flow: grab all data, and pass to setupAndTestNode function |
| 168 | under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*` |
| 169 | * |
| 170 | * @param cfg - dict with params: |
| 171 | MODELS_TARGZ http link to arch with (models|contexts|global_reclass) |
| 172 | modelFile |
| 173 | DockerCName directly passed to setupAndTestNode |
| 174 | EXTRA_FORMULAS directly passed to setupAndTestNode |
| 175 | DISTRIB_REVISION directly passed to setupAndTestNode |
| 176 | reclassVersion directly passed to setupAndTestNode |
| 177 | |
| 178 | Return: true\exception |
| 179 | */ |
| 180 | |
| 181 | def testCCModel(cfg) { |
| 182 | def common = new com.mirantis.mk.Common() |
| 183 | sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true) |
| 184 | sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}") |
| 185 | // unpack data |
| 186 | sh(script: "tar -xzf models.tar.gz ") |
| 187 | common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}") |
| 188 | content = readFile(file: cfg.modelFile) |
| 189 | templateContext = readYaml text: content |
| 190 | clusterName = templateContext.default_context.cluster_name |
| 191 | clusterDomain = templateContext.default_context.cluster_domain |
| 192 | |
| 193 | def testResult = false |
| 194 | testResult = setupAndTestNode( |
| 195 | "cfg01.${clusterDomain}", |
| 196 | clusterName, |
| 197 | cfg.EXTRA_FORMULAS, |
| 198 | cfg.testReclassEnv, // Sync into image exactly one env |
| 199 | 'pkg', |
| 200 | cfg.DISTRIB_REVISION, |
| 201 | cfg.reclassVersion, |
| 202 | 0, |
| 203 | false, |
| 204 | false, |
| 205 | '', |
| 206 | '', |
| 207 | cfg.DockerCName) |
| 208 | if (testResult) { |
azvyagintsev | 5ca0b72 | 2018-08-08 10:52:10 +0200 | [diff] [blame] | 209 | common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS") |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 210 | } else { |
azvyagintsev | 5ca0b72 | 2018-08-08 10:52:10 +0200 | [diff] [blame] | 211 | throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE") |
azvyagintsev | 2b279d8 | 2018-08-07 17:22:54 +0200 | [diff] [blame] | 212 | } |
| 213 | return testResult |
| 214 | } |