Extend setupAndTestNode EXTRA_FORMULAS logic
* Perform auto-ident
Related-Bug: PROD-23127 (PROD:23127)
Change-Id: I4ddece7250dcdee6d849fe90eea92d30f3a0e708
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 239ba69..6d7b9ad 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -3,57 +3,58 @@
/**
* setup and test salt-master
*
- * @param masterName salt master's name
- * @param clusterName model cluster name
- * @param extraFormulas extraFormulas to install
- * @param formulasSource formulas source (git or pkg)
- * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master)
- * @param testDir directory of model
- * @param formulasSource Salt formulas source type (optional, default pkg)
- * @param formulasRevision APT revision for formulas (optional default stable)
+ * @param masterName salt master's name
+ * @param clusterName model cluster name
+ * @param extraFormulas extraFormulas to install. DEPRECATED
+ * @param formulasSource formulas source (git or pkg)
+ * @param reclassVersion Version of used reclass (branch, tag, ...) (optional, default master)
+ * @param testDir directory of model
+ * @param formulasSource Salt formulas source type (optional, default pkg)
+ * @param formulasRevision APT revision for formulas (optional default stable)
* @param ignoreClassNotfound Ignore missing classes for reclass model
- * @param dockerMaxCpus max cpus passed to docker (default 0, disabled)
- * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models)
- * @param aptRepoUrl package repository with salt formulas
- * @param aptRepoGPG GPG key for apt repository with formulas
+ * @param dockerMaxCpus max cpus passed to docker (default 0, disabled)
+ * @param legacyTestingMode do you want to enable legacy testing mode (iterating through the nodes directory definitions instead of reading cluster models)
+ * @param aptRepoUrl package repository with salt formulas
+ * @param aptRepoGPG GPG key for apt repository with formulas
* Return true | false
*/
-def setupAndTestNode(masterName, clusterName, extraFormulas, testDir, formulasSource = 'pkg',
+def setupAndTestNode(masterName, clusterName, extraFormulas = '*', testDir, formulasSource = 'pkg',
formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0,
ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) {
- def common = new com.mirantis.mk.Common()
- // timeout for test execution (40min)
- def testTimeout = 40 * 60
- def TestMarkerResult = false
- def saltOpts = "--retcode-passthrough --force-color"
- def workspace = common.getWorkspace()
- def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
- img.pull()
+ def common = new com.mirantis.mk.Common()
+ // timeout for test execution (40min)
+ def testTimeout = 40 * 60
+ def TestMarkerResult = false
+ def saltOpts = "--retcode-passthrough --force-color"
+ def workspace = common.getWorkspace()
+ def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
+ img.pull()
- if (!extraFormulas || extraFormulas == "") {
- extraFormulas = "linux"
- }
- if (!dockerContainerName) {
- dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString()
- }
- def dockerMaxCpusOpt = "--cpus=4"
- if (dockerMaxCpus > 0) {
- dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}"
- }
- try {
- img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") {
- withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}",
- "DISTRIB_REVISION=${formulasRevision}",
- "DEBUG=1", "MASTER_HOSTNAME=${masterName}",
- "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
- "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}",
- "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5",
- "APT_REPOSITORY_GPG=${aptRepoGPG}"]) {
- try {
- // Currently, we don't have any other point to install
- // runtime dependencies for tests.
- sh("""#!/bin/bash -xe
+ if (!extraFormulas.contains('*') && formulasSource == 'pkg') {
+ common.warningMsg("You have passed deprecated variable:extraFormulas=${extraFormulas}. " +
+ "\n It would be ignored, and all formulas would be installed anyway")
+ }
+ if (!dockerContainerName) {
+ dockerContainerName = 'setupAndTestNode' + UUID.randomUUID().toString()
+ }
+ def dockerMaxCpusOpt = "--cpus=4"
+ if (dockerMaxCpus > 0) {
+ dockerMaxCpusOpt = "--cpus=${dockerMaxCpus}"
+ }
+ try {
+ img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOpt} --name=${dockerContainerName}") {
+ withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}",
+ "DISTRIB_REVISION=${formulasRevision}",
+ "DEBUG=1", "MASTER_HOSTNAME=${masterName}",
+ "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
+ "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}",
+ "APT_REPOSITORY=${aptRepoUrl}", "SALT_STOPSTART_WAIT=5",
+ "APT_REPOSITORY_GPG=${aptRepoGPG}"]) {
+ try {
+ // Currently, we don't have any other point to install
+ // runtime dependencies for tests.
+ sh("""#!/bin/bash -xe
echo "Installing extra-deb dependencies inside docker:"
echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
@@ -64,18 +65,18 @@
apt-get update
apt-get install -y python-netaddr
""")
- sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
- sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
+ sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
+ sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
""")
- // FIXME: should be changed to use reclass from mcp_extra_nigtly?
- sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
+ // FIXME: should be changed to use reclass from mcp_extra_nigtly?
+ sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
-t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
done""")
- timeout(time: testTimeout, unit: 'SECONDS') {
- sh('''#!/bin/bash
+ timeout(time: testTimeout, unit: 'SECONDS') {
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
source_local_envs
@@ -85,68 +86,68 @@
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
saltservice_restart''')
- sh('''#!/bin/bash
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
source_local_envs
saltmaster_init''')
- if (!legacyTestingMode.toBoolean()) {
- sh('''#!/bin/bash
+ if (!legacyTestingMode.toBoolean()) {
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
verify_salt_minions
''')
- }
+ }
+ }
+ // If we didn't dropped for now - test has been passed.
+ TestMarkerResult = true
+ }
+
+ finally {
+ // Collect rendered per-node data.Those info could be simply used
+ // for diff processing. Data was generated via reclass.cli --nodeinfo,
+ /// during verify_salt_minions.
+ sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
+ archiveArtifacts artifacts: "nodesinfo.tar.gz"
+ }
}
- // If we didn't dropped for now - test has been passed.
- TestMarkerResult = true
- }
-
- finally {
- // Collect rendered per-node data.Those info could be simply used
- // for diff processing. Data was generated via reclass.cli --nodeinfo,
- /// during verify_salt_minions.
- sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
- archiveArtifacts artifacts: "nodesinfo.tar.gz"
}
- }
}
- }
- catch (Exception er) {
- common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
- }
+ catch (Exception er) {
+ common.warningMsg("IgnoreMe:Something wrong with img.Message:\n" + er.toString())
+ }
- if (legacyTestingMode.toBoolean()) {
- common.infoMsg("Running legacy mode test for master hostname ${masterName}")
- def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true)
- for (minion in nodes.tokenize()) {
- def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true)
- if (!basename.trim().contains(masterName)) {
- testMinion(basename.trim())
- }
+ if (legacyTestingMode.toBoolean()) {
+ common.infoMsg("Running legacy mode test for master hostname ${masterName}")
+ def nodes = sh(script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true)
+ for (minion in nodes.tokenize()) {
+ def basename = sh(script: "set +x;basename ${minion} .yml", returnStdout: true)
+ if (!basename.trim().contains(masterName)) {
+ testMinion(basename.trim())
+ }
+ }
}
- }
- try {
- common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
- timeout(time: 10, unit: 'SECONDS') {
- sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
+ try {
+ common.warningMsg("IgnoreMe:Force cleanup slave.Ignore docker-daemon errors")
+ timeout(time: 10, unit: 'SECONDS') {
+ sh(script: "set -x; docker kill ${dockerContainerName} || true", returnStdout: true)
+ }
+ timeout(time: 10, unit: 'SECONDS') {
+ sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
+ }
}
- timeout(time: 10, unit: 'SECONDS') {
- sh(script: "set -x; docker rm --force ${dockerContainerName} || true", returnStdout: true)
+ catch (Exception er) {
+ common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
}
- }
- catch (Exception er) {
- common.warningMsg("IgnoreMe:Timeout to delete test docker container with force!Message:\n" + er.toString())
- }
- if (TestMarkerResult) {
- common.infoMsg("Test finished: SUCCESS")
- } else {
- common.warningMsg("Test finished: FAILURE")
- }
- return TestMarkerResult
+ if (TestMarkerResult) {
+ common.infoMsg("Test finished: SUCCESS")
+ } else {
+ common.warningMsg("Test finished: FAILURE")
+ }
+ return TestMarkerResult
}
@@ -157,58 +158,57 @@
*/
def testMinion(minionName) {
- sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
+ sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
}
-
/**
* Wrapper over setupAndTestNode, to test exactly one CC model.
- Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
- Flow: grab all data, and pass to setupAndTestNode function
- under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
+ Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
+ Flow: grab all data, and pass to setupAndTestNode function
+ under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
*
* @param cfg - dict with params:
- MODELS_TARGZ http link to arch with (models|contexts|global_reclass)
- modelFile
- DockerCName directly passed to setupAndTestNode
- EXTRA_FORMULAS directly passed to setupAndTestNode
- DISTRIB_REVISION directly passed to setupAndTestNode
- reclassVersion directly passed to setupAndTestNode
+ MODELS_TARGZ http link to arch with (models|contexts|global_reclass)
+ modelFile
+ DockerCName directly passed to setupAndTestNode
+ EXTRA_FORMULAS directly passed to setupAndTestNode
+ DISTRIB_REVISION directly passed to setupAndTestNode
+ reclassVersion directly passed to setupAndTestNode
- Return: true\exception
+ Return: true\exception
*/
def testCCModel(cfg) {
- def common = new com.mirantis.mk.Common()
- sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
- sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
- // unpack data
- sh(script: "tar -xzf models.tar.gz ")
- common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
- content = readFile(file: cfg.modelFile)
- templateContext = readYaml text: content
- clusterName = templateContext.default_context.cluster_name
- clusterDomain = templateContext.default_context.cluster_domain
+ def common = new com.mirantis.mk.Common()
+ sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
+ sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
+ // unpack data
+ sh(script: "tar -xzf models.tar.gz ")
+ common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
+ content = readFile(file: cfg.modelFile)
+ templateContext = readYaml text: content
+ clusterName = templateContext.default_context.cluster_name
+ clusterDomain = templateContext.default_context.cluster_domain
- def testResult = false
- testResult = setupAndTestNode(
- "cfg01.${clusterDomain}",
- clusterName,
- cfg.EXTRA_FORMULAS,
- cfg.testReclassEnv, // Sync into image exactly one env
- 'pkg',
- cfg.DISTRIB_REVISION,
- cfg.reclassVersion,
- 0,
- false,
- false,
- '',
- '',
- cfg.DockerCName)
- if (testResult) {
- common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
- } else {
- throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
- }
- return testResult
+ def testResult = false
+ testResult = setupAndTestNode(
+ "cfg01.${clusterDomain}",
+ clusterName,
+ '',
+ cfg.testReclassEnv, // Sync into image exactly one env
+ 'pkg',
+ cfg.DISTRIB_REVISION,
+ cfg.reclassVersion,
+ 0,
+ false,
+ false,
+ '',
+ '',
+ cfg.DockerCName)
+ if (testResult) {
+ common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
+ } else {
+ throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
+ }
+ return testResult
}