Merge "Changing deprecated heat CLI by openstack CLI"
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index 43d1869..2d8c8e8 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -254,7 +254,7 @@
* @param ext_variables The list of external variables
* @param results The reports directory
*/
-def runRallyTests(master, target, dockerImageLink, platform, output_dir, repository, branch, scenarios = '', tasks_args_file = '', ext_variables = [], results = '/root/qa_results') {
+def runRallyTests(master, target, dockerImageLink, platform, output_dir, repository, branch, scenarios = '', tasks_args_file = '', ext_variables = [], results = '/root/qa_results', skip_list = '') {
def salt = new com.mirantis.mk.Salt()
def output_file = 'docker-rally.log'
def dest_folder = '/home/rally/qa_results'
@@ -266,6 +266,8 @@
def cmd_rally_task_args = ''
def cmd_report = "rally task export --type junit-xml --to ${dest_folder}/report-rally.xml; " +
"rally task report --out ${dest_folder}/report-rally.html"
+ def cmd_skip_names = ''
+ def cmd_skip_dirs = ''
salt.runSaltProcessStep(master, target, 'file.remove', ["${results}"])
salt.runSaltProcessStep(master, target, 'file.mkdir', ["${results}", "mode=777"])
if (platform == 'openstack') {
@@ -289,13 +291,25 @@
'rally deployment create --fromenv --name=existing; ' +
'rally deployment config; '
cmd_rally_checkout = "git clone -b ${branch ?: 'master'} ${repository} test_config; "
+ if (skip_list != ''){
+ for ( scen in skip_list.split(',') ) {
+ if ( scen.contains('yaml')) {
+ cmd_skip_names += "! -name ${scen} "
+ }
+ else {
+ cmd_skip_dirs += "-path ${scenarios}/${scen} -prune -o "
+ }
+ }
+ }
if (scenarios == '') {
cmd_rally_start = "rally $rally_extra_args task start test_config/rally/scenario.yaml "
} else {
cmd_rally_start = "rally $rally_extra_args task start scenarios.yaml "
cmd_rally_checkout += "if [ -f ${scenarios} ]; then cp ${scenarios} scenarios.yaml; " +
"else " +
- "find -L ${scenarios} -name '*.yaml' -exec cat {} >> scenarios.yaml \\; ; " +
+ "find -L ${scenarios} " + cmd_skip_dirs +
+ " -name '*.yaml' " + cmd_skip_names +
+ "-exec cat {} >> scenarios.yaml \\; ; " +
"sed -i '/---/d' scenarios.yaml; fi; "
}
}
@@ -327,9 +341,10 @@
writeFile file: "${tmp_dir}/k8s-client.crt", text: k8s_client_crt
salt.cmdRun(master, target, "mv ${tmp_dir}/* ${results}/")
salt.runSaltProcessStep(master, target, 'file.rmdir', ["${tmp_dir}"])
- cmd_rally_init = 'set -e ; set -x; if [ ! -w ~/.rally ]; then sudo chown rally:rally ~/.rally ; fi; cd /tmp/; ' +
+ cmd_rally_init = 'set -e ; set -x; cd /tmp/; ' +
"git clone -b ${plugins_branch ?: 'master'} ${plugins_repo} plugins; " +
"sudo pip install --upgrade ./plugins; " +
+ "rally db recreate; " +
"rally env create --name k8s --from-sysenv; " +
"rally env check k8s; "
if (repository == '' ) {
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 7e2874b..9c17d8b 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -204,7 +204,7 @@
// Install docker
if (salt.testTarget(master, "I@docker:host ${extra_tgt}")) {
salt.enforceState(master, "I@docker:host ${extra_tgt}", 'docker.host', true, true, null, false, -1, 3)
- salt.cmdRun(master, "I@docker:host:enabled:true ${extra_tgt}", 'docker ps')
+ salt.cmdRun(master, "I@docker:host and I@docker:host:enabled:true ${extra_tgt}", 'docker ps')
}
// Install keepalived
@@ -693,18 +693,20 @@
def salt = new com.mirantis.mk.Salt()
//Install and Configure Docker
- salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'docker.host')
- salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
- salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'salt.minion.grains')
- salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'mine.update')
- salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'saltutil.refresh_modules')
- sleep(5)
- salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
- if (salt.testTarget(master, "I@docker:swarm:role:manager ${extra_tgt}")){
- salt.enforceState(master, "I@docker:swarm:role:manager ${extra_tgt}", 'docker.swarm')
+ if (salt.testTarget(master, "I@docker:swarm ${extra_tgt}")) {
+ salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'docker.host')
+ salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
+ salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'salt.minion.grains')
+ salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'mine.update')
+ salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'saltutil.refresh_modules')
+ sleep(5)
+ salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
+ if (salt.testTarget(master, "I@docker:swarm:role:manager ${extra_tgt}")){
+ salt.enforceState(master, "I@docker:swarm:role:manager ${extra_tgt}", 'docker.swarm')
+ }
+ sleep(10)
+ salt.cmdRun(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker node ls')
}
- sleep(10)
- salt.cmdRun(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker node ls')
}
@@ -1209,11 +1211,6 @@
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
- if (!common.checkContains('STACK_INSTALL', 'k8s') || !common.checkContains('STACK_INSTALL', 'openstack')) {
- def orchestrate = new com.mirantis.mk.Orchestrate()
- orchestrate.installInfra(master)
- }
-
if (salt.testTarget(master, "I@devops_portal:config ${extra_tgt}")) {
salt.enforceState(master, "I@devops_portal:config ${extra_tgt}", 'devops_portal.config')
salt.enforceState(master, "I@rundeck:client ${extra_tgt}", ['linux.system.user', 'openssh'])
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 6530062..f7724a6 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -248,12 +248,20 @@
def node = out["return"][i];
for(int j=0;j<node.size();j++){
def nodeKey = node.keySet()[j]
- if (!node[nodeKey].contains("Salt command execution success")) {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ if (node[nodeKey] instanceof String) {
+ if (!node[nodeKey].contains("Salt command execution success")) {
+ throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ }
+ } else if (node[nodeKey] instanceof Boolean) {
+ if (!node[nodeKey]) {
+ throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ }
+ } else {
+ throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
}
}
}
- }else{
+ } else {
throw new Exception("Salt Api response doesn't have return param!")
}
}
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 665d0c6..3178ec6 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -22,11 +22,11 @@
def setupAndTestNode(masterName, clusterName, extraFormulas, testDir, formulasSource = 'pkg',
formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0,
ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) {
+ def common = new com.mirantis.mk.Common()
// timeout for test execution (40min)
def testTimeout = 40 * 60
def TestMarkerResult = false
def saltOpts = "--retcode-passthrough --force-color"
- def common = new com.mirantis.mk.Common()
def workspace = common.getWorkspace()
def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
img.pull()
@@ -46,11 +46,24 @@
withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "DISTRIB_REVISION=${formulasRevision}",
"DEBUG=1", "MASTER_HOSTNAME=${masterName}", "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
"RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "APT_REPOSITORY=${aptRepoUrl}",
- "APT_REPOSITORY_GPG=${aptRepoGPG}", "SALT_STOPSTART_WAIT=10"]) {
+ "APT_REPOSITORY_GPG=${aptRepoGPG}", "SALT_STOPSTART_WAIT=5"]) {
+ // Currently, we don't have any other point to install
+ // runtime dependencies for tests.
+ sh('''#!/bin/bash -xe
+ echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
+ echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
+ echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
+ echo 'deb [arch=amd64] http://mirror.mirantis.com/nightly/ubuntu xenial main restricted universe' > /etc/apt/sources.list
+ echo 'deb [arch=amd64] http://mirror.mirantis.com/nightly/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
+ echo "Installing extra-deb dependencies inside docker:"
+ apt-get update
+ apt-get install -y python-netaddr
+ ''')
sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;""")
+ // FIXME: should be changed to use reclass from mcp_extra_nigtly?
sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
-t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
@@ -76,11 +89,17 @@
sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
- verify_salt_minions''')
+ verify_salt_minions
+ ''')
}
}
// If we didn't dropped for now - test has been passed.
TestMarkerResult = true
+ // Collect rendered per-node data.Those info could be simply used
+ // for diff processing. Data was generated via reclass.cli --nodeinfo,
+ /// during verify_salt_minions.
+ sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
+ archiveArtifacts artifacts: "nodesinfo.tar.gz"
}
}
}
@@ -130,3 +149,56 @@
def testMinion(minionName) {
sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
}
+
+
+/**
+ * Wrapper over setupAndTestNode, to test exactly one CC model.
+ Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
+ Flow: grab all data, and pass to setupAndTestNode function
+ under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
+ *
+ * @param cfg - dict with params:
+ MODELS_TARGZ http link to arch with (models|contexts|global_reclass)
+ modelFile
+ DockerCName directly passed to setupAndTestNode
+ EXTRA_FORMULAS directly passed to setupAndTestNode
+ DISTRIB_REVISION directly passed to setupAndTestNode
+ reclassVersion directly passed to setupAndTestNode
+
+ Return: true\exception
+ */
+
+def testCCModel(cfg) {
+ def common = new com.mirantis.mk.Common()
+ sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
+ sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
+ // unpack data
+ sh(script: "tar -xzf models.tar.gz ")
+ common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
+ content = readFile(file: cfg.modelFile)
+ templateContext = readYaml text: content
+ clusterName = templateContext.default_context.cluster_name
+ clusterDomain = templateContext.default_context.cluster_domain
+
+ def testResult = false
+ testResult = setupAndTestNode(
+ "cfg01.${clusterDomain}",
+ clusterName,
+ cfg.EXTRA_FORMULAS,
+ cfg.testReclassEnv, // Sync into image exactly one env
+ 'pkg',
+ cfg.DISTRIB_REVISION,
+ cfg.reclassVersion,
+ 0,
+ false,
+ false,
+ '',
+ '',
+ cfg.DockerCName)
+ if (testResult) {
+ common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
+ } else {
+ throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
+ }
+ return testResult
+}