Merge "Changing deprecated heat CLI by openstack CLI"
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index 43d1869..2d8c8e8 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -254,7 +254,7 @@
  * @param ext_variables     The list of external variables
  * @param results           The reports directory
  */
-def runRallyTests(master, target, dockerImageLink, platform, output_dir, repository, branch, scenarios = '', tasks_args_file = '', ext_variables = [], results = '/root/qa_results') {
+def runRallyTests(master, target, dockerImageLink, platform, output_dir, repository, branch, scenarios = '', tasks_args_file = '', ext_variables = [], results = '/root/qa_results', skip_list = '') {
     def salt = new com.mirantis.mk.Salt()
     def output_file = 'docker-rally.log'
     def dest_folder = '/home/rally/qa_results'
@@ -266,6 +266,8 @@
     def cmd_rally_task_args = ''
     def cmd_report = "rally task export --type junit-xml --to ${dest_folder}/report-rally.xml; " +
         "rally task report --out ${dest_folder}/report-rally.html"
+    def cmd_skip_names = ''
+    def cmd_skip_dirs = ''
     salt.runSaltProcessStep(master, target, 'file.remove', ["${results}"])
     salt.runSaltProcessStep(master, target, 'file.mkdir', ["${results}", "mode=777"])
     if (platform == 'openstack') {
@@ -289,13 +291,25 @@
             'rally deployment create --fromenv --name=existing; ' +
             'rally deployment config; '
         cmd_rally_checkout = "git clone -b ${branch ?: 'master'} ${repository} test_config; "
+        if (skip_list != ''){
+          for ( scen in skip_list.split(',') ) {
+            if ( scen.contains('yaml')) {
+              cmd_skip_names += "! -name ${scen} "
+            }
+            else {
+              cmd_skip_dirs += "-path ${scenarios}/${scen} -prune -o "
+            }
+          }
+        }
         if (scenarios == '') {
           cmd_rally_start = "rally $rally_extra_args task start test_config/rally/scenario.yaml "
         } else {
           cmd_rally_start = "rally $rally_extra_args task start scenarios.yaml "
           cmd_rally_checkout += "if [ -f ${scenarios} ]; then cp ${scenarios} scenarios.yaml; " +
               "else " +
-              "find -L ${scenarios} -name '*.yaml' -exec cat {} >> scenarios.yaml \\; ; " +
+              "find -L ${scenarios} " + cmd_skip_dirs +
+              " -name '*.yaml' " + cmd_skip_names +
+              "-exec cat {} >> scenarios.yaml \\; ; " +
               "sed -i '/---/d' scenarios.yaml; fi; "
         }
       }
@@ -327,9 +341,10 @@
       writeFile file: "${tmp_dir}/k8s-client.crt", text: k8s_client_crt
       salt.cmdRun(master, target, "mv ${tmp_dir}/* ${results}/")
       salt.runSaltProcessStep(master, target, 'file.rmdir', ["${tmp_dir}"])
-      cmd_rally_init = 'set -e ; set -x; if [ ! -w ~/.rally ]; then sudo chown rally:rally ~/.rally ; fi; cd /tmp/; ' +
+      cmd_rally_init = 'set -e ; set -x; cd /tmp/; ' +
           "git clone -b ${plugins_branch ?: 'master'} ${plugins_repo} plugins; " +
           "sudo pip install --upgrade ./plugins; " +
+          "rally db recreate; " +
           "rally env create --name k8s --from-sysenv; " +
           "rally env check k8s; "
       if (repository == '' ) {
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 7e2874b..9c17d8b 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -204,7 +204,7 @@
     // Install docker
     if (salt.testTarget(master, "I@docker:host ${extra_tgt}")) {
         salt.enforceState(master, "I@docker:host ${extra_tgt}", 'docker.host', true, true, null, false, -1, 3)
-        salt.cmdRun(master, "I@docker:host:enabled:true ${extra_tgt}", 'docker ps')
+        salt.cmdRun(master, "I@docker:host and I@docker:host:enabled:true ${extra_tgt}", 'docker ps')
     }
 
     // Install keepalived
@@ -693,18 +693,20 @@
     def salt = new com.mirantis.mk.Salt()
 
     //Install and Configure Docker
-    salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'docker.host')
-    salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
-    salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'salt.minion.grains')
-    salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'mine.update')
-    salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'saltutil.refresh_modules')
-    sleep(5)
-    salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
-    if (salt.testTarget(master, "I@docker:swarm:role:manager ${extra_tgt}")){
-      salt.enforceState(master, "I@docker:swarm:role:manager ${extra_tgt}", 'docker.swarm')
+    if (salt.testTarget(master, "I@docker:swarm ${extra_tgt}")) {
+        salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'docker.host')
+        salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
+        salt.enforceState(master, "I@docker:swarm ${extra_tgt}", 'salt.minion.grains')
+        salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'mine.update')
+        salt.runSaltProcessStep(master, "I@docker:swarm ${extra_tgt}", 'saltutil.refresh_modules')
+        sleep(5)
+        salt.enforceState(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker.swarm')
+        if (salt.testTarget(master, "I@docker:swarm:role:manager ${extra_tgt}")){
+            salt.enforceState(master, "I@docker:swarm:role:manager ${extra_tgt}", 'docker.swarm')
+        }
+        sleep(10)
+        salt.cmdRun(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker node ls')
     }
-    sleep(10)
-    salt.cmdRun(master, "I@docker:swarm:role:master ${extra_tgt}", 'docker node ls')
 }
 
 
@@ -1209,11 +1211,6 @@
   def common = new com.mirantis.mk.Common()
   def salt = new com.mirantis.mk.Salt()
 
-  if (!common.checkContains('STACK_INSTALL', 'k8s') || !common.checkContains('STACK_INSTALL', 'openstack')) {
-    def orchestrate = new com.mirantis.mk.Orchestrate()
-    orchestrate.installInfra(master)
-  }
-
   if (salt.testTarget(master, "I@devops_portal:config ${extra_tgt}")) {
     salt.enforceState(master, "I@devops_portal:config ${extra_tgt}", 'devops_portal.config')
     salt.enforceState(master, "I@rundeck:client ${extra_tgt}", ['linux.system.user', 'openssh'])
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 6530062..f7724a6 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -248,12 +248,20 @@
                 def node = out["return"][i];
                 for(int j=0;j<node.size();j++){
                     def nodeKey = node.keySet()[j]
-                    if (!node[nodeKey].contains("Salt command execution success")) {
-                        throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+                    if (node[nodeKey] instanceof String) {
+                        if (!node[nodeKey].contains("Salt command execution success")) {
+                            throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+                        }
+                    } else if (node[nodeKey] instanceof Boolean) {
+                        if (!node[nodeKey]) {
+                            throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+                        }
+                    } else {
+                        throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
                     }
                 }
             }
-        }else{
+        } else {
             throw new Exception("Salt Api response doesn't have return param!")
         }
     }
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 665d0c6..3178ec6 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -22,11 +22,11 @@
 def setupAndTestNode(masterName, clusterName, extraFormulas, testDir, formulasSource = 'pkg',
                      formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0,
                      ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl = '', aptRepoGPG = '', dockerContainerName = false) {
+  def common = new com.mirantis.mk.Common()
   // timeout for test execution (40min)
   def testTimeout = 40 * 60
   def TestMarkerResult = false
   def saltOpts = "--retcode-passthrough --force-color"
-  def common = new com.mirantis.mk.Common()
   def workspace = common.getWorkspace()
   def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
   img.pull()
@@ -46,11 +46,24 @@
       withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "DISTRIB_REVISION=${formulasRevision}",
                "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
                "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "APT_REPOSITORY=${aptRepoUrl}",
-               "APT_REPOSITORY_GPG=${aptRepoGPG}", "SALT_STOPSTART_WAIT=10"]) {
+               "APT_REPOSITORY_GPG=${aptRepoGPG}", "SALT_STOPSTART_WAIT=5"]) {
+        // Currently, we don't have any other point to install
+        // runtime dependencies for tests.
+        sh('''#!/bin/bash -xe
+              echo "APT::Get::AllowUnauthenticated 'true';"  > /etc/apt/apt.conf.d/99setupAndTestNode
+              echo "APT::Get::Install-Suggests 'false';"  >> /etc/apt/apt.conf.d/99setupAndTestNode
+              echo "APT::Get::Install-Recommends 'false';"  >> /etc/apt/apt.conf.d/99setupAndTestNode
+              echo 'deb [arch=amd64] http://mirror.mirantis.com/nightly/ubuntu xenial main restricted universe' > /etc/apt/sources.list
+              echo 'deb [arch=amd64] http://mirror.mirantis.com/nightly/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
+              echo "Installing extra-deb dependencies inside docker:"
+              apt-get update
+              apt-get install -y python-netaddr
+          ''')
         sh(script: "git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts", returnStdout: true)
         sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2  salt' >> /etc/hosts
               cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
               cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;""")
+        // FIXME: should be changed to use reclass from mcp_extra_nigtly?
         sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
                 sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
                     -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
@@ -76,11 +89,17 @@
             sh('''#!/bin/bash
                   source /srv/salt/scripts/bootstrap.sh
                   cd /srv/salt/scripts
-                  verify_salt_minions''')
+                  verify_salt_minions
+                  ''')
           }
         }
         // If we didn't dropped for now - test has been passed.
         TestMarkerResult = true
+        // Collect rendered per-node data.Those info could be simply used
+        // for diff processing. Data was generated via reclass.cli --nodeinfo,
+        /// during verify_salt_minions.
+        sh(script: "cd /tmp; tar -czf ${env.WORKSPACE}/nodesinfo.tar.gz *reclass*", returnStatus: true)
+        archiveArtifacts artifacts: "nodesinfo.tar.gz"
       }
     }
   }
@@ -130,3 +149,56 @@
 def testMinion(minionName) {
   sh(script: "bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'", returnStdout: true)
 }
+
+
+/**
+ * Wrapper over setupAndTestNode, to test exactly one CC model.
+   Whole workspace and model - should be pre-rendered and passed via MODELS_TARGZ
+   Flow: grab all data, and pass to setupAndTestNode function
+   under-modell will be directly mirrored to `model/{cfg.testReclassEnv}/* /srv/salt/reclass/*`
+ *
+ * @param cfg - dict with params:
+  MODELS_TARGZ       http link to arch with (models|contexts|global_reclass)
+  modelFile
+  DockerCName        directly passed to setupAndTestNode
+  EXTRA_FORMULAS     directly passed to setupAndTestNode
+  DISTRIB_REVISION   directly passed to setupAndTestNode
+  reclassVersion     directly passed to setupAndTestNode
+
+  Return: true\exception
+ */
+
+def testCCModel(cfg) {
+  def common = new com.mirantis.mk.Common()
+  sh(script:  'find . -mindepth 1 -delete || true', returnStatus: true)
+  sh(script: "wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${cfg.MODELS_TARGZ}")
+  // unpack data
+  sh(script: "tar -xzf models.tar.gz ")
+  common.infoMsg("Going to test exactly one context: ${cfg.modelFile}\n, with params: ${cfg}")
+  content = readFile(file: cfg.modelFile)
+  templateContext = readYaml text: content
+  clusterName = templateContext.default_context.cluster_name
+  clusterDomain = templateContext.default_context.cluster_domain
+
+  def testResult = false
+  testResult = setupAndTestNode(
+      "cfg01.${clusterDomain}",
+      clusterName,
+      cfg.EXTRA_FORMULAS,
+      cfg.testReclassEnv, // Sync into image exactly one env
+      'pkg',
+      cfg.DISTRIB_REVISION,
+      cfg.reclassVersion,
+      0,
+      false,
+      false,
+      '',
+      '',
+      cfg.DockerCName)
+  if (testResult) {
+    common.infoMsg("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: SUCCESS")
+  } else {
+    throw new RuntimeException("testCCModel for context: ${cfg.modelFile} model: ${cfg.testReclassEnv} finished: FAILURE")
+  }
+  return testResult
+}