Merge "Small fixes: [1] Script name changed; [2] CVP sanity run command changed"
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index d919379..670b101 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -162,6 +162,33 @@
 }
 
 /**
+ * Execute pytest framework tests
+ *
+ * @param salt_url          Salt master url
+ * @param salt_credentials  Salt credentials
+ * @param test_set          Test set to run
+ * @param env_vars          Additional environment variables for cvp-sanity-checks
+ * @param output_dir        Directory for results
+ */
+def runTests(salt_url, salt_credentials, test_set="", output_dir="validation_artifacts/", env_vars="") {
+    def common = new com.mirantis.mk.Common()
+    def creds = common.getCredentials(salt_credentials)
+    def username = creds.username
+    def password = creds.password
+    def settings = ""
+    if ( env_vars != "" ) {
+        for (var in env_vars.tokenize(";")) {
+            settings += "export ${var}; "
+        }
+    }
+    def script = ". ${env.WORKSPACE}/venv/bin/activate; ${settings}" +
+                 "pytest --junitxml ${output_dir}report.xml --tb=short -sv ${env.WORKSPACE}/${test_set}"
+    withEnv(["SALT_USERNAME=${username}", "SALT_PASSWORD=${password}", "SALT_URL=${salt_url}"]) {
+        def statusCode = sh script:script, returnStatus:true
+    }
+}
+
+/**
  * Execute tempest tests
  *
  * @param target            Host to run tests
@@ -244,7 +271,8 @@
         'rally task start combined_scenario.yaml ' +
         '--task-args-file /opt/devops-qa-tools/rally-scenarios/task_arguments.yaml; '
     if (repository != '' ) {
-        cmd = 'rally deployment create --fromenv --name=existing; ' +
+        cmd = 'rally db create; ' +
+            'rally deployment create --fromenv --name=existing; ' +
             'rally deployment config; '
         if (scenarios == '') {
           cmd += 'rally task start test_config/rally/scenario.yaml '
@@ -271,8 +299,11 @@
     cmd += "rally task export --type junit-xml --to ${dest_folder}/report-rally.xml; " +
         "rally task report --out ${dest_folder}/report-rally.html"
     full_cmd = cmd0 + cmd
+    salt.runSaltProcessStep(master, target, 'file.touch', ["${results}/rally.db"])
+    salt.cmdRun(master, target, "chmod 666 ${results}/rally.db")
     salt.cmdRun(master, target, "docker run -i --rm --net=host -e ${env_vars} " +
         "-v ${results}:${dest_folder} " +
+        "-v ${results}/rally.db:/home/rally/.rally/rally.db " +
         "--entrypoint /bin/bash ${dockerImageLink} " +
         "-c \"${full_cmd}\" > ${results}/${output_file}")
     addFiles(master, target, results, output_dir)
diff --git a/src/com/mirantis/mk/Aptly.groovy b/src/com/mirantis/mk/Aptly.groovy
index c5aaca0..4771084 100644
--- a/src/com/mirantis/mk/Aptly.groovy
+++ b/src/com/mirantis/mk/Aptly.groovy
@@ -60,8 +60,14 @@
     sh("curl -f -X POST -H 'Content-Type: application/json' --data '{\"Name\":\"$snapshot\"}' ${server}/api/repos/${repo}/snapshots")
 }
 
-def cleanupSnapshots(server, config='/etc/aptly-publisher.yaml', opts='-d --timeout 600') {
-    sh("aptly-publisher -c ${config} ${opts} --url ${server} cleanup")
+/**
+ * Cleanup snapshots
+ *
+ * @param server        Server host
+ * @param opts          Options: debug, timeout, ...
+ */
+def cleanupSnapshots(server, opts='-d'){
+    sh("aptly-publisher --url ${server} ${opts} cleanup")
 }
 
 def diffPublish(server, source, target, components=null, opts='--timeout 600') {
@@ -135,7 +141,11 @@
  */
 def dumpPublishes(server, prefix, publishes='all', opts='-d --timeout 600') {
     sh("aptly-publisher dump --url ${server} --save-dir . --prefix ${prefix} -p '${publishes}' ${opts}")
-    archiveArtifacts artifacts: "${prefix}*"
+    if (findFiles(glob: "${prefix}*")) {
+       archiveArtifacts artifacts: "${prefix}*"
+    } else {
+       throw new Exception("Aptly dump publishes for a prefix ${prefix}* failed. No dump files found!")
+    }
 }
 
 /**
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 61e81d3..664871f 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -425,9 +425,25 @@
 def installOpenstackCompute(master) {
     def salt = new com.mirantis.mk.Salt()
     // Configure compute nodes
-    if (salt.testTarget(master, 'I@nova:compute')) {
+    def compute_compound = 'I@nova:compute'
+    if (salt.testTarget(master, compute_compound)) {
+        // In case if infrastructure nodes are used as nova computes too
+        def gluster_compound = 'I@glusterfs:server'
+        // Enforce highstate asynchronous only on compute nodes which are not glusterfs servers
         retry(2) {
-            salt.enforceHighstateWithExclude(master, 'I@nova:compute', 'opencontrail.client')
+            salt.enforceHighstateWithExclude(master, compute_compound + ' and not ' + gluster_compound, 'opencontrail.client')
+        }
+        // Iterate through glusterfs servers and check if they have compute role
+        // TODO: switch to batch once salt 2017.7+ would be used
+        for ( target in salt.getMinionsSorted(master, gluster_compound) ) {
+            for ( cmp_target in salt.getMinionsSorted(master, compute_compound) ) {
+                if ( target == cmp_target ) {
+                    // Enforce highstate one by one on glusterfs servers which are compute nodes
+                    retry(2) {
+                        salt.enforceHighstateWithExclude(master, target, 'opencontrail.client')
+                    }
+                }
+            }
         }
     }
 }
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 4f53de1..77959db 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -7,6 +7,7 @@
  * @param clusterName         model cluster name
  * @param extraFormulas       extraFormulas to install
  * @param formulasSource      formulas source (git or pkg)
+ * @param reclassVersion      Version of used reclass (branch, tag, ...) (optional, default master)
  * @param testDir             directory of model
  * @param formulasSource      Salt formulas source type (optional, default pkg)
  * @param formulasRevision    APT revision for formulas (optional default stable)
@@ -17,21 +18,14 @@
  * @param aptRepoGPG          GPG key for apt repository with formulas
  */
 
-def setupAndTestNode(masterName, clusterName, extraFormulas, testDir, formulasSource = 'pkg', formulasRevision = 'stable', dockerMaxCpus = 0, ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl='', aptRepoGPG='') {
+def setupAndTestNode(masterName, clusterName, extraFormulas, testDir, formulasSource = 'pkg', formulasRevision = 'stable', reclassVersion = "master", dockerMaxCpus = 0, ignoreClassNotfound = false, legacyTestingMode = false, aptRepoUrl='', aptRepoGPG='') {
   // timeout for test execution (40min)
   def testTimeout = 40 * 60
   def saltOpts = "--retcode-passthrough --force-color"
   def common = new com.mirantis.mk.Common()
   def workspace = common.getWorkspace()
-  def imageFound = true
-  def img
-  try {
-    img = docker.image("tcpcloud/salt-models-testing:${formulasRevision}")
-    img.pull()
-  } catch (Throwable e) {
-    img = docker.image("ubuntu:latest")
-    imageFound = false
-  }
+  def img = docker.image("tcpcloud/salt-models-testing:latest")
+  img.pull()
 
   if (!extraFormulas || extraFormulas == "") {
     extraFormulas = "linux"
@@ -43,41 +37,20 @@
   }
 
   img.inside("-u root:root --hostname=${masterName} --ulimit nofile=4096:8192 ${dockerMaxCpusOption}") {
-    if (!imageFound) {
-      sh("""apt-get update && apt-get install -y curl git python-pip sudo python-pip python-dev zlib1g-dev git
-            pip install git+https://github.com/salt-formulas/reclass.git --upgrade
-            mkdir -p /srv/salt/scripts/ || true
-            cp -r ${testDir} /srv/salt/reclass
-            git config --global user.email || git config --global user.email 'ci@ci.local'
-            git config --global user.name || git config --global user.name 'CI'
-            git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts""")
-    }
-
     withEnv(["FORMULAS_SOURCE=${formulasSource}", "EXTRA_FORMULAS=${extraFormulas}", "DISTRIB_REVISION=${formulasRevision}",
             "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
-            "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "APT_REPOSITORY=${aptRepoUrl}",
+            "RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "APT_REPOSITORY=${aptRepoUrl}",
             "APT_REPOSITORY_GPG=${aptRepoGPG}"]){
 
-        if (!imageFound) {
-          sh("""cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
-                cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
-                bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && source_local_envs && system_config_master'
-                bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && source_local_envs && saltmaster_bootstrap'""")
-          sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
-                    sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
-                      -t \"\$s\" git+https://github.com/salt-formulas/reclass.git;
-                  done""")
-        } else {
-          sh("""cp -r ${testDir}/* /srv/salt/reclass && echo '127.0.1.2  salt' >> /etc/hosts
-                cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
-                cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;""")
-          sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
-                    sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
-                      -t \"\$s\" git+https://github.com/salt-formulas/reclass.git;
-                  done""")
-          sh("""timeout ${testTimeout} bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && source_local_envs && configure_salt_master && configure_salt_minion && install_salt_formula_pkg'
-                bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && saltservice_restart'""")
-        }
+        sh("""cp -r ${testDir}/* /srv/salt/reclass && echo '127.0.1.2  salt' >> /etc/hosts
+              cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
+              cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;""")
+        sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
+                  sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
+                    -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${reclassVersion};
+                done""")
+        sh("""timeout ${testTimeout} bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && source_local_envs && configure_salt_master && configure_salt_minion && install_salt_formula_pkg'
+              bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && saltservice_restart'""")
 
         sh("timeout ${testTimeout} bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && source_local_envs && saltmaster_init'")