Merge "add update image pipeline"
diff --git a/ceph-add-osd-host.groovy b/ceph-add-osd-host.groovy
new file mode 100644
index 0000000..fb0eceb
--- /dev/null
+++ b/ceph-add-osd-host.groovy
@@ -0,0 +1,32 @@
+/**
+ *
+ * Add OSD host to existing cluster
+ *
+ * Requred parameters:
+ *  SALT_MASTER_URL             URL of Salt master
+ *  SALT_MASTER_CREDENTIALS     Credentials to the Salt API
+ *  HOST                        Host (minion id) to be added
+ *
+ */
+
+common = new com.mirantis.mk.Common()
+salt = new com.mirantis.mk.Salt()
+orchestrate = new com.mirantis.mk.Orchestrate()
+def python = new com.mirantis.mk.Python()
+
+def pepperEnv = "pepperEnv"
+
+node("python") {
+
+    // create connection to salt master
+    python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+
+    // run basic states
+    stage('Install infra') {
+        orchestrate.installFoundationInfraOnTarget(pepperEnv, HOST)
+    }
+    // Install Ceph
+    stage('Install Ceph') {
+        orchestrate.installCephOsd(pepperEnv, HOST)
+    }
+}
diff --git a/ceph-enforce-weights.groovy b/ceph-enforce-weights.groovy
deleted file mode 100644
index 45ec06b..0000000
--- a/ceph-enforce-weights.groovy
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- *
- * Enforce OSD weights from model
- *
- * Requred parameters:
- *  SALT_MASTER_URL             URL of Salt master
- *  SALT_MASTER_CREDENTIALS     Credentials to the Salt API
- *
- *  ADMIN_HOST                  Host (minion id) with admin keyring
- *
- */
-
-common = new com.mirantis.mk.Common()
-salt = new com.mirantis.mk.Salt()
-def python = new com.mirantis.mk.Python()
-
-def pepperEnv = "pepperEnv"
-
-def runCephCommand(master, cmd) {
-    return salt.cmdRun(master, ADMIN_HOST, cmd)
-}
-
-def grains
-
-node("python") {
-
-    stage('Load cluster information') {
-        python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
-
-        // get list of disk from grains
-        grains = salt.getGrain(pepperEnv, 'I@ceph:osd')['return'][0]
-        common.prettyPrint(grains)
-
-    }
-
-    stage('Enforce weights on OSDs') {
-
-        for (host in grains) {
-            // parse grains
-            def hostGrains = host.value
-            common.prettyPrint(hostGrains)
-
-            def hostname = hostGrains.host
-            def salt_id = hostGrains.id
-            def ceph_host_id = hostGrains.ceph_osd_host_id
-
-            common.infoMsg("Setting weights on host ${hostname} (${salt_id}), ceph_id ${ceph_host_id}")
-            for (disk in hostGrains.ceph_osd_disk) {
-                def osd_id = ceph_host_id + disk.key
-                print(osd_id)
-                print(disk.value)
-                print(disk.key)
-                def cmd = "ceph osd crush set ${osd_id} ${disk.value.weight} host=${hostname}"
-                print(runCephCommand(pepperEnv, cmd))
-            }
-        }
-
-    }
-}
diff --git a/ceph-remove-osd.groovy b/ceph-remove-osd.groovy
index 63f5713..0471f9f 100644
--- a/ceph-remove-osd.groovy
+++ b/ceph-remove-osd.groovy
@@ -28,7 +28,7 @@
 node("python") {
 
     // create connection to salt master
-    python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
 
     if (flags.size() > 0) {
         stage('Set cluster flags') {
@@ -38,18 +38,17 @@
         }
     }
 
-    // get list of disk at the osd
-    def pillar_disks = salt.getPillar(pepperEnv, HOST, 'ceph:osd:disk')['return'][0].values()[0]
-    def hostname_id = salt.getPillar(pepperEnv, HOST, 'ceph:osd:host_id')['return'][0].values()[0]
     def osd_ids = []
 
-    print("host_id is ${hostname_id}")
     print("osds:")
     print(osds)
 
-    for (i in pillar_disks) {
-        def osd_id = (hostname_id + i.key).toInteger().toString()
-        print("Evaluating ${osd_id}")
+    // get list of osd disks of the host
+    def ceph_disks = salt.getGrain(pepperEnv, HOST, 'ceph')['return'][0].values()[0].values()[0]['ceph_disk']
+    common.prettyPrint(ceph_disks)
+
+    for (i in ceph_disks) {
+        def osd_id = i.getKey().toString()
         if (osd_id in osds || OSD == '*') {
             osd_ids.add('osd.' + osd_id)
             print("Will delete " + osd_id)
@@ -114,5 +113,4 @@
             }
         }
     }
-
 }
diff --git a/change-config.groovy b/change-config.groovy
index 2f67b15..0b4538c 100644
--- a/change-config.groovy
+++ b/change-config.groovy
@@ -36,7 +36,7 @@
         }
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         if (common.validInputParam("PULL_MODEL") && PULL_MODEL.toBoolean() == true) {
diff --git a/cicd-lab-pipeline.groovy b/cicd-lab-pipeline.groovy
index d865b25..9fc42d3 100644
--- a/cicd-lab-pipeline.groovy
+++ b/cicd-lab-pipeline.groovy
@@ -117,7 +117,7 @@
             saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
             currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
             saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}"
-            python.setupPepperVirtualenv(venvPepper, saltMasterUrl, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, saltMasterUrl, SALT_MASTER_CREDENTIALS)
         }
 
         //
diff --git a/deploy-k8s-deployments.groovy b/deploy-k8s-deployments.groovy
index bcfaa0e..5989dea 100644
--- a/deploy-k8s-deployments.groovy
+++ b/deploy-k8s-deployments.groovy
@@ -6,7 +6,7 @@
 targetExpression = TARGET_MINIONS ? TARGET_MINIONS : "E@kvm01.*"
 
 node() {
-    python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
 
     common.infoMsg("Enforcing kubernetes state..")
     stage("Update k8s control") {
diff --git a/docker-cleanup-pipeline.groovy b/docker-cleanup-pipeline.groovy
index abc21d1..677efdf 100644
--- a/docker-cleanup-pipeline.groovy
+++ b/docker-cleanup-pipeline.groovy
@@ -16,7 +16,7 @@
 node{
   def saltMaster;
   stage('Setup virtualenv for Pepper') {
-    python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
   }
   stage("Clean old containers"){
     salt.cmdRun(pepperEnv, 'I@jenkins:slave', """
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 4326e7e..d2d380a 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -138,7 +138,7 @@
             }
         }
 
-        stage("Generate config drive") {
+        stage("Generate config drives") {
             // apt package genisoimage is required for this stage
 
             // download create-config-drive
@@ -163,15 +163,38 @@
                 sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" user_data.sh"
             }
 
-            // create config-drive
+            // create cfg config-drive
             sh "./create-config-drive ${args}"
             sh("mkdir output-${clusterName} && mv ${saltMaster}.${clusterDomain}-config.iso output-${clusterName}/")
-            // save iso to artifacts
+
+            // save cfg iso to artifacts
             archiveArtifacts artifacts: "output-${clusterName}/${saltMaster}.${clusterDomain}-config.iso"
+
+            if (templateContext.default_context.offline_deployment && templateContext.default_context.offline_deployment == 'True'){
+                def aptlyServerHostname = templateContext.default_context.aptly_server_hostname
+                def user_data_script_apt_url = "https://raw.githubusercontent.com/richardfelkl/scripts/master/mirror_config.sh"
+                sh "wget -O mirror_config.sh ${user_data_script_apt_url}"
+
+                def smc_apt = [:]
+                smc_apt['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
+                smc_apt['APTLY_DEPLOY_IP'] = templateContext['default_context']['aptly_server_address']
+                smc_apt['APTLY_DEPLOY_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
+                smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${clusterDomain}"
+
+                for (i in common.entries(smc_apt)) {
+                    sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config.sh"
+                }
+
+                // create apt config-drive
+                sh "./create-config-drive --user-data mirror_config.sh --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${clusterDomain}-config.iso"
+                sh("mv ${aptlyServerHostname}.${clusterDomain}-config.iso output-${clusterName}/")
+
+                // save apt iso to artifacts
+                archiveArtifacts artifacts: "output-${clusterName}/${aptlyServerHostname}.${clusterDomain}-config.iso"
+            }
         }
 
         stage ('Save changes reclass model') {
-
             sh(returnStatus: true, script: "tar -zcf output-${clusterName}/${clusterName}.tar.gz -C ${modelEnv} .")
             archiveArtifacts artifacts: "output-${clusterName}/${clusterName}.tar.gz"
 
diff --git a/ironic-node-provision-pipeline.groovy b/ironic-node-provision-pipeline.groovy
index 402254a..05e5313 100644
--- a/ironic-node-provision-pipeline.groovy
+++ b/ironic-node-provision-pipeline.groovy
@@ -151,7 +151,7 @@
 
         outputs.put('salt_api', SALT_MASTER_URL)
 
-        python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+        python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
 
 
 
diff --git a/mk-k8s-simple-deploy-pipeline.groovy b/mk-k8s-simple-deploy-pipeline.groovy
index aa81f5b..e88d482 100644
--- a/mk-k8s-simple-deploy-pipeline.groovy
+++ b/mk-k8s-simple-deploy-pipeline.groovy
@@ -72,7 +72,7 @@
     stage("Connect to Salt master") {
         saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
         saltMasterUrl = "http://${saltMasterHost}:8088"
-        python.setupPepperVirtualenv(venvPepper, saltMasterUrl, SALT_MASTER_CREDENTIALS)
+        python.setupPepperVirtualenv(pepperEnv, saltMasterUrl, SALT_MASTER_CREDENTIALS)
     }
 
     stage("Install core infra") {
diff --git a/opencontrail-upgrade.groovy b/opencontrail-upgrade.groovy
index 7e8412b..311cfef 100644
--- a/opencontrail-upgrade.groovy
+++ b/opencontrail-upgrade.groovy
@@ -54,7 +54,7 @@
 node() {
 
     stage('Setup virtualenv for Pepper') {
-        python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+        python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
     }
 
     if (STAGE_CONTROLLERS_UPGRADE.toBoolean() == true && !errorOccured) {
diff --git a/openstack-compute-install.groovy b/openstack-compute-install.groovy
index bbf07a4..f3b4ec3 100644
--- a/openstack-compute-install.groovy
+++ b/openstack-compute-install.groovy
@@ -23,7 +23,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('List target servers') {
diff --git a/openstack-compute-upgrade.groovy b/openstack-compute-upgrade.groovy
index 3812871..46634bc 100644
--- a/openstack-compute-upgrade.groovy
+++ b/openstack-compute-upgrade.groovy
@@ -29,7 +29,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('List target servers') {
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index b2bd621..76960e9 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -19,7 +19,7 @@
 node() {
 
     stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
     }
 
     if (STAGE_TEST_UPGRADE.toBoolean() == true) {
diff --git a/ovs-gateway-upgrade.groovy b/ovs-gateway-upgrade.groovy
index 394ca98..1b21618 100644
--- a/ovs-gateway-upgrade.groovy
+++ b/ovs-gateway-upgrade.groovy
@@ -29,7 +29,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('List target servers') {
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index 5fff159..586940b 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -16,7 +16,7 @@
 node() {
 
     stage('Setup virtualenv for Pepper') {
-        python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+        python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
     }
 
     stage('Start restore') {
diff --git a/restore-zookeeper.groovy b/restore-zookeeper.groovy
index 54ae8c6..0f32576 100644
--- a/restore-zookeeper.groovy
+++ b/restore-zookeeper.groovy
@@ -16,7 +16,7 @@
 node() {
 
     stage('Setup virtualenv for Pepper') {
-        python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+        python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
     }
 
     stage('Start restore') {
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index e59f0ce..3a33846 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -41,7 +41,7 @@
 
     common.infoMsg("Generating model from context ${modelFile}")
 
-    def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
+    def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight", "ceph"]
     for (product in productList) {
 
         // get templateOutputDir and productDir
diff --git a/test-run-rally.groovy b/test-run-rally.groovy
index d62d9be..d92d988 100644
--- a/test-run-rally.groovy
+++ b/test-run-rally.groovy
@@ -28,7 +28,7 @@
         // Prepare connection
         //
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         //
diff --git a/test-run-tempest.groovy b/test-run-tempest.groovy
index 9da8b16..0011daa 100644
--- a/test-run-tempest.groovy
+++ b/test-run-tempest.groovy
@@ -25,7 +25,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         //
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index 6d544a5..d5727ff 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -47,6 +47,28 @@
     defaultGitUrl = null
 }
 def checkouted = false
+
+
+
+def triggerTestNodeJob(defaultGitUrl, defaultGitRef, clusterName, testTarget, formulasSource) {
+  build job: "test-salt-model-node", parameters: [
+    [$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: defaultGitUrl],
+    [$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: defaultGitRef],
+    [$class: 'StringParameterValue', name: 'CLUSTER_NAME', value: clusterName],
+    [$class: 'StringParameterValue', name: 'NODE_TARGET', value: testTarget],
+    [$class: 'StringParameterValue', name: 'FORMULAS_SOURCE', value: formulasSource],
+    [$class: 'StringParameterValue', name: 'EXTRA_FORMULAS', value: EXTRA_FORMULAS],
+    [$class: 'StringParameterValue', name: 'FORMULAS_REVISION', value: FORMULAS_REVISION],
+    [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: CREDENTIALS_ID],
+    [$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: SYSTEM_GIT_URL],
+    [$class: 'StringParameterValue', name: 'MAX_CPU_PER_JOB', value: MAX_CPU_PER_JOB],
+    [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: SYSTEM_GIT_REF],
+    [$class: 'BooleanParameterValue', name: 'LEGACY_TEST_MODE', value: LEGACY_TEST_MODE.toBoolean()],
+    [$class: 'BooleanParameterValue', name: 'RECLASS_IGNORE_CLASS_NOTFOUND', value: RECLASS_IGNORE_CLASS_NOTFOUND.toBoolean()]
+  ]
+}
+
+
 node("python") {
   try{
     stage("checkout") {
@@ -114,6 +136,7 @@
         }
 
         def branches = [:]
+        def failedNodes = []
         def acc = 0
 
         for (int i = 0; i < infraYMLs.size(); i++) {
@@ -138,26 +161,54 @@
           }
 
           branches[clusterName] = {
-            build job: "test-salt-model-node", parameters: [
-              [$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: defaultGitUrl],
-              [$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: defaultGitRef],
-              [$class: 'StringParameterValue', name: 'CLUSTER_NAME', value: clusterName],
-              [$class: 'StringParameterValue', name: 'NODE_TARGET', value: testTarget],
-              [$class: 'StringParameterValue', name: 'FORMULAS_SOURCE', value: formulasSource],
-              [$class: 'StringParameterValue', name: 'EXTRA_FORMULAS', value: EXTRA_FORMULAS],
-              [$class: 'StringParameterValue', name: 'FORMULAS_REVISION', value: FORMULAS_REVISION],
-              [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: CREDENTIALS_ID],
-              [$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: SYSTEM_GIT_URL],
-              [$class: 'StringParameterValue', name: 'MAX_CPU_PER_JOB', value: MAX_CPU_PER_JOB],
-              [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: SYSTEM_GIT_REF],
-              [$class: 'BooleanParameterValue', name: 'LEGACY_TEST_MODE', value: LEGACY_TEST_MODE.toBoolean()],
-              [$class: 'BooleanParameterValue', name: 'RECLASS_IGNORE_CLASS_NOTFOUND', value: RECLASS_IGNORE_CLASS_NOTFOUND.toBoolean()]
-            ]}
+            try {
+                triggerTestNodeJob(defaultGitUrl, defaultGitRef, clusterName, testTarget, formulasSource)
+            } catch (Exception e) {
+              failedNodes << [defaultGitUrl, defaultGitRef, clusterName, testTarget, formulasSource]
+              common.warningMsg("Test of ${clusterName} failed :  ${e}")
+            }
+          }
           acc++;
         }
         if (acc != 0) {
           parallel branches
         }
+
+        def nbRetry = 2
+        def maxNbRetry = infraYMLs.size() > 10 ? infraYMLs.size() / 2 : 10
+        for (int i = 0; i < nbRetry && failedNodes && failedNodes.size() <= maxNbRetry; ++i) {
+          branches = [:]
+          acc = 0
+          retryNodes = failedNodes
+          failedNodes = []
+          for (int j = 0; j < retryNodes.size(); j++) {
+            if (acc >= PARALLEL_NODE_GROUP_SIZE.toInteger()) {
+              parallel branches
+              branches = [:]
+              acc = 0
+            }
+
+            def currentNode = retryNodes[j]
+
+            common.infoMsg("Test of ${currentNode[2]} failed, retrigger it to make sure")
+            branches[currentNode[2]] = {
+              try {
+
+                  triggerTestNodeJob(currentNode[0], currentNode[1], currentNode[2], currentNode[3], currentNode[4])
+              } catch (Exception e) {
+                failedNodes << currentNode
+                common.warningMsg("Test of ${currentNode[2]} failed :  ${e}")
+              }
+            }
+            acc++
+          }
+          if (acc != 0) {
+            parallel branches
+          }
+        }
+        if (failedNodes) {
+          currentBuild.result = "FAILURE"
+        }
       }
     }
   } catch (Throwable e) {
diff --git a/test-service.groovy b/test-service.groovy
index a03865a..0b6da19 100644
--- a/test-service.groovy
+++ b/test-service.groovy
@@ -28,7 +28,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         //
diff --git a/update-jenkins-master-jobs.groovy b/update-jenkins-master-jobs.groovy
index 65a16ca..1361632 100644
--- a/update-jenkins-master-jobs.groovy
+++ b/update-jenkins-master-jobs.groovy
@@ -20,7 +20,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('Update Jenkins jobs') {
diff --git a/update-package.groovy b/update-package.groovy
index 552a361..c71f598 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -30,7 +30,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('List target servers') {
diff --git a/update-reclass-metadata.groovy b/update-reclass-metadata.groovy
index 80a71ec..be695ca 100644
--- a/update-reclass-metadata.groovy
+++ b/update-reclass-metadata.groovy
@@ -20,7 +20,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('Update Reclass model') {
diff --git a/update-salt-master-formulas.groovy b/update-salt-master-formulas.groovy
index 0cb995c..9d556f0 100644
--- a/update-salt-master-formulas.groovy
+++ b/update-salt-master-formulas.groovy
@@ -20,7 +20,7 @@
     try {
 
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('Update Salt formulas') {
diff --git a/validate-cloud.groovy b/validate-cloud.groovy
index 9e13f7c..8e15aa0 100644
--- a/validate-cloud.groovy
+++ b/validate-cloud.groovy
@@ -14,11 +14,13 @@
  *   RUN_K8S_TESTS               If not false, run Kubernetes tests
  *   RUN_SPT_TESTS               If not false, run SPT tests
  *   SPT_SSH_USER                The name of the user which should be used for ssh to nodes
- *   SPT_FLOATING_NETWORK        The name of the external(floating) network
  *   SPT_IMAGE                   The name of the image for SPT tests
- *   SPT_USER                    The name of the user for SPT image
+ *   SPT_IMAGE_USER              The name of the user for SPT image
  *   SPT_FLAVOR                  The name of the flavor for SPT image
- *   SPT_AVAILABILITY_ZONE       The name of availability zone
+ *   AVAILABILITY_ZONE           The name of availability zone
+ *   FLOATING_NETWORK            The name of the external(floating) network
+ *   RALLY_IMAGE                 The name of the image for Rally tests
+ *   RALLY_FLAVOR                The name of the flavor for Rally image
  *   TEST_K8S_API_SERVER         Kubernetes API address
  *   TEST_K8S_CONFORMANCE_IMAGE  Path to docker image with conformance e2e tests
  *   TEST_K8S_NODE               Kubernetes node to run tests from
@@ -39,7 +41,7 @@
 node() {
     try{
         stage('Setup virtualenv for Pepper') {
-            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
         }
 
         stage('Configure') {
@@ -48,11 +50,13 @@
                 sh "rm -r ${artifacts_dir}"
             }
             sh "mkdir -p ${artifacts_dir}"
-            def spt_variables = "-e spt_ssh_user=${SPT_SSH_USER} " +
-                    "-e spt_floating_network=${SPT_FLOATING_NETWORK} " +
-                    "-e spt_image=${SPT_IMAGE} -e spt_user=${SPT_USER} " +
-                    "-e spt_flavor=${SPT_FLAVOR} -e spt_availability_zone=${SPT_AVAILABILITY_ZONE} "
-            validate.runContainerConfiguration(pepperEnv, TEST_IMAGE, TARGET_NODE, artifacts_dir, spt_variables)
+            def ext_variables = "-e spt_ssh_user=${SPT_SSH_USER} " +
+                    "-e spt_floating_network=${FLOATING_NETWORK} " +
+                    "-e spt_image=${SPT_IMAGE} -e spt_user=${SPT_IMAGE_USER} " +
+                    "-e spt_flavor=${SPT_FLAVOR} -e spt_availability_zone=${AVAILABILITY_ZONE} " +
+                    "-e floating_network=${FLOATING_NETWORK} -e rally_image=${RALLY_IMAGE} " +
+                    "-e rally_flavor=${RALLY_FLAVOR} -e availability_zone=${AVAILABILITY_ZONE} "
+            validate.runContainerConfiguration(pepperEnv, TEST_IMAGE, TARGET_NODE, artifacts_dir, ext_variables)
         }
 
         stage('Run Tempest tests') {
diff --git a/xtrabackup-restore-mysql-db.groovy b/xtrabackup-restore-mysql-db.groovy
index ba51972..50b2bce 100644
--- a/xtrabackup-restore-mysql-db.groovy
+++ b/xtrabackup-restore-mysql-db.groovy
@@ -16,7 +16,7 @@
 node() {
 
     stage('Setup virtualenv for Pepper') {
-        python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+        python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
     }
 
     stage('Start restore') {