Merge "Centralize redundant install infra call"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 1e03773..a541fe0 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -224,6 +224,17 @@
                             envParams.put('cfg_saltversion', SALT_VERSION)
                         }
 
+                        // If stack wasn't removed by the same user which has created it,
+                        // nova key pair won't be removed, so need to make sure that no
+                        // key pair with the same name exists before creating the stack.
+                        if (openstack.getKeyPair(openstackCloud, STACK_NAME, venv)){
+                            try {
+                                openstack.deleteKeyPair(openstackCloud, STACK_NAME, venv)
+                            } catch (Exception e) {
+                                common.errorMsg("Key pair failed to remove with error ${e.message}")
+                            }
+                        }
+
                         openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, venv)
                     }
 
diff --git a/cvp-func.groovy b/cvp-func.groovy
index 94f3eeb..d1fff1a 100644
--- a/cvp-func.groovy
+++ b/cvp-func.groovy
@@ -35,7 +35,7 @@
             sh "rm -rf ${artifacts_dir}"
             salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
             salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
-            validate.configureContainer(saltMaster, TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO, TEMPEST_ENDPOINT_TYPE, TEMPEST_VERSION)
+            validate.configureContainer(saltMaster, TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO, TEMPEST_ENDPOINT_TYPE)
         }
 
         stage('Run Tempest tests') {
diff --git a/docker-mirror-images.groovy b/docker-mirror-images.groovy
index 08ac439..ebbfc86 100644
--- a/docker-mirror-images.groovy
+++ b/docker-mirror-images.groovy
@@ -7,6 +7,7 @@
  *   TARGET_REGISTRY                           Target Docker Registry name
  *   REGISTRY_URL                              Target Docker Registry URL
  *   IMAGE_TAG                                 Tag to use when pushing images
+ *   SOURCE_IMAGE_TAG                          Tag to use when pulling images(optional,if SOURCE_IMAGE_TAG has been found)
  *   IMAGE_LIST                                List of images to mirror
  *
  */
@@ -39,6 +40,10 @@
                     }
                     imageArray = image.trim().tokenize(' ')
                     imagePath = imageArray[0]
+                    if (imagePath.contains('SUBS_SOURCE_IMAGE_TAG')) {
+                        common.warningMsg("Replacing SUBS_SOURCE_IMAGE_TAG => ${SOURCE_IMAGE_TAG}")
+                        imagePath.replace('SUBS_SOURCE_IMAGE_TAG', SOURCE_IMAGE_TAG)
+                    }
                     targetRegistry = imageArray[1]
                     imageName = getImageName(imagePath)
                     sh """docker pull ${imagePath}
@@ -52,4 +57,4 @@
             throw e
         }
     }
-}
\ No newline at end of file
+}
diff --git a/openstack-compute-install.groovy b/openstack-compute-install.groovy
index 7602dcf..2b37fba 100644
--- a/openstack-compute-install.groovy
+++ b/openstack-compute-install.groovy
@@ -80,7 +80,9 @@
 
             stage("Highstate compute") {
                 // Execute highstate without state opencontrail.client.
-                salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.highstate', ['exclude=opencontrail.client'], null, true)
+                common.retry(2){
+                    salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.highstate', ['exclude=opencontrail.client'], null, true)
+                }
 
                 // Apply nova state to remove libvirt default bridge virbr0.
                 salt.enforceState(pepperEnv, targetLiveAll, 'nova', true)
diff --git a/release-mcp-version.groovy b/release-mcp-version.groovy
index 8af3fbe..b1b3d77 100644
--- a/release-mcp-version.groovy
+++ b/release-mcp-version.groovy
@@ -20,7 +20,7 @@
  *   NOTIFY_RECIPIENTS
  *   NOTIFY_TEXT
  *
- */
+*/
 
 common = new com.mirantis.mk.Common()
 git = new com.mirantis.mk.Git()
@@ -35,71 +35,73 @@
     [$class: 'BooleanParameterValue', name: 'RECREATE', value: recreate],
     [$class: 'StringParameterValue', name: 'SOURCE', value: source],
     [$class: 'StringParameterValue', name: 'STORAGES', value: storages],
-    [$class: 'StringParameterValue', name: 'TARGET', value: target]
+    [$class: 'StringParameterValue', name: 'TARGET', value: target],
   ]
 }
 
-def triggerDockerMirrorJob(dockerCredentials, dockerRegistryUrl, targetTag, imageList) {
+def triggerDockerMirrorJob(dockerCredentials, dockerRegistryUrl, targetTag, imageList, sourceImageTag) {
   build job: "docker-images-mirror", parameters: [
     [$class: 'StringParameterValue', name: 'TARGET_REGISTRY_CREDENTIALS_ID', value: dockerCredentials],
     [$class: 'StringParameterValue', name: 'REGISTRY_URL', value: dockerRegistryUrl],
     [$class: 'StringParameterValue', name: 'IMAGE_TAG', value: targetTag],
-    [$class: 'StringParameterValue', name: 'IMAGE_LIST', value: imageList]
+    [$class: 'StringParameterValue', name: 'IMAGE_LIST', value: imageList],
+    [$class: 'StringParameterValue', name: 'SOURCE_IMAGE_TAG', value: sourceImageTag],
   ]
 }
 
 def triggerMirrorRepoJob(snapshotId, snapshotName) {
   build job: "mirror-snapshot-name-all", parameters: [
     [$class: 'StringParameterValue', name: 'SNAPSHOT_NAME', value: snapshotName],
-    [$class: 'StringParameterValue', name: 'SNAPSHOT_ID', value: snapshotId]
+    [$class: 'StringParameterValue', name: 'SNAPSHOT_ID', value: snapshotId],
   ]
 }
 
-def triggerGitTagJob(gitRepoList, gitCredentials, tag) {
+def triggerGitTagJob(gitRepoList, gitCredentials, tag, sourceTag) {
   build job: "tag-git-repos-stable", parameters: [
     [$class: 'StringParameterValue', name: 'GIT_REPO_LIST', value: gitRepoList],
     [$class: 'StringParameterValue', name: 'GIT_CREDENTIALS', value: gitCredentials],
-    [$class: 'StringParameterValue', name: 'TAG', value: tag]
+    [$class: 'StringParameterValue', name: 'TAG', value: tag],
+    [$class: 'StringParameterValue', name: 'SOURCE_TAG', value: sourceTag],
   ]
 }
 
 timeout(time: 12, unit: 'HOURS') {
-    node() {
-        try {
-            stage("Promote"){
-                if(RELEASE_APTLY.toBoolean())
-                {
-                    common.infoMsg("Promoting Aptly")
-                    triggerAptlyPromoteJob(APTLY_URL, 'all', false, true, 'all', false, "(.*)/${SOURCE_REVISION}", APTLY_STORAGES, "{0}/${TARGET_REVISION}")
-                }
+  node() {
+    try {
+      stage("Promote"){
+        if(RELEASE_APTLY.toBoolean())
+        {
+          common.infoMsg("Promoting Aptly")
+          triggerAptlyPromoteJob(APTLY_URL, 'all', false, true, 'all', false, "(.*)/${SOURCE_REVISION}", APTLY_STORAGES, "{0}/${TARGET_REVISION}")
+        }
 
-                if(RELEASE_DEB_MIRRORS.toBoolean()){
-                    common.infoMsg("Promoting Debmirrors")
-                    triggerMirrorRepoJob(SOURCE_REVISION, TARGET_REVISION)
-                }
+        if(RELEASE_DEB_MIRRORS.toBoolean()){
+          common.infoMsg("Promoting Debmirrors")
+          triggerMirrorRepoJob(SOURCE_REVISION, TARGET_REVISION)
+        }
 
-                if(RELEASE_DOCKER.toBoolean())
-                {
-                    common.infoMsg("Promoting Docker images")
-                    triggerDockerMirrorJob(DOCKER_CREDENTIALS, DOCKER_URL, TARGET_REVISION, DOCKER_IMAGES)
-                }
+        if(RELEASE_DOCKER.toBoolean())
+        {
+          common.infoMsg("Promoting Docker images")
+          triggerDockerMirrorJob(DOCKER_CREDENTIALS, DOCKER_URL, TARGET_REVISION, DOCKER_IMAGES, SOURCE_REVISION)
+        }
 
-                if(RELEASE_GIT.toBoolean())
-                {
-                    common.infoMsg("Promoting Git repositories")
-                    triggerGitTagJob(GIT_REPO_LIST, GIT_CREDENTIALS, TARGET_REVISION)
+        if(RELEASE_GIT.toBoolean())
+        {
+          common.infoMsg("Promoting Git repositories")
+          triggerGitTagJob(GIT_REPO_LIST, GIT_CREDENTIALS, TARGET_REVISION, SOURCE_REVISION)
 
-                }
-                if (EMAIL_NOTIFY.toBoolean()) {
-                    emailext(to: NOTIFY_RECIPIENTS,
-                        body: NOTIFY_TEXT,
-                        subject: "MCP Promotion has been done")
-                }
-            }
-        } catch (Throwable e) {
+        }
+        if (EMAIL_NOTIFY.toBoolean()) {
+          emailext(to: NOTIFY_RECIPIENTS,
+            body: NOTIFY_TEXT,
+            subject: "MCP Promotion has been done")
+        }
+      }
+      } catch (Throwable e) {
             // If there was an error or exception thrown, the build failed
             currentBuild.result = "FAILURE"
             throw e
+          }
         }
-    }
-}
\ No newline at end of file
+      }
diff --git a/tag-git-repos.groovy b/tag-git-repos.groovy
index 52344d1..dabbb7f 100644
--- a/tag-git-repos.groovy
+++ b/tag-git-repos.groovy
@@ -1,46 +1,58 @@
+
 /**
- *
- * Tag Git repositories
- *
- * Expected parameters:
- *   GIT_REPO_LIST
- *   GIT_CREDENTIALS
- *   TAG
- *
- */
+*
+* Tag Git repositories
+*
+* Expected parameters:
+*   GIT_REPO_LIST
+*   GIT_CREDENTIALS
+*   TAG
+*   SOURCE_TAG initial commit\tag to be tagged with TAG
+*
+*/
 
 common = new com.mirantis.mk.Common()
 git = new com.mirantis.mk.Git()
 
 def gitRepoAddTag(repoURL, repoName, tag, credentials, ref = "HEAD"){
-    git.checkoutGitRepository(repoName, repoURL, "master", credentials)
-    dir(repoName) {
-        sh "git tag -f -a ${tag} ${ref} -m \"Release of mcp version ${tag}\""
-        sshagent([credentials]) {
-            sh "git push -f origin ${tag}:refs/tags/${tag}"
-        }
+  common.infoMsg("Tagging: ${repoURL} ${ref} => ${tag}")
+  git.checkoutGitRepository(repoName, repoURL, "master", credentials)
+  dir(repoName) {
+    sh "git tag -f -a ${tag} ${ref} -m \"Release of mcp version ${tag}\""
+    sshagent([credentials]) {
+      sh "git push -f origin ${tag}:refs/tags/${tag}"
     }
+  }
 }
 
 timeout(time: 12, unit: 'HOURS') {
-    node() {
-        try {
-            def repos = GIT_REPO_LIST.tokenize('\n')
-            def repoUrl, repoName, repoCommit, repoArray
-            for (repo in repos){
-                if(repo.trim().indexOf(' ') == -1){
-                    throw new IllegalArgumentException("Wrong format of repository and commit input")
-                }
-                repoArray = repo.trim().tokenize(' ')
-                repoName = repoArray[0]
-                repoUrl = repoArray[1]
-                repoCommit = repoArray[2]
-                gitRepoAddTag(repoUrl, repoName, TAG, GIT_CREDENTIALS, repoCommit)
-            }
-        } catch (Throwable e) {
+  node() {
+    try {
+      def repos = GIT_REPO_LIST.tokenize('\n')
+      def repoUrl, repoName, repoCommit, repoArray
+      for (repo in repos){
+        if(repo.startsWith('#')){
+          common.warningMsg("Skipping repo ${repo}")
+          continue
+        }
+        if(repo.trim().indexOf(' ') == -1){
+          throw new IllegalArgumentException("Wrong format of repository and commit input")
+        }
+        repoArray = repo.trim().tokenize(' ')
+        repoName = repoArray[0]
+        repoUrl = repoArray[1]
+        repoCommit = repoArray[2]
+        if (repoCommit.contains('SUBS_SOURCE_REF')) {
+          common.warningMsg("Replacing SUBS_SOURCE_REF => ${SOURCE_TAG}")
+          repoCommit.replace('SUBS_SOURCE_REF', SOURCE_TAG
+            )
+        }
+        gitRepoAddTag(repoUrl, repoName, TAG, GIT_CREDENTIALS, repoCommit)
+      }
+      } catch (Throwable e) {
             // If there was an error or exception thrown, the build failed
             currentBuild.result = "FAILURE"
             throw e
+          }
         }
-    }
-}
\ No newline at end of file
+      }
diff --git a/test-drivetrain.groovy b/test-drivetrain.groovy
index 21d1a8a..fe7c87c 100644
--- a/test-drivetrain.groovy
+++ b/test-drivetrain.groovy
@@ -8,6 +8,7 @@
  *   TARGET_MCP_VERSION                            MCP version to upgrade to
  *   FUNC_TEST_SETTINGS                            Settings for functional tests
  *   ENVIRONMENT_IP                                IP of already deployed environment
+ *   DELETE_STACK                                  Option to delete Heat Stack
  */
 
 
@@ -34,10 +35,21 @@
 }
 
 def runJobOnJenkins(jenkinsUrl, userName, password, jobName, parameters){
+    def status = "null"
     def jenkinsDownCmd = "curl -OL ${jenkinsUrl}/jnlpJars/jenkins-cli.jar --output ./jenkins-cli.jar"
-    def runJobFromSaltMasterCmd = "java -jar jenkins-cli.jar -s ${jenkinsUrl} -noKeyAuth -auth admin:${password} build ${jobName} ${parameters} -s | grep -E 'SUCCESS|UNSTABLE'"
+    def runJobFromSaltMasterCmd = "java -jar jenkins-cli.jar -s ${jenkinsUrl} -noKeyAuth -auth ${userName}:${password} build ${jobName} ${parameters} -w"
+    def waitJobFromSaltMasterCmd = "curl -s -X GET '${jenkinsUrl}/job/${jobName}/lastBuild/api/json?tree=result' --user ${userName}:${password} | jq -r '.result'"
     salt.cmdRun(pepperEnv, "I@salt:master", jenkinsDownCmd)
     salt.cmdRun(pepperEnv, "I@salt:master", runJobFromSaltMasterCmd)
+    while (status == "null" || status.contains("parse error")){
+        status = salt.cmdRun(pepperEnv, "I@salt:master", waitJobFromSaltMasterCmd, false)
+        status = status.get("return")[0].values()[0].trim()
+        println("The job ${jobName} result is $status")
+        if(status == "FAILURE"){
+            throw new Exception("The job ${jobName} result is FAILURE.")
+        }
+        sleep(10)
+    }
 }
 
 timeout(time: 12, unit: 'HOURS') {
@@ -48,8 +60,8 @@
             def saltCreds = [:]
             def mcpEnvJobIP
 
-            if(ENVIRONMENT_IP == ""){
-                stage('Trigger deploy job') {
+            stage('Trigger deploy job') {
+                if(ENVIRONMENT_IP == ""){
                     mcpEnvJob = build(job: "create-mcp-env", parameters: [
                         [$class: 'StringParameterValue', name: 'OS_AZ', value: 'mcp-mk'],
                         [$class: 'StringParameterValue', name: 'OS_PROJECT_NAME', value: 'mcp-mk'],
@@ -59,12 +71,11 @@
                         [$class: 'BooleanParameterValue', name: 'RUN_TESTS', value: false],
                         [$class: 'TextParameterValue', name: 'COOKIECUTTER_TEMPLATE_CONTEXT', value: COOKIECUTTER_TEMPLATE_CONTEXT]
                     ])
+                    def mcpEnvJobDesc = mcpEnvJob.getDescription().tokenize(" ")
+                    mcpEnvJobIP = mcpEnvJobDesc[2]
+                }else{
+                    mcpEnvJobIP = ENVIRONMENT_IP
                 }
-
-                def mcpEnvJobDesc = mcpEnvJob.getDescription().tokenize(" ")
-                mcpEnvJobIP = mcpEnvJobDesc[2]
-            }else{
-                mcpEnvJobIP = ENVIRONMENT_IP
             }
 
             def saltMasterUrl = "http://${mcpEnvJobIP}:6969"
@@ -80,8 +91,16 @@
             def stackCicdAddr = saltReturn.get("return")[0].values()[0]
             def jenkinsUrl = "http://${stackCicdAddr}:8081"
 
+            salt.cmdRun(pepperEnv, "I@salt:master", 'cd /srv/salt/reclass && echo -e ".gitignore\\nclasses/service/\\nnodes/_generated/" >> .gitignore')
+            salt.cmdRun(pepperEnv, "I@salt:master", "cd /srv/salt/reclass && git reset --hard")
+            salt.cmdRun(pepperEnv, "I@salt:master", "cd /srv/salt/reclass/classes/system && git reset --hard && git clean -fd")
+
+            //TODO: Temporary fix. Remove the line below after 2a3757a (reclass-system) is in stable tag.
+            salt.cmdRun(pepperEnv, "cid*", "mkdir /etc/aptly", false)
+
             stage('Run CVP before upgrade') {
                 runJobOnJenkins(jenkinsUrl, "admin", stackCicdPassword, "cvp-sanity", "-p TESTS_SET=cvp-sanity-checks/cvp_checks/tests/test_drivetrain.py -p TESTS_SETTINGS='drivetrain_version=\"${SOURCE_MCP_VERSION}\"'")
+                //TODO: Enable functional tests after they become implemented.
                 //runJobOnJenkins(jenkinsUrl, "admin", stackCicdPassword, "cvp-dt-func", "-p SETTINGS=${FUNC_TEST_SETTINGS}")
             }
 
@@ -91,12 +110,20 @@
 
             stage('Run CVP after upgrade') {
                 runJobOnJenkins(jenkinsUrl, "admin", stackCicdPassword, "cvp-sanity", "-p TESTS_SET=cvp-sanity-checks/cvp_checks/tests/test_drivetrain.py -p TESTS_SETTINGS='drivetrain_version=\"${TARGET_MCP_VERSION}\"'")
+                //TODO: Enable functional tests after they become implemented.
                 //runJobOnJenkins(jenkinsUrl, "admin", stackCicdPassword, "cvp-dt-func", "-p SETTINGS=${FUNC_TEST_SETTINGS}")
             }
 
         } catch (Throwable e) {
             currentBuild.result = 'FAILURE'
             throw e
+        } finally{
+            if(DELETE_STACK.toBoolean() && ENVIRONMENT_IP == ""){
+                mcpEnvJob = build(job: "delete-heat-stack-for-mcp-env", parameters: [
+                    [$class: 'StringParameterValue', name: 'OS_PROJECT_NAME', value: 'mcp-mk'],
+                    [$class: 'StringParameterValue', name: 'STACK_NAME', value: 'jenkins-drivetrain-test-' + currentBuild.number],
+                ])
+            }
         }
     }
 }
\ No newline at end of file
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 08796c9..62e5622 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -67,12 +67,16 @@
 timeout(time: 12, unit: 'HOURS') {
     node("python") {
         try {
+            def gitMcpVersion = MCP_VERSION
             workspace = common.getWorkspace()
             python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
 
             if(MCP_VERSION == ""){
                 error("You must specify MCP version")
             }
+            if(MCP_VERSION == "testing"){
+                gitMcpVersion = "master"
+            }
 
             stage("Update Reclass"){
                 def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
@@ -92,7 +96,7 @@
                 catch(Exception ex){
                     error("You have unstaged changes in your Reclass system model repository. Please reset them and rerun the pipeline.")
                 }
-                salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout $MCP_VERSION")
+                salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout $gitMcpVersion")
             }
 
             if(UPDATE_LOCAL_REPOS.toBoolean()){
diff --git a/validate-cloud.groovy b/validate-cloud.groovy
index 503b375..7d3b2e2 100644
--- a/validate-cloud.groovy
+++ b/validate-cloud.groovy
@@ -49,7 +49,7 @@
 
 def pepperEnv = "pepperEnv"
 def artifacts_dir = 'validation_artifacts/'
-if (!env.JOB_TIMEOUT){
+if (env.JOB_TIMEOUT == ''){
     job_timeout = 12
 } else {
     job_timeout = env.JOB_TIMEOUT.toInteger()