Merge "Allow upgrade etcd for k8s." into release/2019.2.0
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 3084f4b..6b6ec4e 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -433,6 +433,7 @@
                     // Setup kubernetes addons for opencontrail. More info in the definition of the func.
                     orchestrate.setupKubeAddonForContrail(venvPepper, extra_tgt)
                 }
+                orchestrate.installKubernetesClient(venvPepper, extra_tgt)
             }
 
             // install ceph
diff --git a/cvp-func.groovy b/cvp-func.groovy
index 0c657a5..4a231dc 100644
--- a/cvp-func.groovy
+++ b/cvp-func.groovy
@@ -31,6 +31,10 @@
     try{
         stage('Initialization') {
             sh "rm -rf ${artifacts_dir}"
+            if (!TARGET_NODE) {
+              // This pillar will return us cid01
+              TARGET_NODE = "I@gerrit:client"
+            }
             saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
             salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
             salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
diff --git a/cvp-ha.groovy b/cvp-ha.groovy
index b33cda6..e96a34c 100644
--- a/cvp-ha.groovy
+++ b/cvp-ha.groovy
@@ -37,6 +37,10 @@
         try {
             stage('Initialization') {
                 sh "rm -rf ${artifacts_dir}"
+                if (!TEMPEST_TARGET_NODE) {
+                  // This pillar will return us cid01
+                  TEMPEST_TARGET_NODE = "I@gerrit:client"
+                }
                 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
                 salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
                 salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
diff --git a/cvp-perf.groovy b/cvp-perf.groovy
index 74c9a63..62f5226 100644
--- a/cvp-perf.groovy
+++ b/cvp-perf.groovy
@@ -27,6 +27,10 @@
     try{
         stage('Initialization') {
             sh "rm -rf ${artifacts_dir}"
+            if (!TARGET_NODE) {
+              // This pillar will return us cid01
+              TARGET_NODE = "I@gerrit:client"
+            }
             saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
             salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
             salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
diff --git a/galera-cluster-verify-restore.groovy b/galera-cluster-verify-restore.groovy
index 2a575cc..7a63858 100644
--- a/galera-cluster-verify-restore.groovy
+++ b/galera-cluster-verify-restore.groovy
@@ -4,6 +4,8 @@
  * Expected parameters:
  *   SALT_MASTER_CREDENTIALS    Credentials to the Salt API.
  *   SALT_MASTER_URL            Full Salt API address [http://10.10.10.1:8000].
+ *   ASK_CONFIRMATION           Ask confirmation for restore
+ *   VERIFICATION_RETRIES       Number of restries to verify the restoration.
  *
 **/
 
@@ -15,6 +17,13 @@
 def pepperEnv = "pepperEnv"
 def resultCode = 99
 
+askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
+if (common.validInputParam(VERIFICATION_RETRIES) && VERIFICATION_RETRIES.isInteger()) {
+    verificationRetries = VERIFICATION_RETRIES.toInteger()
+} else {
+    verificationRetries = 5
+}
+
 timeout(time: 12, unit: 'HOURS') {
     node() {
         stage('Setup virtualenv for Pepper') {
@@ -29,9 +38,11 @@
                 if (resultCode == 129) {
                     common.errorMsg("Unable to obtain Galera slave minions list". "Without fixing this issue, pipeline cannot continue in verification and restoration.")
                     currentBuild.result = "FAILURE"
+                    return
                 } else if (resultCode == 130) {
                     common.errorMsg("Neither master or slaves are reachable. Without fixing this issue, pipeline cannot continue in verification and restoration.")
                     currentBuild.result = "FAILURE"
+                    return
                 }
             }
             if (resultCode == 1) {
@@ -49,13 +60,14 @@
             }
         }
         stage('Verify restoration result') {
-            exitCode = openstack.verifyGaleraStatus(pepperEnv, false)
-            if (exitCode >= 1) {
-                common.errorMsg("Restoration procedure was probably not successful. See verification report for more information.")
-                currentBuild.result = "FAILURE"
-            } else {
-                common.infoMsg("Restoration procedure seems to be successful. See verification report to be sure.")
-                currentBuild.result = "SUCCESS"
+            common.retry(verificationRetries, 15) {
+                exitCode = openstack.verifyGaleraStatus(pepperEnv, false)
+                if (exitCode >= 1) {
+                    error("Verification attempt finished with an error. This may be caused by cluster not having enough time to come up or to sync. Next verification attempt in 5 seconds.")
+                } else {
+                    common.infoMsg("Restoration procedure seems to be successful. See verification report to be sure.")
+                    currentBuild.result = "SUCCESS"
+                }
             }
         }
     }
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index b229134..49f92c3 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -186,9 +186,11 @@
                     salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
                         "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.salt' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.salt/system.linux.system.repo.mcp.apt_mirantis.salt-formulas/g'")
                     salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
-                        "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.contrail' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.salt/system.linux.system.repo.mcp.apt_mirantis.contrail/g'")
+                        "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.contrail' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.contrail/system.linux.system.repo.mcp.apt_mirantis.contrail/g'")
                     salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
-                        "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.salt/system.linux.system.repo.mcp.apt_mirantis.update/g'")
+                        "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.updates/system.linux.system.repo.mcp.apt_mirantis.update/g'")
+                    salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
+                            "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
                     salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout ${reclassSystemBranch}")
                     // Add kubernetes-extra repo
                     if (salt.testTarget(venvPepper, "I@kubernetes:master")) {