Merge "Add INTERACTIVE flag to upgrade pipelines"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index e4847ee..7f7337a 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -342,9 +342,8 @@
                 }
 
                 if (common.checkContains('STACK_INSTALL', 'contrail')) {
-                    stage('Install Contrail for Kubernetes') {
+                    stage('Install Contrail control') {
                         orchestrate.installContrailNetwork(venvPepper)
-                        orchestrate.installContrailCompute(venvPepper)
                     }
                 }
 
@@ -356,6 +355,12 @@
                     archiveArtifacts(artifacts: 'kubeconfig')
                 }
 
+                if (common.checkContains('STACK_INSTALL', 'contrail')) {
+                    stage('Install Contrail compute') {
+                        orchestrate.installContrailCompute(venvPepper)
+                    }
+                }
+
                 stage('Install Kubernetes computes') {
                     if (common.validInputParam('STACK_COMPUTE_COUNT')) {
                         if (STACK_COMPUTE_COUNT > 0) {
diff --git a/cloud-update.groovy b/cloud-update.groovy
index 8c8b8b6..b6a44b6 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -425,6 +425,22 @@
     }
 }
 
+def periodicCheck(pepperEnv, target, maxRetries=50) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def count = 0
+    while(count < maxRetries) {
+        try {
+            sleep(10)
+            salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+            break
+        } catch (Exception e) {
+            common.warningMsg("${target} not ready yet. Waiting ...")
+        }
+        count++
+    }
+}
+
 def highstate(pepperEnv, target, type) {
     def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
@@ -457,9 +473,17 @@
     // optionally reboot
     if (reboots.contains(type)) {
         stage("Reboot ${target} nodes") {
-            salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
-            sleep(10)
-            salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+            if (type == 'cfg') {
+                try {
+                    salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
+                } catch (Exception e) {
+                    periodicCheck(pepperEnv, target)
+                }
+            } else {
+                salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
+                sleep 10
+                salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+            }
         }
     }
 }
@@ -683,13 +707,26 @@
 def verifyAPIs(pepperEnv, target) {
     def salt = new com.mirantis.mk.Salt()
     def common = new com.mirantis.mk.Common()
-    def out = salt.cmdRun(pepperEnv, target, '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
-    if (out.toString().toLowerCase().contains('error')) {
-        common.errorMsg(out)
-        if (INTERACTIVE.toBoolean()) {
-            input message: "APIs are not working as expected. Please fix it manually."
-        } else {
-            throw new Exception("APIs are not working as expected")
+    def cmds = ["openstack service list",
+                "openstack image list",
+                "openstack flavor list",
+                "openstack compute service list",
+                "openstack server list",
+                "openstack network list",
+                "openstack volume list",
+                "openstack orchestration service list"]
+    def sourcerc = ". /root/keystonercv3;"
+    def cmdOut = ">/dev/null 2>&1;echo \$?"
+    for (c in cmds) {
+        def command = sourcerc + c + cmdOut
+        def out = salt.cmdRun(pepperEnv, target, "${command}")
+        if (!out.toString().toLowerCase().contains('0')) {
+            common.errorMsg(out)
+            if (INTERACTIVE.toBoolean()) {
+                input message: "APIs are not working as expected. Please fix it manually."
+            } else {
+                throw new Exception("APIs are not working as expected")
+            }
         }
     }
 }
@@ -700,7 +737,7 @@
     def out
     while(count < maxRetries) {
         try {
-            out = salt.getReturnValues(salt.cmdRun(pepperEnv, target, 'salt-call mysql.status | grep -A1 wsrep_cluster_size'))
+            out = salt.getReturnValues(salt.cmdRun(pepperEnv, target, 'salt-call -l quiet mysql.status | grep -A1 wsrep_cluster_size'))
         } catch (Exception er) {
             common.infoMsg(er)
         }
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index fb8b0e6..0c5c971 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -6,25 +6,58 @@
  *   SALT_MASTER_URL            Salt API server location
  *   SALT_MASTER_CREDENTIALS    Credentials to the Salt API
  *   MCP_VERSION                Version of MCP to upgrade to
+ *   UPDATE_CLUSTER_MODEL       Update MCP version parameter in cluster model
+ *   UPDATE_PIPELINES           Update pipeline repositories on Gerrit
  *   UPDATE_LOCAL_REPOS         Update local repositories
  */
 
+import hudson.model.*
+
 // Load shared libs
 salt = new com.mirantis.mk.Salt()
 common = new com.mirantis.mk.Common()
 python = new com.mirantis.mk.Python()
+jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
 venvPepper = "venvPepper"
 
+def triggerMirrorJob(jobName){
+    params = jenkinsUtils.getJobParameters(jobName)
+    build job: jobName, parameters: [
+        [$class: 'StringParameterValue', name: 'BRANCHES', value: params.get("BRANCHES")],
+        [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get("CREDENTIALS_ID")],
+        [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get("SOURCE_URL")],
+        [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get("TARGET_URL")]
+    ]
+}
+
 timeout(time: 12, unit: 'HOURS') {
     node("python") {
         try {
             python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
 
+            if(MCP_VERSION == ""){
+                error("You must specify MCP version")
+            }
+
             stage("Update Reclass"){
-                common.infoMsg("Updating reclass model")
-                salt.cmdRun(venvPepper, "I@salt:master", 'cd /srv/salt/reclass && git pull -r && git submodule update', false)
-                salt.cmdRun(venvPepper, 'I@salt:master', 'reclass-salt --top')
-                salt.enforceState(venvPepper, "I@salt:master", 'reclass', true)
+                def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
+                if(UPDATE_CLUSTER_MODEL.toBoolean()){
+                    try{
+                        salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/ && git diff-index --quiet HEAD --")
+                    }
+                    catch(Exception ex){
+                        error("You have uncommited changes in your Reclass cluster model repository. Please commit or reset them and rerun the pipeline.")
+                    }
+                    salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -r --exclude-dir=aptly -l 'apt_mk_version: .*' * | xargs sed -i 's/apt_mk_version: .*/apt_mk_version: \"$MCP_VERSION\"/g'")
+                }
+
+                try{
+                    salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git diff-index --quiet HEAD --")
+                }
+                catch(Exception ex){
+                    error("You have unstaged changes in your Reclass system model repository. Please reset them and rerun the pipeline.")
+                }
+                salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout $MCP_VERSION")
             }
 
             if(UPDATE_LOCAL_REPOS.toBoolean()){
@@ -41,6 +74,8 @@
                         common.infoMsg("Aptly isn't running as Docker container. Going to use aptly user for executing aptly commands")
                     }
 
+                    salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/cicd/aptly && git checkout $MCP_VERSION")
+
                     if(runningOnDocker){
                         salt.cmdRun(venvPepper, 'I@aptly:server', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true)
                     }
@@ -69,17 +104,49 @@
                 }
             }
 
-            stage("Update APT repos"){
-                common.infoMsg("Updating APT repositories")
-                salt.enforceState(venvPepper, "I@linux:system", 'linux.system.repo', true)
-            }
+            stage("Update Drivetrain"){
+                salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_salt.list")
+                salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_salt.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
+                salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades salt-formula-*")
+                salt.fullRefresh(venvPepper, 'I@salt:master')
 
-            stage("Update formulas"){
-                common.infoMsg("Updating salt formulas")
-                salt.cmdRun(venvPepper, "I@salt:master", 'apt-get clean && apt-get update && apt-get install -y salt-formula-*')
+                try{
+                    salt.enforceState(venvPepper, "I@salt:master", 'reclass', true)
+                }
+                catch(Exception ex){
+                    error("Reclass fails rendering. Pay attention to your cluster model.")
+                }
 
-                common.infoMsg("Running salt sync-all")
-                salt.runSaltProcessStep(venvPepper, '*', 'saltutil.sync_all', [], null, true)
+                salt.fullRefresh(venvPepper, '*')
+
+                try{
+                    salt.cmdRun(venvPepper, 'I@salt:master', "reclass-salt --top")
+                }
+                catch(Exception ex){
+                    error("Reclass fails rendering. Pay attention to your cluster model.")
+                }
+
+                if(UPDATE_PIPELINES.toBoolean()){
+                    triggerMirrorJob("git-mirror-downstream-mk-pipelines")
+                    triggerMirrorJob("git-mirror-downstream-pipeline-library")
+                }
+
+                salt.enforceState(venvPepper, "I@jenkins:client", 'jenkins.client', true)
+
+                salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client' state.sls docker.client --async")
+
+                sleep(180)
+
+                common.infoMsg("Checking if Docker containers are up")
+
+                try{
+                    common.retry(10, 30){
+                        salt.cmdRun(venvPepper, 'I@jenkins:client and I@docker:client', "! docker service ls | tail -n +2 | grep -v -E '\\s([0-9])/\\1\\s'")
+                    }
+                }
+                catch(Exception ex){
+                    error("Docker containers for CI/CD services are having troubles with starting.")
+                }
             }
         }
         catch (Throwable e) {
diff --git a/validate-cloud.groovy b/validate-cloud.groovy
index 4a2d893..7d3b2e2 100644
--- a/validate-cloud.groovy
+++ b/validate-cloud.groovy
@@ -15,7 +15,8 @@
  *   TEMPEST_VERSION             Version of Tempest (tag, branch or commit)
  *   RUN_TEMPEST_TESTS           If not false, run Tempest tests
  *   RUN_RALLY_TESTS             If not false, run Rally tests
- *   RUN_K8S_TESTS               If not false, run Kubernetes tests
+ *   K8S_RALLY                   If not false, run Kubernetes Rally tests
+ *   RUN_K8S_TESTS               If not false, run Kubernetes e2e/conformance tests
  *   RUN_SPT_TESTS               If not false, run SPT tests
  *   SPT_SSH_USER                The name of the user which should be used for ssh to nodes
  *   SPT_IMAGE                   The name of the image for SPT tests
@@ -25,6 +26,8 @@
  *   FLOATING_NETWORK            The name of the external(floating) network
  *   RALLY_IMAGE                 The name of the image for Rally tests
  *   RALLY_FLAVOR                The name of the flavor for Rally image
+ *   RALLY_PLUGINS_REPO          Git repository with Rally plugins
+ *   RALLY_PLUGINS_BRANCH        Git branch which will be used during the checkout
  *   RALLY_CONFIG_REPO           Git repository with files for Rally
  *   RALLY_CONFIG_BRANCH         Git branch which will be used during the checkout
  *   RALLY_SCENARIOS             Path to file or directory with rally scenarios
@@ -36,6 +39,7 @@
  *   GENERATE_REPORT             If not false, run report generation command
  *   ACCUMULATE_RESULTS          If true, results from the previous build will be used
  *   JOB_TIMEOUT                 Job timeout in hours
+ *
  */
 
 common = new com.mirantis.mk.Common()
@@ -76,11 +80,20 @@
             stage('Run Rally tests') {
                 if (RUN_RALLY_TESTS.toBoolean() == true) {
                     def report_dir = env.REPORT_DIR ?: '/root/qa_results'
-                    def rally_variables = ["floating_network=${FLOATING_NETWORK}",
-                                           "rally_image=${RALLY_IMAGE}",
-                                           "rally_flavor=${RALLY_FLAVOR}",
-                                           "availability_zone=${AVAILABILITY_ZONE}"]
-                    validate.runRallyTests(pepperEnv, TARGET_NODE, TEST_IMAGE, artifacts_dir, RALLY_CONFIG_REPO, RALLY_CONFIG_BRANCH, RALLY_SCENARIOS, RALLY_TASK_ARGS_FILE, rally_variables, report_dir)
+                    def platform
+                    def rally_variables
+                    if (K8S_RALLY.toBoolean() == false) {
+                      platform = 'openstack'
+                      rally_variables = ["floating_network=${FLOATING_NETWORK}",
+                                         "rally_image=${RALLY_IMAGE}",
+                                         "rally_flavor=${RALLY_FLAVOR}",
+                                         "availability_zone=${AVAILABILITY_ZONE}"]
+                    } else {
+                      platform = 'k8s'
+                      rally_variables = ["plugins_repo":"${RALLY_PLUGINS_REPO}",
+                                         "plugins_branch":"${RALLY_PLUGINS_BRANCH}"]
+                    }
+                    validate.runRallyTests(pepperEnv, TARGET_NODE, TEST_IMAGE, platform, artifacts_dir, RALLY_CONFIG_REPO, RALLY_CONFIG_BRANCH, RALLY_SCENARIOS, RALLY_TASK_ARGS_FILE, rally_variables, report_dir)
                 } else {
                     common.infoMsg("Skipping Rally tests")
                 }
@@ -100,7 +113,7 @@
                 }
             }
 
-            stage('Run k8s bootstrap tests') {
+            stage('Run K8S bootstrap tests') {
                 if (RUN_K8S_TESTS.toBoolean() == true) {
                     def image = 'tomkukral/k8s-scripts'
                     def output_file = 'k8s-bootstrap-tests.txt'
@@ -114,7 +127,7 @@
                 }
             }
 
-            stage('Run k8s conformance e2e tests') {
+            stage('Run K8S conformance e2e tests') {
                 if (RUN_K8S_TESTS.toBoolean() == true) {
                     def image = TEST_K8S_CONFORMANCE_IMAGE
                     def output_file = 'report-k8s-e2e-tests.txt'
diff --git a/xtrabackup-restore-mysql-db.groovy b/xtrabackup-restore-mysql-db.groovy
index 60b8caf..da3d463 100644
--- a/xtrabackup-restore-mysql-db.groovy
+++ b/xtrabackup-restore-mysql-db.groovy
@@ -81,6 +81,12 @@
             }
             sleep(5)
             salt.cmdRun(pepperEnv, 'I@galera:master', "su root -c 'salt-call mysql.status | grep -A1 wsrep_cluster_size'")
+
+            try {
+                salt.runSaltProcessStep(pepperEnv, 'I@galera:master or I@galera:slave', 'file.touch', ["/var/lib/mysql/.galera_bootstrap"], null, true)
+            } catch (Exception er) {
+                common.warningMsg('File is already present')
+            }
         }
     }
 }