Merge "Add mine.update before network.host state."
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 2d75d1e..ff05ce3 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -38,10 +38,11 @@
             export DEBUG="${debug}" &&
             export LD_LIBRARY_PATH=\${LD_LIBRARY_PATH:+"\$LD_LIBRARY_PATH:"}/usr/lib/libeatmydata &&
             export LD_PRELOAD=\${LD_PRELOAD:+"\$LD_PRELOAD "}libeatmydata.so &&
-            [[ -z "${extraRepoUrl}" && "${extraRepoUrl}" != "null" ]] || echo "${extraRepoUrl}" >/etc/apt/sources.list.d/extra.list &&
+            [[ -z "${extraRepoUrl}" && "${extraRepoUrl}" != "null" ]] || echo "${extraRepoUrl}" | tr ";" "\\\\n" >/etc/apt/sources.list.d/extra.list &&
             [[ -z "${extraRepoKeyUrl}" && "${extraRepoKeyUrl}" != "null" ]] || (
                 which curl || (apt-get update && apt-get install -y curl) &&
-                curl --insecure -ss -f "${extraRepoKeyUrl}" | apt-key add -
+                EXTRAKEY=`echo "${extraRepoKeyUrl}" | tr ";" " "` &&
+                for RepoKey in \${EXTRAKEY}; do curl --insecure -ss -f "\${RepoKey}" | apt-key add - ; done
             ) &&
             apt-get update && apt-get install -y build-essential devscripts equivs sudo &&
             groupadd -g ${jenkinsGID} jenkins &&
@@ -53,7 +54,6 @@
             sudo -H -E -u jenkins debuild --preserve-envvar DEBUG --no-lintian -uc -us -b'""")
     }
 
-
 }
 
 /*
diff --git a/src/com/mirantis/mk/Docker.groovy b/src/com/mirantis/mk/Docker.groovy
index 1357dc1..7b57e5b 100644
--- a/src/com/mirantis/mk/Docker.groovy
+++ b/src/com/mirantis/mk/Docker.groovy
@@ -22,6 +22,7 @@
         sh "git checkout -f ${dockerFile}; sed -i -e 's,^FROM.*,FROM ${baseImg},g' ${dockerFile}"
     }
 
+    params << "--no-cache"
     params << "-f ${dockerFile}"
     params << imageDir
 
@@ -51,4 +52,4 @@
     }
 
     return img
-}
\ No newline at end of file
+}
diff --git a/src/com/mirantis/mk/Git.groovy b/src/com/mirantis/mk/Git.groovy
index 65b80ab..9e3c460 100644
--- a/src/com/mirantis/mk/Git.groovy
+++ b/src/com/mirantis/mk/Git.groovy
@@ -56,7 +56,7 @@
 def changeGitBranch(path, branch) {
     dir(path) {
         git_cmd = sh (
-            script: "git checkout -b ${branch}",
+            script: "git checkout ${branch}",
             returnStdout: true
         ).trim()
     }
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index 91dfbfe..85f014f 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -378,16 +378,35 @@
  * @param probe single node on which to list service names
  * @param target all targeted nodes
  * @param services  lists of type of services to be stopped
+ * @param confirm enable/disable manual service stop confirmation
  * @return output of salt commands
  */
-def stopServices(env, probe, target, services=[]) {
+def stopServices(env, probe, target, services=[], confirm=false) {
     def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
     for (s in services) {
         def outputServicesStr = salt.getReturnValues(salt.cmdRun(env, "${probe}*", "service --status-all | grep ${s} | awk \'{print \$4}\'"))
-        def servicesList = outputServicesStr.tokenize("\n")
-        for (name in servicesList) {
-            if (!name.contains('Salt command')) {
-                salt.runSaltProcessStep(env, "${target}*", 'service.stop', ["${name}"])
+        def servicesList = outputServicesStr.tokenize("\n").init()
+        if (confirm) {
+            if (servicesList) {
+                try {
+                    input message: "Click PROCEED to stop ${servicesList}. Otherwise click ABORT to skip stopping them."
+                    for (name in servicesList) {
+                        if (!name.contains('Salt command')) {
+                            salt.runSaltProcessStep(env, "${target}*", 'service.stop', ["${name}"])
+                        }
+                    }
+                } catch (Exception er) {
+                    common.infoMsg("skipping stopping ${servicesList} services")
+                }
+            }
+        } else {
+            if (servicesList) {
+                for (name in servicesList) {
+                    if (!name.contains('Salt command')) {
+                        salt.runSaltProcessStep(env, "${target}*", 'service.stop', ["${name}"])
+                    }
+                }
             }
         }
     }
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 33889df..4fc8854 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -34,7 +34,7 @@
     if (staticMgmtNet) {
         salt.runSaltProcessStep(master, '*', 'cmd.shell', ["salt-call state.sls linux.network; salt-call service.restart salt-minion"], null, true, 60)
     }
-    salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'ntp'])
+    salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'ntp', 'rsyslog'])
     salt.enforceState(master, '*', ['salt.minion'], true, false, null, false, 60, 2)
     sleep(5)
     salt.runSaltProcessStep(master, '*', 'mine.update', [], null, true)
@@ -53,7 +53,7 @@
     salt.enforceState(master, target, ['salt.minion'], true, false, null, false, 60, 2)
     salt.enforceState(master, target, ['salt.minion'])
 
-    salt.enforceState(master, target, ['linux', 'openssh', 'ntp'])
+    salt.enforceState(master, target, ['linux', 'openssh', 'ntp', 'rsyslog'])
     sleep(5)
     salt.runSaltProcessStep(master, target, 'mine.update', [], null, true)
     salt.enforceState(master, target, ['linux.network.host'])
@@ -64,7 +64,7 @@
     salt.fullRefresh(master, 'I@linux:system')
 
     salt.enforceState(master, 'I@salt:control', ['salt.minion'], true, false, null, false, 60, 2)
-    salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp'])
+    salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp', 'rsyslog'])
     salt.enforceState(master, 'I@salt:control', 'libvirt')
     salt.enforceState(master, 'I@salt:control', 'salt.control')
 
@@ -94,6 +94,7 @@
     // Install galera
     if (salt.testTarget(master, 'I@galera:master') || salt.testTarget(master, 'I@galera:slave')) {
         salt.enforceState(master, 'I@galera:master', 'galera', true, true, null, false, -1, 2)
+        salt.enforceState(master, 'I@galera:slave', 'galera', true, true, null, false, -1, 2)
 
         // Check galera status
         salt.runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
@@ -105,6 +106,7 @@
             salt.enforceState(master, 'I@mysql:client', 'mysql.client')
         }
     }
+    installBackup(master, 'mysql')
 
     // Install docker
     if (salt.testTarget(master, 'I@docker:host')) {
@@ -155,6 +157,7 @@
         }
         salt.enforceState(master, 'I@redis:server', 'redis')
     }
+    installBackup(master, 'common')
 }
 
 def installOpenstackInfra(master) {
@@ -172,6 +175,10 @@
     if (salt.testTarget(master, 'I@horizon:server')) {
         salt.enforceState(master, 'I@horizon:server', 'horizon')
     }
+    // Install sphinx server
+    if (salt.testTarget(master, 'I@sphinx:server')) {
+        salt.enforceState(master, 'I@sphinx:server', 'sphinx')
+    }
     if (salt.testTarget(master, 'I@nginx:server')) {
         salt.enforceState(master, 'I@nginx:server', 'salt.minion')
         salt.enforceState(master, 'I@nginx:server', 'nginx')
@@ -295,7 +302,7 @@
         salt.enforceState(master, 'I@manila:scheduler', 'manila.scheduler')
     }
 
-    // Install designate service
+    // Install designate services
     if (salt.testTarget(master, 'I@designate:server:enabled')) {
         if (salt.testTarget(master, 'I@designate:server:backend:bind9')) {
             salt.enforceState(master, 'I@bind:server', 'bind.server')
@@ -304,7 +311,7 @@
             salt.enforceState(master, 'I@powerdns:server', 'powerdns.server')
         }
         salt.enforceState(master, 'I@designate:server and *01*', 'designate.server')
-        salt.enforceState(master, 'I@designate:server', 'designate.server')
+        salt.enforceState(master, 'I@designate:server', 'designate')
     }
 
     // Install octavia api service
@@ -331,26 +338,31 @@
 
     // Install gnocchi server
     if (salt.testTarget(master, 'I@gnocchi:server')) {
+        salt.enforceState(master, 'I@gnocchi:server and *01*', 'gnocchi.server')
         salt.enforceState(master, 'I@gnocchi:server', 'gnocchi.server')
     }
 
     // Install gnocchi statsd
     if (salt.testTarget(master, 'I@gnocchi:statsd')) {
+        salt.enforceState(master, 'I@gnocchi:statsd and *01*', 'gnocchi.statsd')
         salt.enforceState(master, 'I@gnocchi:statsd', 'gnocchi.statsd')
     }
 
     // Install panko server
     if (salt.testTarget(master, 'I@panko:server')) {
+        salt.enforceState(master, 'I@panko:server and *01*', 'panko')
         salt.enforceState(master, 'I@panko:server', 'panko')
     }
 
     // Install ceilometer server
     if (salt.testTarget(master, 'I@ceilometer:server')) {
+        salt.enforceState(master, 'I@ceilometer:server and *01*', 'ceilometer')
         salt.enforceState(master, 'I@ceilometer:server', 'ceilometer')
     }
 
     // Install aodh server
     if (salt.testTarget(master, 'I@aodh:server')) {
+        salt.enforceState(master, 'I@aodh:server and *01*', 'aodh')
         salt.enforceState(master, 'I@aodh:server', 'aodh')
     }
 }
@@ -438,6 +450,7 @@
     if (salt.testTarget(master, 'I@docker:client and I@opencontrail:control')) {
         salt.enforceState(master, 'I@opencontrail:control or I@opencontrail:collector', 'docker.client')
     }
+    installBackup(master, 'contrail')
 }
 
 
@@ -640,7 +653,7 @@
     // Install galera
     if (common.checkContains('STACK_INSTALL', 'k8s')) {
         salt.enforceState(master, 'I@galera:master', 'galera', true, true, null, false, -1, 2)
-        salt.enforceState(master, 'I@galera:slave', 'galera')
+        salt.enforceState(master, 'I@galera:slave', 'galera', true, true, null, false, -1, 2)
 
         // Check galera status
         salt.runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
@@ -654,13 +667,18 @@
     sleep(5)
 
     // Configure Prometheus in Docker Swarm
-    salt.enforceState(master, 'I@docker:swarm and I@prometheus:server', 'prometheus', true, false)
+    salt.enforceState(master, 'I@docker:swarm and I@prometheus:server', 'prometheus')
 
     //Configure Remote Collector in Docker Swarm for Openstack deployments
     if (!common.checkContains('STACK_INSTALL', 'k8s')) {
         salt.enforceState(master, 'I@docker:swarm and I@prometheus:server', 'heka.remote_collector', true, false)
     }
 
+    // Install sphinx server
+    if (salt.testTarget(master, 'I@sphinx:server')) {
+        salt.enforceState(master, 'I@sphinx:server', 'sphinx')
+    }
+
     //Configure Grafana
     def pillar = salt.getPillar(master, 'ctl01*', '_param:stacklight_monitor_address')
     common.prettyPrint(pillar)
@@ -790,6 +808,76 @@
     }
 }
 
+//
+// backups
+//
+
+def installBackup(master, component='common') {
+    def salt = new com.mirantis.mk.Salt()
+    if (component == 'common') {
+        // Install Backupninja
+        if (salt.testTarget(master, 'I@backupninja:client')) {
+            salt.enforceState(master, 'I@backupninja:client', 'salt.minion.grains')
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'saltutil.sync_grains')
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'mine.flush')
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'mine.update')
+            salt.enforceState(master, 'I@backupninja:client', 'backupninja')
+        }
+        if (salt.testTarget(master, 'I@backupninja:server')) {
+            salt.enforceState(master, 'I@backupninja:server', 'salt.minion.grains')
+            salt.enforceState(master, 'I@backupninja:server', 'backupninja')
+        }
+    } else if (component == 'mysql') {
+        // Install Xtrabackup
+        if (salt.testTarget(master, 'I@xtrabackup:client')) {
+            salt.enforceState(master, 'I@xtrabackup:client', 'salt.minion.grains')
+            salt.runSaltProcessStep(master, 'I@xtrabackup:client', 'saltutil.sync_grains')
+            salt.runSaltProcessStep(master, 'I@xtrabackup:client', 'mine.flush')
+            salt.runSaltProcessStep(master, 'I@xtrabackup:client', 'mine.update')
+            salt.enforceState(master, 'I@xtrabackup:client', 'xtrabackup')
+        }
+        if (salt.testTarget(master, 'I@xtrabackup:server')) {
+            salt.enforceState(master, 'I@xtrabackup:server', 'xtrabackup')
+        }
+    } else if (component == 'contrail') {
+
+        // Install Cassandra backup
+        if (salt.testTarget(master, 'I@cassandra:backup:client')) {
+            salt.enforceState(master, 'I@cassandra:backup:client', 'salt.minion.grains')
+            salt.runSaltProcessStep(master, 'I@cassandra:backup:client', 'saltutil.sync_grains')
+            salt.runSaltProcessStep(master, 'I@cassandra:backup:client', 'mine.flush')
+            salt.runSaltProcessStep(master, 'I@cassandra:backup:client', 'mine.update')
+            salt.enforceState(master, 'I@cassandra:backup:client', 'cassandra.backup')
+        }
+        if (salt.testTarget(master, 'I@cassandra:backup:server')) {
+            salt.enforceState(master, 'I@cassandra:backup:server', 'cassandra.backup')
+        }
+        // Install Zookeeper backup
+        if (salt.testTarget(master, 'I@zookeeper:backup:client')) {
+            salt.enforceState(master, 'I@zookeeper:backup:client', 'salt.minion.grains')
+            salt.runSaltProcessStep(master, 'I@zookeeper:backup:client', 'saltutil.sync_grains')
+            salt.runSaltProcessStep(master, 'I@zookeeper:backup:client', 'mine.flush')
+            salt.runSaltProcessStep(master, 'I@zookeeper:backup:client', 'mine.update')
+            salt.enforceState(master, 'I@zookeeper:backup:client', 'zookeeper.backup')
+        }
+        if (salt.testTarget(master, 'I@zookeeper:backup:server')) {
+            salt.enforceState(master, 'I@zookeeper:backup:server', 'zookeeper.backup')
+        }
+    } else if (component == 'ceph') {
+        // Install Ceph backup
+        if (salt.testTarget(master, 'I@ceph:backup:client')) {
+            salt.enforceState(master, 'I@ceph:backup:client', 'salt.minion.grains')
+            salt.runSaltProcessStep(master, 'I@ceph:backup:client', 'saltutil.sync_grains')
+            salt.runSaltProcessStep(master, 'I@ceph:backup:client', 'mine.flush')
+            salt.runSaltProcessStep(master, 'I@ceph:backup:client', 'mine.update')
+            salt.enforceState(master, 'I@ceph:backup:client', 'ceph.backup')
+        }
+        if (salt.testTarget(master, 'I@ceph:backup:server')) {
+            salt.enforceState(master, 'I@ceph:backup:server', 'ceph.backup')
+        }
+    }
+
+}
 
 //
 // Ceph
@@ -823,6 +911,7 @@
     salt.enforceState(master, target, 'ceph.osd.custom')
     salt.runSaltProcessStep(master, 'I@ceph:osd', 'saltutil.sync_grains')
     salt.runSaltProcessStep(master, 'I@ceph:osd', 'mine.update')
+    installBackup(master, 'ceph')
 
     // setup pools, keyrings and maybe crush
     if (salt.testTarget(master, 'I@ceph:setup') && setup) {
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index 10236b5..eb4b19c 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -311,7 +311,8 @@
     def common = new com.mirantis.mk.Common()
 
     // virtualenv setup
-    requirements = ['salt-pepper>=0.5.2']
+    // pin pepper till https://mirantis.jira.com/browse/PROD-18188 is fixed
+    requirements = ['salt-pepper>=0.5.2,<0.5.4']
     setupVirtualenv(path, 'python2', requirements, null, true, true)
 
     // pepperrc creation