Merge "Add mine.update before network.host state."
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 2d75d1e..ff05ce3 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -38,10 +38,11 @@
export DEBUG="${debug}" &&
export LD_LIBRARY_PATH=\${LD_LIBRARY_PATH:+"\$LD_LIBRARY_PATH:"}/usr/lib/libeatmydata &&
export LD_PRELOAD=\${LD_PRELOAD:+"\$LD_PRELOAD "}libeatmydata.so &&
- [[ -z "${extraRepoUrl}" && "${extraRepoUrl}" != "null" ]] || echo "${extraRepoUrl}" >/etc/apt/sources.list.d/extra.list &&
+ [[ -z "${extraRepoUrl}" && "${extraRepoUrl}" != "null" ]] || echo "${extraRepoUrl}" | tr ";" "\\\\n" >/etc/apt/sources.list.d/extra.list &&
[[ -z "${extraRepoKeyUrl}" && "${extraRepoKeyUrl}" != "null" ]] || (
which curl || (apt-get update && apt-get install -y curl) &&
- curl --insecure -ss -f "${extraRepoKeyUrl}" | apt-key add -
+ EXTRAKEY=`echo "${extraRepoKeyUrl}" | tr ";" " "` &&
+ for RepoKey in \${EXTRAKEY}; do curl --insecure -ss -f "\${RepoKey}" | apt-key add - ; done
) &&
apt-get update && apt-get install -y build-essential devscripts equivs sudo &&
groupadd -g ${jenkinsGID} jenkins &&
@@ -53,7 +54,6 @@
sudo -H -E -u jenkins debuild --preserve-envvar DEBUG --no-lintian -uc -us -b'""")
}
-
}
/*
diff --git a/src/com/mirantis/mk/Docker.groovy b/src/com/mirantis/mk/Docker.groovy
index 1357dc1..7b57e5b 100644
--- a/src/com/mirantis/mk/Docker.groovy
+++ b/src/com/mirantis/mk/Docker.groovy
@@ -22,6 +22,7 @@
sh "git checkout -f ${dockerFile}; sed -i -e 's,^FROM.*,FROM ${baseImg},g' ${dockerFile}"
}
+ params << "--no-cache"
params << "-f ${dockerFile}"
params << imageDir
@@ -51,4 +52,4 @@
}
return img
-}
\ No newline at end of file
+}
diff --git a/src/com/mirantis/mk/Git.groovy b/src/com/mirantis/mk/Git.groovy
index 65b80ab..9e3c460 100644
--- a/src/com/mirantis/mk/Git.groovy
+++ b/src/com/mirantis/mk/Git.groovy
@@ -56,7 +56,7 @@
def changeGitBranch(path, branch) {
dir(path) {
git_cmd = sh (
- script: "git checkout -b ${branch}",
+ script: "git checkout ${branch}",
returnStdout: true
).trim()
}
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index 91dfbfe..85f014f 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -378,16 +378,35 @@
* @param probe single node on which to list service names
* @param target all targeted nodes
* @param services lists of type of services to be stopped
+ * @param confirm enable/disable manual service stop confirmation
* @return output of salt commands
*/
-def stopServices(env, probe, target, services=[]) {
+def stopServices(env, probe, target, services=[], confirm=false) {
def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
for (s in services) {
def outputServicesStr = salt.getReturnValues(salt.cmdRun(env, "${probe}*", "service --status-all | grep ${s} | awk \'{print \$4}\'"))
- def servicesList = outputServicesStr.tokenize("\n")
- for (name in servicesList) {
- if (!name.contains('Salt command')) {
- salt.runSaltProcessStep(env, "${target}*", 'service.stop', ["${name}"])
+ def servicesList = outputServicesStr.tokenize("\n").init()
+ if (confirm) {
+ if (servicesList) {
+ try {
+ input message: "Click PROCEED to stop ${servicesList}. Otherwise click ABORT to skip stopping them."
+ for (name in servicesList) {
+ if (!name.contains('Salt command')) {
+ salt.runSaltProcessStep(env, "${target}*", 'service.stop', ["${name}"])
+ }
+ }
+ } catch (Exception er) {
+ common.infoMsg("skipping stopping ${servicesList} services")
+ }
+ }
+ } else {
+ if (servicesList) {
+ for (name in servicesList) {
+ if (!name.contains('Salt command')) {
+ salt.runSaltProcessStep(env, "${target}*", 'service.stop', ["${name}"])
+ }
+ }
}
}
}
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 33889df..4fc8854 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -34,7 +34,7 @@
if (staticMgmtNet) {
salt.runSaltProcessStep(master, '*', 'cmd.shell', ["salt-call state.sls linux.network; salt-call service.restart salt-minion"], null, true, 60)
}
- salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'ntp'])
+ salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'ntp', 'rsyslog'])
salt.enforceState(master, '*', ['salt.minion'], true, false, null, false, 60, 2)
sleep(5)
salt.runSaltProcessStep(master, '*', 'mine.update', [], null, true)
@@ -53,7 +53,7 @@
salt.enforceState(master, target, ['salt.minion'], true, false, null, false, 60, 2)
salt.enforceState(master, target, ['salt.minion'])
- salt.enforceState(master, target, ['linux', 'openssh', 'ntp'])
+ salt.enforceState(master, target, ['linux', 'openssh', 'ntp', 'rsyslog'])
sleep(5)
salt.runSaltProcessStep(master, target, 'mine.update', [], null, true)
salt.enforceState(master, target, ['linux.network.host'])
@@ -64,7 +64,7 @@
salt.fullRefresh(master, 'I@linux:system')
salt.enforceState(master, 'I@salt:control', ['salt.minion'], true, false, null, false, 60, 2)
- salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp'])
+ salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp', 'rsyslog'])
salt.enforceState(master, 'I@salt:control', 'libvirt')
salt.enforceState(master, 'I@salt:control', 'salt.control')
@@ -94,6 +94,7 @@
// Install galera
if (salt.testTarget(master, 'I@galera:master') || salt.testTarget(master, 'I@galera:slave')) {
salt.enforceState(master, 'I@galera:master', 'galera', true, true, null, false, -1, 2)
+ salt.enforceState(master, 'I@galera:slave', 'galera', true, true, null, false, -1, 2)
// Check galera status
salt.runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
@@ -105,6 +106,7 @@
salt.enforceState(master, 'I@mysql:client', 'mysql.client')
}
}
+ installBackup(master, 'mysql')
// Install docker
if (salt.testTarget(master, 'I@docker:host')) {
@@ -155,6 +157,7 @@
}
salt.enforceState(master, 'I@redis:server', 'redis')
}
+ installBackup(master, 'common')
}
def installOpenstackInfra(master) {
@@ -172,6 +175,10 @@
if (salt.testTarget(master, 'I@horizon:server')) {
salt.enforceState(master, 'I@horizon:server', 'horizon')
}
+ // Install sphinx server
+ if (salt.testTarget(master, 'I@sphinx:server')) {
+ salt.enforceState(master, 'I@sphinx:server', 'sphinx')
+ }
if (salt.testTarget(master, 'I@nginx:server')) {
salt.enforceState(master, 'I@nginx:server', 'salt.minion')
salt.enforceState(master, 'I@nginx:server', 'nginx')
@@ -295,7 +302,7 @@
salt.enforceState(master, 'I@manila:scheduler', 'manila.scheduler')
}
- // Install designate service
+ // Install designate services
if (salt.testTarget(master, 'I@designate:server:enabled')) {
if (salt.testTarget(master, 'I@designate:server:backend:bind9')) {
salt.enforceState(master, 'I@bind:server', 'bind.server')
@@ -304,7 +311,7 @@
salt.enforceState(master, 'I@powerdns:server', 'powerdns.server')
}
salt.enforceState(master, 'I@designate:server and *01*', 'designate.server')
- salt.enforceState(master, 'I@designate:server', 'designate.server')
+ salt.enforceState(master, 'I@designate:server', 'designate')
}
// Install octavia api service
@@ -331,26 +338,31 @@
// Install gnocchi server
if (salt.testTarget(master, 'I@gnocchi:server')) {
+ salt.enforceState(master, 'I@gnocchi:server and *01*', 'gnocchi.server')
salt.enforceState(master, 'I@gnocchi:server', 'gnocchi.server')
}
// Install gnocchi statsd
if (salt.testTarget(master, 'I@gnocchi:statsd')) {
+ salt.enforceState(master, 'I@gnocchi:statsd and *01*', 'gnocchi.statsd')
salt.enforceState(master, 'I@gnocchi:statsd', 'gnocchi.statsd')
}
// Install panko server
if (salt.testTarget(master, 'I@panko:server')) {
+ salt.enforceState(master, 'I@panko:server and *01*', 'panko')
salt.enforceState(master, 'I@panko:server', 'panko')
}
// Install ceilometer server
if (salt.testTarget(master, 'I@ceilometer:server')) {
+ salt.enforceState(master, 'I@ceilometer:server and *01*', 'ceilometer')
salt.enforceState(master, 'I@ceilometer:server', 'ceilometer')
}
// Install aodh server
if (salt.testTarget(master, 'I@aodh:server')) {
+ salt.enforceState(master, 'I@aodh:server and *01*', 'aodh')
salt.enforceState(master, 'I@aodh:server', 'aodh')
}
}
@@ -438,6 +450,7 @@
if (salt.testTarget(master, 'I@docker:client and I@opencontrail:control')) {
salt.enforceState(master, 'I@opencontrail:control or I@opencontrail:collector', 'docker.client')
}
+ installBackup(master, 'contrail')
}
@@ -640,7 +653,7 @@
// Install galera
if (common.checkContains('STACK_INSTALL', 'k8s')) {
salt.enforceState(master, 'I@galera:master', 'galera', true, true, null, false, -1, 2)
- salt.enforceState(master, 'I@galera:slave', 'galera')
+ salt.enforceState(master, 'I@galera:slave', 'galera', true, true, null, false, -1, 2)
// Check galera status
salt.runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
@@ -654,13 +667,18 @@
sleep(5)
// Configure Prometheus in Docker Swarm
- salt.enforceState(master, 'I@docker:swarm and I@prometheus:server', 'prometheus', true, false)
+ salt.enforceState(master, 'I@docker:swarm and I@prometheus:server', 'prometheus')
//Configure Remote Collector in Docker Swarm for Openstack deployments
if (!common.checkContains('STACK_INSTALL', 'k8s')) {
salt.enforceState(master, 'I@docker:swarm and I@prometheus:server', 'heka.remote_collector', true, false)
}
+ // Install sphinx server
+ if (salt.testTarget(master, 'I@sphinx:server')) {
+ salt.enforceState(master, 'I@sphinx:server', 'sphinx')
+ }
+
//Configure Grafana
def pillar = salt.getPillar(master, 'ctl01*', '_param:stacklight_monitor_address')
common.prettyPrint(pillar)
@@ -790,6 +808,76 @@
}
}
+//
+// backups
+//
+
+def installBackup(master, component='common') {
+ def salt = new com.mirantis.mk.Salt()
+ if (component == 'common') {
+ // Install Backupninja
+ if (salt.testTarget(master, 'I@backupninja:client')) {
+ salt.enforceState(master, 'I@backupninja:client', 'salt.minion.grains')
+ salt.runSaltProcessStep(master, 'I@backupninja:client', 'saltutil.sync_grains')
+ salt.runSaltProcessStep(master, 'I@backupninja:client', 'mine.flush')
+ salt.runSaltProcessStep(master, 'I@backupninja:client', 'mine.update')
+ salt.enforceState(master, 'I@backupninja:client', 'backupninja')
+ }
+ if (salt.testTarget(master, 'I@backupninja:server')) {
+ salt.enforceState(master, 'I@backupninja:server', 'salt.minion.grains')
+ salt.enforceState(master, 'I@backupninja:server', 'backupninja')
+ }
+ } else if (component == 'mysql') {
+ // Install Xtrabackup
+ if (salt.testTarget(master, 'I@xtrabackup:client')) {
+ salt.enforceState(master, 'I@xtrabackup:client', 'salt.minion.grains')
+ salt.runSaltProcessStep(master, 'I@xtrabackup:client', 'saltutil.sync_grains')
+ salt.runSaltProcessStep(master, 'I@xtrabackup:client', 'mine.flush')
+ salt.runSaltProcessStep(master, 'I@xtrabackup:client', 'mine.update')
+ salt.enforceState(master, 'I@xtrabackup:client', 'xtrabackup')
+ }
+ if (salt.testTarget(master, 'I@xtrabackup:server')) {
+ salt.enforceState(master, 'I@xtrabackup:server', 'xtrabackup')
+ }
+ } else if (component == 'contrail') {
+
+ // Install Cassandra backup
+ if (salt.testTarget(master, 'I@cassandra:backup:client')) {
+ salt.enforceState(master, 'I@cassandra:backup:client', 'salt.minion.grains')
+ salt.runSaltProcessStep(master, 'I@cassandra:backup:client', 'saltutil.sync_grains')
+ salt.runSaltProcessStep(master, 'I@cassandra:backup:client', 'mine.flush')
+ salt.runSaltProcessStep(master, 'I@cassandra:backup:client', 'mine.update')
+ salt.enforceState(master, 'I@cassandra:backup:client', 'cassandra.backup')
+ }
+ if (salt.testTarget(master, 'I@cassandra:backup:server')) {
+ salt.enforceState(master, 'I@cassandra:backup:server', 'cassandra.backup')
+ }
+ // Install Zookeeper backup
+ if (salt.testTarget(master, 'I@zookeeper:backup:client')) {
+ salt.enforceState(master, 'I@zookeeper:backup:client', 'salt.minion.grains')
+ salt.runSaltProcessStep(master, 'I@zookeeper:backup:client', 'saltutil.sync_grains')
+ salt.runSaltProcessStep(master, 'I@zookeeper:backup:client', 'mine.flush')
+ salt.runSaltProcessStep(master, 'I@zookeeper:backup:client', 'mine.update')
+ salt.enforceState(master, 'I@zookeeper:backup:client', 'zookeeper.backup')
+ }
+ if (salt.testTarget(master, 'I@zookeeper:backup:server')) {
+ salt.enforceState(master, 'I@zookeeper:backup:server', 'zookeeper.backup')
+ }
+ } else if (component == 'ceph') {
+ // Install Ceph backup
+ if (salt.testTarget(master, 'I@ceph:backup:client')) {
+ salt.enforceState(master, 'I@ceph:backup:client', 'salt.minion.grains')
+ salt.runSaltProcessStep(master, 'I@ceph:backup:client', 'saltutil.sync_grains')
+ salt.runSaltProcessStep(master, 'I@ceph:backup:client', 'mine.flush')
+ salt.runSaltProcessStep(master, 'I@ceph:backup:client', 'mine.update')
+ salt.enforceState(master, 'I@ceph:backup:client', 'ceph.backup')
+ }
+ if (salt.testTarget(master, 'I@ceph:backup:server')) {
+ salt.enforceState(master, 'I@ceph:backup:server', 'ceph.backup')
+ }
+ }
+
+}
//
// Ceph
@@ -823,6 +911,7 @@
salt.enforceState(master, target, 'ceph.osd.custom')
salt.runSaltProcessStep(master, 'I@ceph:osd', 'saltutil.sync_grains')
salt.runSaltProcessStep(master, 'I@ceph:osd', 'mine.update')
+ installBackup(master, 'ceph')
// setup pools, keyrings and maybe crush
if (salt.testTarget(master, 'I@ceph:setup') && setup) {
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index 10236b5..eb4b19c 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -311,7 +311,8 @@
def common = new com.mirantis.mk.Common()
// virtualenv setup
- requirements = ['salt-pepper>=0.5.2']
+ // pin pepper till https://mirantis.jira.com/browse/PROD-18188 is fixed
+ requirements = ['salt-pepper>=0.5.2,<0.5.4']
setupVirtualenv(path, 'python2', requirements, null, true, true)
// pepperrc creation