Merge "update ceph pipline for refactored grains" into release/proposed/2019.2.0
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 5ca197b..6e74379 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -253,39 +253,6 @@
}
/**
-* Upgrade packages on given target.
-*
-* @param env Salt Connection object or env Salt command map
-* @param target Salt target to upgrade packages on.
-*/
-def osUpgrade(env, target, batch=null) {
- def common = new com.mirantis.mk.Common()
- def salt = new com.mirantis.mk.Salt()
-
- common.infoMsg("Running upgrade on ${target}")
-
- salt.runSaltProcessStep(env, target, 'pkg.refresh_db', [], batch, true)
- def cmd = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" upgrade'
- salt.cmdRun(env, target, cmd, true, batch)
-}
-
-/**
-* Running dist-upgrade on given target.
-*
-* @param env Salt Connection object or env Salt command map
-* @param target Salt target to upgrade packages on.
-*/
-def osDistUpgrade(env, target, batch=null) {
- def salt = new com.mirantis.mk.Salt()
- def common = new com.mirantis.mk.Common()
-
- common.infoMsg("Running dist-upgrade on ${target}")
- salt.runSaltProcessStep(env, target, 'pkg.refresh_db', [], batch, true)
- def cmd = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade'
- salt.cmdRun(env, target, cmd, true, batch)
-}
-
-/**
* Reboot specified target, and wait when minion is UP.
*
* @param env Salt Connection object or env Salt command map
@@ -318,22 +285,28 @@
* @param attempts Number of attemps to wait for.
*/
def osUpgradeNode(env, target, mode, postponeReboot=false, timeout=30, attempts=10, batch=null) {
- def common = new com.mirantis.mk.Common()
- def salt = new com.mirantis.mk.Salt()
+ if(mode in ['upgrade', 'dist-upgrade']) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
+ def rebootRequired = false
- def rebootRequired = false
- if (mode == 'dist-upgrade') {
- osDistUpgrade(env, target, batch)
- } else if (mode == 'upgrade') {
- osUpgrade(env, target, batch)
- }
- rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
- if (rebootRequired) {
- if (!postponeReboot) {
- common.infoMsg("Reboot is required after upgrade on ${target} Rebooting...")
- osReboot(env, target, timeout, attempts)
- } else {
- common.infoMsg("Postponing reboot on node ${target}")
+ common.infoMsg("Running apt ${mode} on ${target}")
+ common.retry(3, 5) {
+ salt.cmdRun(env, target, 'salt-call pkg.refresh_db failhard=true', true, batch)
}
+ def cmd = "export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ${mode}"
+ salt.cmdRun(env, target, cmd, true, batch)
+
+ rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
+ if (rebootRequired) {
+ if (!postponeReboot) {
+ common.infoMsg("Reboot is required after upgrade on ${target} Rebooting...")
+ osReboot(env, target, timeout, attempts)
+ } else {
+ common.infoMsg("Postponing reboot on node ${target}")
+ }
+ }
+ } else {
+ common.errorMsg("Invalid upgrade mode specified: ${mode}. Has to be 'upgrade' or 'dist-upgrade'")
}
}
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 21635f5..c8ff08f 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -30,11 +30,11 @@
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['linux.system']])
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.master'], failOnError: false, read_timeout: 120, retries: 2])
- salt.fullRefresh(master, "* ${extra_tgt}", batch)
+ salt.fullRefresh(master, "* ${extra_tgt}", null, true)
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.minion'], failOnError: false, read_timeout: 60, retries: 2])
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.minion']])
- salt.fullRefresh(master, "* ${extra_tgt}", batch)
+ salt.fullRefresh(master, "* ${extra_tgt}", null, true)
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.network.proxy'], batch: batch, failOnError: false, read_timeout: 180, retries: 2])
// Make sure all repositories are in place before proceeding with package installation from other states
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.system.repo'], batch: batch, failOnError: false, read_timeout: 180, retries: 2])
@@ -52,12 +52,13 @@
sleep(5)
salt.enforceState([saltId: master, target: "I@linux:system ${extra_tgt}", state: ['linux', 'openssh', 'ntp', 'rsyslog'], batch: batch])
-
- salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['salt.minion'], failOnError: false, batch: batch, read_timeout: 180, retries: 2])
-
+ def saltMinionNodes = salt.getMinions(master, "* ${extra_tgt}")
+ for (minion in saltMinionNodes) {
+ salt.enforceState([saltId: master, target: minion, state: ['salt.minion'], failOnError: false, batch: null, read_timeout: 180, retries: 2])
+ }
sleep(5)
- salt.fullRefresh(master, "* ${extra_tgt}", batch)
+ salt.fullRefresh(master, "* ${extra_tgt}", null, true)
salt.runSaltProcessStep(master, "* ${extra_tgt}", 'mine.update', [], batch, true)
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.network.host'], batch: batch])
// WA for PROD-33911
@@ -806,20 +807,26 @@
salt.enforceStateWithTest([saltId: master, target: "I@python:environment ${extra_tgt}", state: 'python'])
- withEnv(['ASK_ON_ERROR=false']){
- retry(2){
- try{
- salt.enforceState([saltId: master, target: "I@gerrit:client ${extra_tgt}", state: 'gerrit'])
- }catch(e){
- salt.fullRefresh(master, "I@gerrit:client ${extra_tgt}")
+ // Workaround for PROD-35056: jeepyb package may trigger update of requests and urllib3 packages
+ // which are actively used by salt-minion, so salt-minion has to be restart to apply changes
+ salt.upgradePackageAndRestartSaltMinion(master, jenkins_compound, 'python-requests')
+ salt.upgradePackageAndRestartSaltMinion(master, jenkins_compound, 'python-urllib3')
+ withEnv(['ASK_ON_ERROR=false']) {
+ retry(2) {
+ try {
+ salt.enforceState([saltId: master, target: gerrit_compound, state: 'gerrit'])
+ } catch (e) {
+ salt.restartSaltMinion(master, gerrit_compound)
+ salt.fullRefresh(master, gerrit_compound)
throw e //rethrow for retry handler
}
}
- retry(2){
- try{
- salt.enforceState([saltId: master, target: "I@jenkins:client ${extra_tgt}", state: 'jenkins'])
- }catch(e){
- salt.fullRefresh(master, "I@jenkins:client ${extra_tgt}")
+ retry(2) {
+ try {
+ salt.enforceState([saltId: master, target: jenkins_compound, state: 'jenkins'])
+ } catch (e) {
+ salt.restartSaltMinion(master, jenkins_compound)
+ salt.fullRefresh(master, jenkins_compound)
throw e //rethrow for retry handler
}
}
@@ -1041,7 +1048,10 @@
// Install collectd, heka and sensu services on the nodes, this will also
// generate the metadata that goes into the grains and eventually into Salt Mine
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'collectd'])
- salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'salt.minion', retries: 2])
+ def saltMinionsNodes = salt.getMinions(master, "* ${extra_tgt}")
+ for (minion in saltMinionsNodes) {
+ salt.enforceState([saltId: master, target: minion, state: 'salt.minion', retries: 2])
+ }
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'heka'])
// Gather the Grafana metadata as grains
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index f4147e0..30957e5 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -687,12 +687,22 @@
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
* @param target Get pillar target
* @param batch Batch param to salt (integer or string with percents)
+ * @param oneByOne Refresh each node separately
* @return output of salt command
*/
-def fullRefresh(saltId, target, batch=20){
- runSaltProcessStep(saltId, target, 'saltutil.refresh_pillar', [], batch, true)
- runSaltProcessStep(saltId, target, 'saltutil.refresh_grains', [], batch, true)
- runSaltProcessStep(saltId, target, 'saltutil.sync_all', [], batch, true)
+def fullRefresh(saltId, target, batch=20, oneByOne=false) {
+ if (oneByOne) {
+ def minions = getMinions(saltId, target)
+ for (minion in minions) {
+ runSaltProcessStep(saltId, minion, 'saltutil.refresh_pillar', [], null, true, 60)
+ runSaltProcessStep(saltId, minion, 'saltutil.refresh_grains', [], null, true, 60)
+ runSaltProcessStep(saltId, minion, 'saltutil.sync_all', [], null, true, 180)
+ }
+ } else {
+ runSaltProcessStep(saltId, target, 'saltutil.refresh_pillar', [], batch, true)
+ runSaltProcessStep(saltId, target, 'saltutil.refresh_grains', [], batch, true)
+ runSaltProcessStep(saltId, target, 'saltutil.sync_all', [], batch, true)
+ }
}
/**