Add correct salt minions update
osUpgradeNode is not aware of how to update salt minions, thus is failing
when salt master looses the connection to minions. This patch add the proper
procedure to update salt minions before other packages will be updated.
Closes-Bug: PROD-35355
Change-Id: I185a121999f4a49c8b5a2a126cca0f3710149118
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 6e74379..6d74ab0 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -294,9 +294,10 @@
common.retry(3, 5) {
salt.cmdRun(env, target, 'salt-call pkg.refresh_db failhard=true', true, batch)
}
+ /* first try to upgrade salt components since they demand asynchronous upgrade */
+ upgradeSaltPackages(env, target)
def cmd = "export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ${mode}"
salt.cmdRun(env, target, cmd, true, batch)
-
rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
if (rebootRequired) {
if (!postponeReboot) {
@@ -310,3 +311,36 @@
common.errorMsg("Invalid upgrade mode specified: ${mode}. Has to be 'upgrade' or 'dist-upgrade'")
}
}
+
+/**
+* Upgrade salt packages on target asynchronously, wait minions' availability.
+*
+* @param env Salt Connection object or env Salt command map
+* @param target Salt target to upgrade packages on.
+* @param timeout Sleep timeout when doing retries.
+* @param attempts Number of attemps to wait for.
+*/
+def upgradeSaltPackages(env, target, timeout=60, attempts=20) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
+ def saltUpgradeCmd =
+ 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q ' +
+ '-o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ' +
+ 'install --only-upgrade salt-master salt-common salt-api salt-minion'
+
+ common.infoMsg("Upgrading SaltStack on ${target}")
+ salt.cmdRun(env, target, saltUpgradeCmd, false, null, true, [], [], true)
+ /* wait for 2 mins before checking the availability of minions to give
+ apt some time to finish updating so the dpkg releases its locks */
+ sleep(120)
+ /* taken from upgrade-mcp-release */
+ common.retry(attempts, timeout) {
+ salt.minionsReachable(env, 'I@salt:master', target)
+ def running = salt.runSaltProcessStep(env, target, 'saltutil.running', [], null, true, 5)
+ for (value in running.get("return")[0].values()) {
+ if (value != []) {
+ throw new Exception("Not all salt-minions are ready for execution")
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 30957e5..2c58e9a 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -332,9 +332,10 @@
* @param output do you want to print output
* @param saltArgs additional salt args eq. ["runas=aptly"]
* @param replacing list with maps for deletion in info message (passwords, logins, etc)
+ * @param async run commands with async client (default false)
* @return output of salt command
*/
-def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = [], replacing = []) {
+def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = [], replacing = [], async = false) {
def common = new com.mirantis.mk.Common()
def originalCmd = cmd
common.infoSensitivityMsg("Running command ${cmd} on ${target}", true, replacing)
@@ -345,7 +346,8 @@
// add cmd name to salt args list
saltArgs << cmd
- def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.run', batch, saltArgs.reverse())
+ def client = async ? 'local_async' : 'local'
+ def out = runSaltCommand(saltId, client, ['expression': target, 'type': 'compound'], 'cmd.run', batch, saltArgs.reverse())
if (checkResponse) {
// iterate over all affected nodes and check success return code
if (out["return"]){