Retry dist-upgrades
If the 'apt-get dist-upgrade' operations running long on the minions in batch
salt master may return an empty string for some minions. The pipelines do not
expectd this and threat it as an error. This patch adds a retry which one one
hand will succeed on the second run since the operation will be much shorter,
on the other hand it will not hide the real issue if it happens.
Closes-Bug: PROD-35355
Change-Id: I95d4c33bf7e28b369f5dc3f468944be15a5d5d6a
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 6d74ab0..47cd772 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -294,10 +294,21 @@
common.retry(3, 5) {
salt.cmdRun(env, target, 'salt-call pkg.refresh_db failhard=true', true, batch)
}
+
/* first try to upgrade salt components since they demand asynchronous upgrade */
upgradeSaltPackages(env, target)
def cmd = "export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ${mode}"
- salt.cmdRun(env, target, cmd, true, batch)
+
+ /*
+ * This is a long running batch operation that may return empty response
+ * which is a pretty typical salt behavior. This does not represent an error
+ * but might hide the error if it's ignored. If there is no persistent error
+ * with the procedure itself, the consequent run will succeed.
+ */
+ common.retry(2, 120) {
+ salt.cmdRun(env, target, cmd, true, batch)
+ }
+
rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
if (rebootRequired) {
if (!postponeReboot) {
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 2c58e9a..48309ff 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -357,14 +357,14 @@
def nodeKey = node.keySet()[j]
if (node[nodeKey] instanceof String) {
if (!node[nodeKey].contains("Salt command execution success")) {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
}
} else if (node[nodeKey] instanceof Boolean) {
if (!node[nodeKey]) {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
}
} else {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns unexpected data type: ${node[nodeKey]}")
}
}
}