Merge "Retry dist-upgrades" into release/proposed/2019.2.0
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 6d74ab0..47cd772 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -294,10 +294,21 @@
common.retry(3, 5) {
salt.cmdRun(env, target, 'salt-call pkg.refresh_db failhard=true', true, batch)
}
+
/* first try to upgrade salt components since they demand asynchronous upgrade */
upgradeSaltPackages(env, target)
def cmd = "export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ${mode}"
- salt.cmdRun(env, target, cmd, true, batch)
+
+ /*
+ * This is a long running batch operation that may return empty response
+ * which is a pretty typical salt behavior. This does not represent an error
+ * but might hide the error if it's ignored. If there is no persistent error
+ * with the procedure itself, the consequent run will succeed.
+ */
+ common.retry(2, 120) {
+ salt.cmdRun(env, target, cmd, true, batch)
+ }
+
rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
if (rebootRequired) {
if (!postponeReboot) {
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 2c58e9a..48309ff 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -357,14 +357,14 @@
def nodeKey = node.keySet()[j]
if (node[nodeKey] instanceof String) {
if (!node[nodeKey].contains("Salt command execution success")) {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
}
} else if (node[nodeKey] instanceof Boolean) {
if (!node[nodeKey]) {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
}
} else {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns unexpected data type: ${node[nodeKey]}")
}
}
}