Merge "Retry dist-upgrades" into release/proposed/2019.2.0
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 6d74ab0..47cd772 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -294,10 +294,21 @@
         common.retry(3, 5) {
             salt.cmdRun(env, target, 'salt-call pkg.refresh_db failhard=true', true, batch)
         }
+
         /* first try to upgrade salt components since they demand asynchronous upgrade */
         upgradeSaltPackages(env, target)
         def cmd = "export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ${mode}"
-        salt.cmdRun(env, target, cmd, true, batch)
+
+        /*
+         * This is a long running batch operation that may return empty response
+         * which is a pretty typical salt behavior. This does not represent an error
+         * but might hide the error if it's ignored. If there is no persistent error
+         * with the procedure itself, the consequent run will succeed.
+         */
+        common.retry(2, 120) {
+            salt.cmdRun(env, target, cmd, true, batch)
+        }
+
         rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
         if (rebootRequired) {
             if (!postponeReboot) {
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 2c58e9a..48309ff 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -357,14 +357,14 @@
                     def nodeKey = node.keySet()[j]
                     if (node[nodeKey] instanceof String) {
                         if (!node[nodeKey].contains("Salt command execution success")) {
-                            throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+                            throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
                         }
                     } else if (node[nodeKey] instanceof Boolean) {
                         if (!node[nodeKey]) {
-                            throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+                            throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
                         }
                     } else {
-                        throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
+                        throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns unexpected data type: ${node[nodeKey]}")
                     }
                 }
             }