commit e36ab7cdc2442db7dd7e7f131fb300e5430b075d
author Vasyl Saienko <vsaienko@mirantis.com> Mon Jul 17 14:35:48 2017 +0300
committer Vasyl Saienko <vsaienko@mirantis.com> Wed Jul 19 05:28:30 2017 +0000
tree 4b35389cfa0742a26a085a7786b6130b5bbc9321
parent 115a78fd1cb4f59ea13b9b32285dd05f717c1bec

Don't stuck when salt minion/master is restarted

When applying salt.{master|minion} states salt may be restarted.
As result existed call stuck. This patch allows to set response
timeout for salt api POST calls, and if it is reached, retry it.
Enable retries for all salt.master|salt.api calls with read_timeout=60

Change-Id: Ibdf0f5ce0bc2e4640b7502b0abe71f81557e244d

src/com/mirantis/mk/Orchestrate.groovy

diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index fa5e8d3..f33e198 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -17,17 +17,18 @@
 def installFoundationInfra(master) {
     def salt = new com.mirantis.mk.Salt()
 
-    salt.enforceState(master, 'I@salt:master', ['salt.master', 'reclass'], true)
+    salt.enforceState(master, 'I@salt:master', ['salt.master', 'reclass'], true, false, null, false, 60, 2)
 
     salt.runSaltProcessStep(master, '*', 'saltutil.refresh_pillar', [], null, true)
     salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
 
     salt.enforceState(master, 'I@salt:master', ['linux.system'], true)
+    salt.enforceState(master, 'I@salt:master', ['salt.minion'], true, false, null, false, 60, 2)
     salt.enforceState(master, 'I@salt:master', ['salt.minion'], true)
 
     salt.enforceState(master, '*', ['linux.system'], true)
-    salt.enforceState(master, '*', ['salt.minion'], true)
-    salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
+    salt.enforceState(master, '*', ['salt.minion'], true, false, null, false, 60, 2)
+    salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'ntp'], true)
 }
 
 def installInfraKvm(master) {
@@ -36,7 +37,8 @@
     salt.runSaltProcessStep(master, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
     salt.runSaltProcessStep(master, 'I@linux:system', 'saltutil.sync_all', [], null, true)
 
-    salt.enforceState(master, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
+    salt.enforceState(master, 'I@salt:control' ['salt.minion'], true, false, null, false, 60, 2)
+    salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp'], true)
     salt.enforceState(master, 'I@salt:control', 'libvirt', true)
     salt.enforceState(master, 'I@salt:control', 'salt.control', true)