commit e36ab7cdc2442db7dd7e7f131fb300e5430b075d
author Vasyl Saienko <vsaienko@mirantis.com> Mon Jul 17 14:35:48 2017 +0300
committer Vasyl Saienko <vsaienko@mirantis.com> Wed Jul 19 05:28:30 2017 +0000
tree 4b35389cfa0742a26a085a7786b6130b5bbc9321
parent 115a78fd1cb4f59ea13b9b32285dd05f717c1bec

Don't stuck when salt minion/master is restarted

When applying salt.{master|minion} states salt may be restarted.
As result existed call stuck. This patch allows to set response
timeout for salt api POST calls, and if it is reached, retry it.
Enable retries for all salt.master|salt.api calls with read_timeout=60

Change-Id: Ibdf0f5ce0bc2e4640b7502b0abe71f81557e244d

src/com/mirantis/mk/Salt.groovy

diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index c040fb5..d7a962a 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -54,9 +54,10 @@
  * @param args     Additional arguments to function
  * @param kwargs   Additional key-value arguments to function
  * @param timeout  Additional argument salt api timeout
+ * @param read_timeout http session read timeout
  */
 @NonCPS
-def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null, timeout = -1) {
+def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null, timeout = -1, read_timeout = -1) {
     def http = new com.mirantis.mk.Http()
 
     data = [
@@ -87,7 +88,7 @@
       'X-Auth-Token': "${master.authToken}"
     ]
 
-    return http.sendHttpPostRequest("${master.url}/", data, headers)
+    return http.sendHttpPostRequest("${master.url}/", data, headers, read_timeout)
 }
 
 /**
@@ -128,10 +129,11 @@
  * @param output print output (optional, default true)
  * @param failOnError throw exception on salt state result:false (optional, default true)
  * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
- * @param optional don't fail on empty response from salt caused by 'No minions matched the targed' if set to true (default false)
+ * @param read_timeout http session read timeout
+ * @param retries Retry count for salt state.
  * @return output of salt command
  */
-def enforceState(master, target, state, output = true, failOnError = true, batch = null, optional = false) {
+def enforceState(master, target, state, output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1) {
     def common = new com.mirantis.mk.Common()
     def run_states
 
@@ -142,14 +144,19 @@
     }
 
     common.infoMsg("Enforcing state ${run_states} on ${target}")
-    if (optional==false){
-        def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states])
+    def out
+
+    if (optional == false || testTarget(master, target)){
+        if (retries != -1){
+            retry(retries){
+                out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states], null, -1, read_timeout)
+            }
+            }
+        else {
+            out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states], null, -1, read_timeout)
+        }
         checkResult(out, failOnError, output)
         return out
-    } else if (testTarget(master, target)) {
-        def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states])
-        checkResult(out, failOnError, output)
-        return out        
     } else {
         common.infoMsg("No Minions matched the target given, but 'optional' param was set to true - Pipeline continues. ")
     }