Don't stuck when salt minion/master is restarted
When applying salt.{master|minion} states salt may be restarted.
As result existed call stuck. This patch allows to set response
timeout for salt api POST calls, and if it is reached, retry it.
Enable retries for all salt.master|salt.api calls with read_timeout=60
Change-Id: Ibdf0f5ce0bc2e4640b7502b0abe71f81557e244d
diff --git a/src/com/mirantis/mk/Http.groovy b/src/com/mirantis/mk/Http.groovy
index c8cc156..7e22ca6 100644
--- a/src/com/mirantis/mk/Http.groovy
+++ b/src/com/mirantis/mk/Http.groovy
@@ -12,10 +12,15 @@
* @param method HTTP method to use (default GET)
* @param data JSON data to POST or PUT
* @param headers Map of additional request headers
+ * @param read_timeout http session read timeout
*/
@NonCPS
-def sendHttpRequest(url, method = 'GET', data = null, headers = [:]) {
+def sendHttpRequest(url, method = 'GET', data = null, headers = [:], read_timeout=-1) {
def connection = new URL(url).openConnection()
+
+ if (read_timeout != -1){
+ connection.setReadTimeout(read_timeout*1000)
+ }
if (method != 'GET') {
connection.setRequestMethod(method)
}
@@ -80,9 +85,10 @@
*
* @param url URL which will requested
* @param data JSON data to PUT
+ * @param read_timeout http session read timeout
*/
-def sendHttpPostRequest(url, data = null, headers = [:]) {
- return sendHttpRequest(url, 'POST', data, headers)
+def sendHttpPostRequest(url, data = null, headers = [:], read_timeout=-1) {
+ return sendHttpRequest(url, 'POST', data, headers, read_timeout)
}
/**
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index fa5e8d3..f33e198 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -17,17 +17,18 @@
def installFoundationInfra(master) {
def salt = new com.mirantis.mk.Salt()
- salt.enforceState(master, 'I@salt:master', ['salt.master', 'reclass'], true)
+ salt.enforceState(master, 'I@salt:master', ['salt.master', 'reclass'], true, false, null, false, 60, 2)
salt.runSaltProcessStep(master, '*', 'saltutil.refresh_pillar', [], null, true)
salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
salt.enforceState(master, 'I@salt:master', ['linux.system'], true)
+ salt.enforceState(master, 'I@salt:master', ['salt.minion'], true, false, null, false, 60, 2)
salt.enforceState(master, 'I@salt:master', ['salt.minion'], true)
salt.enforceState(master, '*', ['linux.system'], true)
- salt.enforceState(master, '*', ['salt.minion'], true)
- salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
+ salt.enforceState(master, '*', ['salt.minion'], true, false, null, false, 60, 2)
+ salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'ntp'], true)
}
def installInfraKvm(master) {
@@ -36,7 +37,8 @@
salt.runSaltProcessStep(master, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
salt.runSaltProcessStep(master, 'I@linux:system', 'saltutil.sync_all', [], null, true)
- salt.enforceState(master, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
+ salt.enforceState(master, 'I@salt:control' ['salt.minion'], true, false, null, false, 60, 2)
+ salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp'], true)
salt.enforceState(master, 'I@salt:control', 'libvirt', true)
salt.enforceState(master, 'I@salt:control', 'salt.control', true)
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index c040fb5..d7a962a 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -54,9 +54,10 @@
* @param args Additional arguments to function
* @param kwargs Additional key-value arguments to function
* @param timeout Additional argument salt api timeout
+ * @param read_timeout http session read timeout
*/
@NonCPS
-def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null, timeout = -1) {
+def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null, timeout = -1, read_timeout = -1) {
def http = new com.mirantis.mk.Http()
data = [
@@ -87,7 +88,7 @@
'X-Auth-Token': "${master.authToken}"
]
- return http.sendHttpPostRequest("${master.url}/", data, headers)
+ return http.sendHttpPostRequest("${master.url}/", data, headers, read_timeout)
}
/**
@@ -128,10 +129,11 @@
* @param output print output (optional, default true)
* @param failOnError throw exception on salt state result:false (optional, default true)
* @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
- * @param optional don't fail on empty response from salt caused by 'No minions matched the targed' if set to true (default false)
+ * @param read_timeout http session read timeout
+ * @param retries Retry count for salt state.
* @return output of salt command
*/
-def enforceState(master, target, state, output = true, failOnError = true, batch = null, optional = false) {
+def enforceState(master, target, state, output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1) {
def common = new com.mirantis.mk.Common()
def run_states
@@ -142,14 +144,19 @@
}
common.infoMsg("Enforcing state ${run_states} on ${target}")
- if (optional==false){
- def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states])
+ def out
+
+ if (optional == false || testTarget(master, target)){
+ if (retries != -1){
+ retry(retries){
+ out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states], null, -1, read_timeout)
+ }
+ }
+ else {
+ out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states], null, -1, read_timeout)
+ }
checkResult(out, failOnError, output)
return out
- } else if (testTarget(master, target)) {
- def out = runSaltCommand(master, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, [run_states])
- checkResult(out, failOnError, output)
- return out
} else {
common.infoMsg("No Minions matched the target given, but 'optional' param was set to true - Pipeline continues. ")
}