Make runPepper more resistant against SaltReqTimeoutError
* In rare condition, salt zmq threads may flap with raise
SaltReqTimeoutError. During those period, salt-api may return
50X answer and drop whole deployment process.
Unfortunatly, in salt no option to increase SaltReqTimeoutError
for all threads, (default 3 for many different threads).
Salt itself allow to configure only few *_tries option for exact
threads(like auth or job render).
* Those patch add simply crutch with retry, in case 50X error
has been detected in stderr from pepper call
* Misc:
- Extend mk.Python.runVirtualenvCommand
- Extend mk.Common.runPepperCommand
- add failover for 50X and retry
- Refactor mirantis.mk.Common.shCmdStatus
Prod-relaeted: PROD-30839(PROD:30839)
Change-Id: I18b152c5f22c8fb602a21a34ea06a4c543d8ae26
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index d0f3598..b5760da 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -505,18 +505,19 @@
*/
def shCmdStatus(cmd) {
+ // Set +x , to hide odd messages about temp file manipulations
def res = [:]
- def stderr = sh(script: 'mktemp', returnStdout: true).trim()
- def stdout = sh(script: 'mktemp', returnStdout: true).trim()
+ def stderr = sh(script: 'set +x ; mktemp', returnStdout: true).trim()
+ def stdout = sh(script: 'set +x ; mktemp', returnStdout: true).trim()
try {
def status = sh(script: "${cmd} 1>${stdout} 2>${stderr}", returnStatus: true)
- res['stderr'] = sh(script: "cat ${stderr}", returnStdout: true)
- res['stdout'] = sh(script: "cat ${stdout}", returnStdout: true)
+ res['stderr'] = sh(script: "set +x; cat ${stderr}", returnStdout: true).trim()
+ res['stdout'] = sh(script: "set +x; cat ${stdout}", returnStdout: true).trim()
res['status'] = status
} finally {
- sh(script: "rm ${stderr}", returnStdout: true)
- sh(script: "rm ${stdout}", returnStdout: true)
+ sh(script: "set +x; rm ${stderr}")
+ sh(script: "set +x; rm ${stdout}")
}
return res
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index c326bf6..8d9d19b 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -53,22 +53,27 @@
/**
* Run command in specific python virtualenv
*
- * @param path Path to virtualenv
- * @param cmd Command to be executed
+ * @param path Path to virtualenv
+ * @param cmd Command to be executed
* @param silent dont print any messages (optional, default false)
+ * @param flexAnswer return answer like a dict, with format ['status' : int, 'stderr' : str, 'stdout' : str ]
*/
-def runVirtualenvCommand(path, cmd, silent = false) {
+def runVirtualenvCommand(path, cmd, silent = false, flexAnswer = false) {
def common = new com.mirantis.mk.Common()
-
- virtualenv_cmd = "set +x; . ${path}/bin/activate; ${cmd}"
+ def res
+ def virtualenv_cmd = "set +x; . ${path}/bin/activate; ${cmd}"
if (!silent) {
common.infoMsg("[Python ${path}] Run command ${cmd}")
}
- output = sh(
- returnStdout: true,
- script: virtualenv_cmd
- ).trim()
- return output
+ if (flexAnswer) {
+ res = common.shCmdStatus(virtualenv_cmd)
+ } else {
+ res = sh(
+ returnStdout: true,
+ script: virtualenv_cmd
+ ).trim()
+ }
+ return res
}
/**
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index f3a5493..21bb4c9 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -1125,35 +1125,53 @@
* @param venv Path to virtualenv with
*/
-def runPepperCommand(data, venv) {
+def runPepperCommand(data, venv) {
def common = new com.mirantis.mk.Common()
def python = new com.mirantis.mk.Python()
def dataStr = new groovy.json.JsonBuilder(data).toString()
+ // TODO(alexz): parametrize?
+ int retry = 10
def pepperCmdFile = "${venv}/pepper-cmd.json"
writeFile file: pepperCmdFile, text: dataStr
def pepperCmd = "pepper -c ${venv}/pepperrc --make-token -x ${venv}/.peppercache --json-file ${pepperCmdFile}"
- if (venv) {
- output = python.runVirtualenvCommand(venv, pepperCmd, true)
- } else {
- echo("[Command]: ${pepperCmd}")
- output = sh (
- script: pepperCmd,
- returnStdout: true
- ).trim()
- }
-
+ int tries = 0
+ def FullOutput = ['status': 1]
def outputObj
+ while (tries++ < retry) {
+ try {
+ if (venv) {
+ FullOutput = python.runVirtualenvCommand(venv, pepperCmd, true, true)
+ } else {
+ FullOutput = common.shCmdStatus(pepperCmd)
+ }
+ if (FullOutput['status'] != 0) {
+ error()
+ }
+ break
+ } catch (e) {
+ // Check , if we get failed pepper HTTP call, and retry
+ common.errorMsg("Command: ${pepperCmd} failed to execute with error:\n${FullOutput['stderr']}")
+ if (FullOutput['stderr'].contains('Error with request: HTTP Error 50') || FullOutput['stderr'].contains('Pepper error: Server error')) {
+ common.errorMsg("Pepper HTTP Error detected. Most probably, " +
+ "master SaltReqTimeoutError in master zmq thread issue...lets retry ${tries}/${retry}")
+ sleep(5)
+ continue
+ }
+ }
+ }
+ // Try to parse json output. No sense to check exit code, since we always expect json answer only.
try {
- outputObj = new groovy.json.JsonSlurperClassic().parseText(output)
- } catch(Exception e) {
- common.errorMsg("Parsing Salt API JSON response failed! Response: " + output)
- throw e
+ outputObj = new groovy.json.JsonSlurperClassic().parseText(FullOutput['stdout'])
+ } catch (Exception jsonE) {
+ common.errorMsg('Parsing Salt API JSON response failed! Response: ' + FullOutput)
+ throw jsonE
}
return outputObj
}
+
/**
* Check time settings on defined nodes, compares them
* and evaluates the results
@@ -1288,4 +1306,4 @@
}
}
return true
-}
\ No newline at end of file
+}