Retrigger salt-model tests only for timeout
Change-Id: I520a1de1d1ac07f0c494a81e4a5f47de9c512b3b
diff --git a/test-salt-model-node.groovy b/test-salt-model-node.groovy
index 8e44bfc..937d71c 100644
--- a/test-salt-model-node.groovy
+++ b/test-salt-model-node.groovy
@@ -61,7 +61,16 @@
if (checkouted) {
def workspace = common.getWorkspace()
common.infoMsg("Running salt model test for node ${NODE_TARGET} in cluster ${CLUSTER_NAME}")
- saltModelTesting.setupAndTestNode(NODE_TARGET, CLUSTER_NAME, EXTRA_FORMULAS, workspace, FORMULAS_SOURCE, FORMULAS_REVISION, MAX_CPU_PER_JOB.toInteger(), RECLASS_IGNORE_CLASS_NOTFOUND, LEGACY_TEST_MODE, APT_REPOSITORY, APT_REPOSITORY_GPG)
+ try {
+ setupAndTestNode(NODE_TARGET, CLUSTER_NAME, EXTRA_FORMULAS, workspace, FORMULAS_SOURCE, FORMULAS_REVISION, MAX_CPU_PER_JOB.toInteger(), RECLASS_IGNORE_CLASS_NOTFOUND, LEGACY_TEST_MODE, APT_REPOSITORY, APT_REPOSITORY_GPG)
+ } catch (Exception e) {
+ if (e.getMessage() == "script returned exit code 124") {
+ common.errorMsg("Impossible to test node due to timeout of salt-master, ABORTING BUILD")
+ currentBuild.result = "ABORTED"
+ } else {
+ throw e
+ }
+ }
}
}
} catch (Throwable e) {
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index fcfba0c..ca8353e 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -18,7 +18,7 @@
def ssh = new com.mirantis.mk.Ssh()
def git = new com.mirantis.mk.Git()
-def config_node_name_pattern
+def config_node_name_pattern
try {
config_node_name_pattern = CONFIG_NODE_NAME_PATTERN
} catch (MissingPropertyException e) {
@@ -50,7 +50,7 @@
def checkouted = false
futureNodes = []
-failedNodes = []
+failedNodes = false
common = new com.mirantis.mk.Common()
def setupRunner() {
@@ -58,7 +58,7 @@
def branches = [:]
for (int i = 0; i < PARALLEL_NODE_GROUP_SIZE.toInteger() && i < futureNodes.size(); i++) {
branches["Runner ${i}"] = {
- while (futureNodes) {
+ while (futureNodes && !failedNodes) {
def currentNode = futureNodes[0] ? futureNodes[0] : null
if (!currentNode) {
continue
@@ -69,13 +69,18 @@
try {
triggerTestNodeJob(currentNode[0], currentNode[1], currentNode[2], currentNode[3], currentNode[4])
} catch (Exception e) {
- failedNodes << currentNode
- common.warningMsg("Test of ${clusterName} failed : ${e}")
+ if (e.getMessage().contains("completed with status ABORTED")) {
+ common.warningMsg("Test of ${clusterName} failed because the test was aborted : ${e}")
+ futureNodes << currentNode
+ } else {
+ common.warningMsg("Test of ${clusterName} failed : ${e}")
+ failedNodes = true
+ }
}
}
}
}
- failedNodes = []
+
if (branches) {
parallel branches
}
@@ -194,16 +199,6 @@
setupRunner()
- def maxNodes = infraYMLs.size() > 10 ? infraYMLs.size() / 2 : 5
- if (failedNodes && failedNodes.size() <= maxNodes) {
- common.infoMsg("Some tests failed. They will be retriggered to make sure the failure is correct")
- for (int retry = 0; retry < 2 && failedNodes; retry++) {
- futureNodes = failedNodes
- failedNodes = []
- setupRunner()
- }
- }
-
if (failedNodes) {
currentBuild.result = "FAILURE"
}