Enabled parallel testing of nodes in salt models test pipeline, refactoring
Change-Id: I46e22ee9a045bba5f2671762f69b3b12da25d0c9
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index f7a6101..9a641f8 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -29,62 +29,6 @@
def checkouted = false
def merged = false
-def testMinion(minion)
-{
- sh("service salt-master restart && service salt-minion restart && sleep 5 && bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && verify_salt_minion ${minion}'")
-}
-
-
-
-def setupandtest(master) {
- def img = docker.image("ubuntu:latest")
- def saltOpts = "--retcode-passthrough --force-color"
- def common = new com.mirantis.mk.Common()
- def workspace = common.getWorkspace()
-
-// -v ${workspace}:/srv/salt/reclass
- img.inside("-u root:root --hostname=${master}") {
- wrap([$class: 'AnsiColorBuildWrapper']) {
-
- sh("mkdir -p /srv/salt/ || true")
- sh("cp -r ${workspace} /srv/salt/reclass")
- sh("apt-get update && apt-get install -y curl subversion git python-pip sudo python-pip python-dev zlib1g-dev git")
- sh("svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts")
- sh("git config --global user.email || git config --global user.email 'ci@ci.local'")
- sh("git config --global user.name || git config --global user.name 'CI'")
- sh("pip install git+https://github.com/epcim/reclass.git@pr/fix/fix_raise_UndefinedVariableError")
- sh("ls -lRa /srv/salt/reclass")
-
- // setup iniot and verify salt master and minions
- withEnv(["FORMULAS_SOURCE=pkg", "DEBUG=1", "MASTER_HOSTNAME=${master}", "MINION_ID=${master}", "HOSTNAME=cfg01", "DOMAIN=mk-ci.local"]){
- sh("bash -c 'echo $MASTER_HOSTNAME'")
- sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && system_config'")
- sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_bootstrap'")
- sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_init'")
- }
-
- sh("ls -lRa /srv/salt/reclass/classes/service/")
-
- def nodes
- if (DEFAULT_GIT_URL.contains("mk-ci")) {
- nodes = sh script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
- } else {
- nodes = sh script:"find /srv/salt/reclass/nodes/_generated -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
- }
- for (minion in nodes.tokenize()) {
- def basename = sh script: "basename ${minion} .yml", returnStdout: true
- if (!basename.trim().contains(master)) {
- testMinion(basename.trim())
- }
- }
-
- }
-
- }
-
- return 0
-}
-
node("python&&docker") {
try{
stage("checkout") {
@@ -117,19 +61,21 @@
}
}
- def nodes
- dir ('nodes') {
- nodes = sh script: "find ./ -type f -name 'cfg*.yml'", returnStdout: true
- }
-
stage("test-nodes") {
- def buildSteps = [:]
- for (masterNode in nodes.tokenize()) {
- def basename = sh script: "basename ${masterNode} .yml", returnStdout: true
- buildSteps[basename.trim()] = { setupandtest(basename.trim()) }
+ dir ('nodes') {
+ def nodes = sh script: "find ./ -type f -name 'cfg*.yml'", returnStdout: true
+ def buildSteps = [:]
+ def partitions = common.partitionList(nodes.tokenize(), 3)
+ for (int i=0; i< partitions.size();i++) {
+ def partition = partitions[i]
+ buildSteps.put("partition-${i}", new ArrayList<org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
+ for(int k=0; k < partition.size;k++){
+ def basename = sh script: "basename ${partition[k]} .yml", returnStdout: true
+ buildSteps.get("partition-${i}").add({ setupAndTestNode(basename.trim()) })
+ }
+ }
+ common.serial(buildSteps)
}
- //parallel buildSteps
- common.serial(buildSteps)
}
} catch (Throwable e) {
@@ -140,3 +86,50 @@
common.sendNotification(currentBuild.result,"",["slack"])
}
}
+
+def setupAndTestNode(masterName) {
+ def img = docker.image("ubuntu:latest")
+ def saltOpts = "--retcode-passthrough --force-color"
+ def common = new com.mirantis.mk.Common()
+ def workspace = common.getWorkspace()
+
+ img.inside("-u root:root --hostname=${masterName}") {
+ wrap([$class: 'AnsiColorBuildWrapper']) {
+ sh("mkdir -p /srv/salt/ || true")
+ sh("cp -r ${workspace} /srv/salt/reclass")
+ sh("apt-get update && apt-get install -y curl subversion git python-pip sudo python-pip python-dev zlib1g-dev git")
+ sh("svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts")
+ sh("git config --global user.email || git config --global user.email 'ci@ci.local'")
+ sh("git config --global user.name || git config --global user.name 'CI'")
+ sh("pip install git+https://github.com/epcim/reclass.git@pr/fix/fix_raise_UndefinedVariableError")
+ sh("ls -lRa /srv/salt/reclass")
+
+ // setup iniot and verify salt master and minions
+ withEnv(["FORMULAS_SOURCE=pkg", "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "MINION_ID=${masterName}", "HOSTNAME=cfg01", "DOMAIN=mk-ci.local"]){
+ sh("bash -c 'echo $MASTER_HOSTNAME'")
+ sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && system_config'")
+ sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_bootstrap'")
+ sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_init'")
+ }
+ sh("ls -lRa /srv/salt/reclass/classes/service/")
+
+ def nodes
+ if (DEFAULT_GIT_URL.contains("mk-ci")) {
+ nodes = sh script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
+ } else {
+ nodes = sh script:"find /srv/salt/reclass/nodes/_generated -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
+ }
+ for (minion in nodes.tokenize()) {
+ def basename = sh script: "basename ${minion} .yml", returnStdout: true
+ if (!basename.trim().contains(masterName)) {
+ testMinion(basename.trim())
+ }
+ }
+ }
+ }
+}
+
+def testMinion(minionName)
+{
+ sh("service salt-master restart && service salt-minion restart && sleep 5 && bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'")
+}
\ No newline at end of file