Merge "Fix wrong capitalization of saltMaster"
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index a9ca48f..e64abda 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -17,11 +17,13 @@
common = new com.mirantis.mk.Common()
git = new com.mirantis.mk.Git()
python = new com.mirantis.mk.Python()
+saltModelTesting = new com.mirantis.mk.SaltModelTesting()
timestamps {
- node() {
+ node("python&&docker") {
def templateEnv = "${env.WORKSPACE}/template"
def modelEnv = "${env.WORKSPACE}/model"
+ def testEnv = "${env.WORKSPACE}/test"
try {
def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
@@ -48,68 +50,20 @@
}
}
- stage('Generate base infrastructure') {
- templateDir = "${templateEnv}/cluster_product/infra"
- templateOutputDir = "${env.WORKSPACE}/template/output/infra"
- sh "mkdir -p ${templateOutputDir}"
- sh "mkdir -p ${outputDestination}"
- python.setupCookiecutterVirtualenv(cutterEnv)
- python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
- sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
- }
-
- stage('Generate product CI/CD') {
- if (templateContext.default_context.cicd_enabled.toBoolean()) {
- templateDir = "${templateEnv}/cluster_product/cicd"
- templateOutputDir = "${env.WORKSPACE}/template/output/cicd"
- sh "mkdir -p ${templateOutputDir}"
- python.setupCookiecutterVirtualenv(cutterEnv)
- python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
- sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
- }
- }
-
- stage('Generate product OpenContrail') {
- if (templateContext.default_context.opencontrail_enabled.toBoolean()) {
- templateDir = "${templateEnv}/cluster_product/opencontrail"
- templateOutputDir = "${env.WORKSPACE}/template/output/opencontrail"
- sh "mkdir -p ${templateOutputDir}"
- python.setupCookiecutterVirtualenv(cutterEnv)
- python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
- sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
- }
- }
-
- stage('Generate product Kubernetes') {
- if (templateContext.default_context.kubernetes_enabled.toBoolean()) {
- templateDir = "${templateEnv}/cluster_product/kubernetes"
- templateOutputDir = "${env.WORKSPACE}/template/output/kubernetes"
- sh "mkdir -p ${templateOutputDir}"
- python.setupCookiecutterVirtualenv(cutterEnv)
- python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
- sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
- }
- }
-
- stage('Generate product OpenStack') {
- if (templateContext.default_context.openstack_enabled.toBoolean()) {
- templateDir = "${templateEnv}/cluster_product/openstack"
- templateOutputDir = "${env.WORKSPACE}/template/output/openstack"
- sh "mkdir -p ${templateOutputDir}"
- python.setupCookiecutterVirtualenv(cutterEnv)
- python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
- sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
- }
- }
-
- stage('Generate product StackLight') {
- if (templateContext.default_context.stacklight_enabled.toBoolean()) {
- templateDir = "${templateEnv}/cluster_product/stacklight"
- templateOutputDir = "${env.WORKSPACE}/template/output/stacklight"
- sh "mkdir -p ${templateOutputDir}"
- python.setupCookiecutterVirtualenv(cutterEnv)
- python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
- sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
+ for (product in productList) {
+ def stagename = (product == "infra") ? "Generate base infrastructure" : "Generate product ${product}"
+ stage(stagename) {
+ if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+ && templateContext.default_context["${product}_enabled"].toBoolean())) {
+ templateDir = "${templateEnv}/cluster_product/${product}"
+ templateOutputDir = "${env.WORKSPACE}/template/output/${product}"
+ sh "mkdir -p ${templateOutputDir}"
+ sh "mkdir -p ${outputDestination}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
}
}
@@ -130,6 +84,15 @@
writeFile(file: nodeFile, text: nodeString)
}
+ stage("Test") {
+ if (RECLASS_MODEL_URL == "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
+ sh("cp -r ${modelEnv} ${testEnv}")
+ def defaultReclassModel = "ssh://jenkins-mk@gerrit.mcp.mirantis.net:29418/salt-models/reclass-system"
+ git.checkoutGitRepository("${testEnv}/classes/system", defaultReclassModel, RECLASS_MODEL_BRANCH, RECLASS_MODEL_CREDENTIALS)
+ saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", "", testEnv)
+ }
+ }
+
stage ('Save changes to Reclass model') {
if (env.getEnvironment().containsKey('COMMIT_CHANGES') && COMMIT_CHANGES.toBoolean() && RECLASS_MODEL_URL != null && RECLASS_MODEL_URL != "") {
git.changeGitBranch(modelEnv, targetBranch)
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index c062fac..a1e1082 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -53,11 +53,25 @@
salt = new com.mirantis.mk.Salt()
test = new com.mirantis.mk.Test()
+
+
+
_MAX_PERMITTED_STACKS = 2
overwriteFile = "/srv/salt/reclass/classes/cluster/overwrite.yml"
timestamps {
node {
+ // try to get STACK_INSTALL or fallback to INSTALL if exists
+ try {
+ def temporary = STACK_INSTALL
+ } catch (MissingPropertyException e) {
+ try {
+ STACK_INSTALL = INSTALL
+ env['STACK_INSTALL'] = INSTALL
+ } catch (MissingPropertyException e2) {
+ common.errorMsg("Property STACK_INSTALL or INSTALL not found!")
+ }
+ }
try {
//
// Prepare machines
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
new file mode 100644
index 0000000..0d8d319
--- /dev/null
+++ b/test-cookiecutter-reclass.groovy
@@ -0,0 +1,120 @@
+common = new com.mirantis.mk.Common()
+git = new com.mirantis.mk.Git()
+python = new com.mirantis.mk.Python()
+saltModelTesting = new com.mirantis.mk.SaltModelTesting()
+
+def generateSaltMaster(modelEnv) {
+ def nodeFile = "${modelEnv}/nodes/cfg01.${clusterDomain}.yml"
+ def nodeString = """classes:
+- cluster.${clusterName}.infra.config
+parameters:
+ _param:
+ linux_system_codename: xenial
+ reclass_data_revision: master
+ linux:
+ system:
+ name: cfg01
+ domain: ${clusterDomain}
+"""
+ sh "mkdir -p ${modelEnv}/nodes/"
+ writeFile(file: nodeFile, text: nodeString)
+}
+
+def generate(contextFile) {
+ def templateEnv = "${env.WORKSPACE}/template"
+ def baseName = sh(script: "basename ${contextFile} .yml", returnStdout: true)
+ def modelEnv = "${env.WORKSPACE}/model-${baseName}"
+ def cookiecutterTemplateContext = readFile(file: "${env.WORKSPACE}/contexts/contextFile")
+ def templateContext = readYaml text: cookiecutterTemplateContext
+ def clusterDomain = templateContext.default_context.cluster_domain
+ def clusterName = templateContext.default_context.cluster_name
+ def cutterEnv = "${env.WORKSPACE}/cutter"
+ def jinjaEnv = "${env.WORKSPACE}/jinja"
+ def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
+ def targetBranch = "feature/${clusterName}"
+ def templateBaseDir = "${env.WORKSPACE}/template"
+ def templateDir = "${templateEnv}/template/dir"
+ def templateOutputDir = templateBaseDir
+ sh("rm -rf ${templateBaseDir} || true")
+
+ def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
+ for (product in productList) {
+ def stagename = (product == "infra") ? "Generate base infrastructure" : "Generate product ${product}"
+ println stagename
+ if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+ && templateContext.default_context["${product}_enabled"].toBoolean())) {
+ templateDir = "${templateEnv}/cluster_product/${product}"
+ templateOutputDir = "${env.WORKSPACE}/template/output/${product}"
+ sh "mkdir -p ${templateOutputDir}"
+ sh "mkdir -p ${outputDestination}"
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, cookiecutterTemplateContext, templateOutputDir, cutterEnv, templateBaseDir)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ }
+ }
+ generateSaltMaster(modelEnv)
+}
+
+def testModel(contextFile) {
+ def baseName = sh(script: "basename ${contextFile} .yml", returnStdout: true)
+ def modelEnv = "${env.WORKSPACE}/model-${baseName}"
+ git.checkoutGitRepository("${modelEnv}/classes/system", RECLASS_MODEL_URL, RECLASS_MODEL_BRANCH, RECLASS_MODEL_CREDENTIALS)
+ saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", "", modelEnv)
+}
+
+timestamps {
+ node("python&&docker") {
+ def templateEnv = "${env.WORKSPACE}/template"
+
+ try {
+ stage ('Download Cookiecutter template') {
+ if (gerritRef) {
+ def gerritChange = gerrit.getGerritChange(GERRIT_NAME, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID)
+ merged = gerritChange.status == "MERGED"
+ if(!merged){
+ checkouted = gerrit.gerritPatchsetCheckout ([
+ credentialsId : CREDENTIALS_ID
+ ])
+ } else{
+ common.successMsg("Change ${GERRIT_CHANGE_NUMBER} is already merged, no need to gate them")
+ }
+ } else {
+ gerrit.gerritPatchsetCheckout(COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, "HEAD", CREDENTIALS_ID)
+ }
+ }
+
+ def contextFiles
+ dir("contexts") {
+ contextFiles = findFiles(glob: "*.yml")
+ }
+
+ for (contextFile in contextFiles) {
+ generate(contextFile)
+ }
+
+ stage("test-nodes") {
+ def partitions = common.partitionList(contextFiles, 3)
+ def buildSteps = [:]
+ for (int i = 0; i < partitions.size(); i++) {
+ def partition = partitions[i]
+ buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
+ for(int k = 0; k < partition.size; k++){
+ def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
+ def modelEnv = "${env.WORKSPACE}/model-${baseName}"
+ buildSteps.get("partition-${i}").put(basename, { saltModelTesting.setupAndTestNode(basename, "", modelEnv) })
+ }
+ }
+ common.serial(buildSteps)
+ }
+
+ } catch (Throwable e) {
+ currentBuild.result = "FAILURE"
+ throw e
+ } finally {
+ stage ('Clean workspace directories') {
+ sh(returnStatus: true, script: "rm -rfv *")
+ }
+ common.sendNotification(currentBuild.result,"",["slack"])
+ }
+ }
+}
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index 0936113..3ed8f61 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -11,6 +11,7 @@
def gerrit = new com.mirantis.mk.Gerrit()
def ssh = new com.mirantis.mk.Ssh()
def git = new com.mirantis.mk.Git()
+def saltModelTesting = new com.mirantis.mk.SaltModelTesting()
def gerritRef
try {
@@ -71,6 +72,7 @@
}
stage("test-nodes") {
+ def workspace = common.getWorkspace()
def nodes = sh(script: "find ./nodes -type f -name 'cfg*.yml'", returnStdout: true).tokenize()
def buildSteps = [:]
if(nodes.size() > 1){
@@ -78,7 +80,7 @@
common.infoMsg("Found <=3 cfg nodes, running parallel test")
for(int i=0; i < nodes.size();i++){
def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
- buildSteps.put("node-${basename}", { setupAndTestNode(basename) })
+ buildSteps.put("node-${basename}", { saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace) })
}
parallel buildSteps
}else{
@@ -89,7 +91,7 @@
buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
for(int k=0; k < partition.size;k++){
def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
- buildSteps.get("partition-${i}").put(basename, { setupAndTestNode(basename) })
+ buildSteps.get("partition-${i}").put(basename, { saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace) })
}
}
common.serial(buildSteps)
@@ -97,7 +99,7 @@
}else{
common.infoMsg("Found one cfg node, running single test")
def basename = sh(script: "basename ${nodes[0]} .yml", returnStdout: true).trim()
- setupAndTestNode(basename)
+ saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace)
}
}
@@ -110,61 +112,6 @@
}
}
-def setupAndTestNode(masterName) {
- def saltOpts = "--retcode-passthrough --force-color"
- def common = new com.mirantis.mk.Common()
- def workspace = common.getWorkspace()
- def imageFound = true
- def img
- try {
- img = docker.image("tcpcloud/salt-models-testing")
- img.pull()
- } catch (Throwable e) {
- img = docker.image("ubuntu:latest")
- imageFound = false
- }
-
- img.inside("-u root:root --hostname=${masterName}") {
- wrap([$class: 'AnsiColorBuildWrapper']) {
- if (!imageFound) {
- sh("apt-get update && apt-get install -y curl subversion git python-pip sudo python-pip python-dev zlib1g-dev git")
- sh("pip install git+https://github.com/epcim/reclass.git@pr/fix/fix_raise_UndefinedVariableError")
- }
- sh("mkdir -p /srv/salt/ || true")
- sh("cp -r ${workspace} /srv/salt/reclass")
- sh("svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts")
- sh("git config --global user.email || git config --global user.email 'ci@ci.local'")
- sh("git config --global user.name || git config --global user.name 'CI'")
-
- // setup iniot and verify salt master and minions
- withEnv(["FORMULAS_SOURCE=pkg", "EXTRA_FORMULAS=${EXTRA_FORMULAS}", "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "MINION_ID=${masterName}", "HOSTNAME=cfg01", "DOMAIN=mk-ci.local"]){
- sh("bash -c 'echo $MASTER_HOSTNAME'")
- sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && system_config'")
- sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_bootstrap'")
- sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_init'")
- }
-
- def nodes
- if (DEFAULT_GIT_URL.contains("mk-ci")) {
- nodes = sh script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
- } else {
- nodes = sh script:"find /srv/salt/reclass/nodes/_generated -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
- }
- for (minion in nodes.tokenize()) {
- def basename = sh script: "basename ${minion} .yml", returnStdout: true
- if (!basename.trim().contains(masterName)) {
- testMinion(basename.trim())
- }
- }
- }
- }
-}
-
-def testMinion(minionName)
-{
- sh("service salt-master restart && service salt-minion restart && sleep 5 && bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'")
-}
-
@NonCPS
def _getRunningTriggeredTestsBuildNumbers(jobName, gerritChangeNumber, excludePatchsetNumber){
def gerrit = new com.mirantis.mk.Gerrit()