Merge "Add parameters for OpenStack connections for v3 auth"
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index a9ca48f..e64abda 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -17,11 +17,13 @@
 common = new com.mirantis.mk.Common()
 git = new com.mirantis.mk.Git()
 python = new com.mirantis.mk.Python()
+saltModelTesting = new com.mirantis.mk.SaltModelTesting()
 
 timestamps {
-    node() {
+    node("python&&docker") {
         def templateEnv = "${env.WORKSPACE}/template"
         def modelEnv = "${env.WORKSPACE}/model"
+        def testEnv = "${env.WORKSPACE}/test"
 
         try {
             def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
@@ -48,68 +50,20 @@
                 }
             }
 
-            stage('Generate base infrastructure') {
-                templateDir = "${templateEnv}/cluster_product/infra"
-                templateOutputDir = "${env.WORKSPACE}/template/output/infra"
-                sh "mkdir -p ${templateOutputDir}"
-                sh "mkdir -p ${outputDestination}"
-                python.setupCookiecutterVirtualenv(cutterEnv)
-                python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-            }
-
-            stage('Generate product CI/CD') {
-                if (templateContext.default_context.cicd_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/cicd"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/cicd"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product OpenContrail') {
-                if (templateContext.default_context.opencontrail_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/opencontrail"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/opencontrail"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product Kubernetes') {
-                if (templateContext.default_context.kubernetes_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/kubernetes"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/kubernetes"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product OpenStack') {
-                if (templateContext.default_context.openstack_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/openstack"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/openstack"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                }
-            }
-
-            stage('Generate product StackLight') {
-                if (templateContext.default_context.stacklight_enabled.toBoolean()) {
-                    templateDir = "${templateEnv}/cluster_product/stacklight"
-                    templateOutputDir = "${env.WORKSPACE}/template/output/stacklight"
-                    sh "mkdir -p ${templateOutputDir}"
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+            def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
+            for (product in productList) {
+                def stagename = (product == "infra") ? "Generate base infrastructure" : "Generate product ${product}"
+                stage(stagename) {
+                    if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+                        && templateContext.default_context["${product}_enabled"].toBoolean())) {
+                        templateDir = "${templateEnv}/cluster_product/${product}"
+                        templateOutputDir = "${env.WORKSPACE}/template/output/${product}"
+                        sh "mkdir -p ${templateOutputDir}"
+                        sh "mkdir -p ${outputDestination}"
+                        python.setupCookiecutterVirtualenv(cutterEnv)
+                        python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
+                        sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+                    }
                 }
             }
 
@@ -130,6 +84,15 @@
                 writeFile(file: nodeFile, text: nodeString)
             }
 
+            stage("Test") {
+                if (RECLASS_MODEL_URL == "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
+                    sh("cp -r ${modelEnv} ${testEnv}")
+                    def defaultReclassModel = "ssh://jenkins-mk@gerrit.mcp.mirantis.net:29418/salt-models/reclass-system"
+                    git.checkoutGitRepository("${testEnv}/classes/system", defaultReclassModel, RECLASS_MODEL_BRANCH, RECLASS_MODEL_CREDENTIALS)
+                    saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", "", testEnv)
+                }
+            }
+
             stage ('Save changes to Reclass model') {
                 if (env.getEnvironment().containsKey('COMMIT_CHANGES') && COMMIT_CHANGES.toBoolean() && RECLASS_MODEL_URL != null && RECLASS_MODEL_URL != "") {
                     git.changeGitBranch(modelEnv, targetBranch)
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index 6848f3f..1b7facb 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -56,11 +56,25 @@
 salt = new com.mirantis.mk.Salt()
 test = new com.mirantis.mk.Test()
 
+
+
+
 _MAX_PERMITTED_STACKS = 2
 overwriteFile = "/srv/salt/reclass/classes/cluster/overwrite.yml"
 
 timestamps {
     node {
+        // try to get STACK_INSTALL or fallback to INSTALL if exists
+        try {
+          def temporary = STACK_INSTALL
+        } catch (MissingPropertyException e) {
+          try {
+            STACK_INSTALL = INSTALL
+            env['STACK_INSTALL'] = INSTALL
+          } catch (MissingPropertyException e2) {
+            common.errorMsg("Property STACK_INSTALL or INSTALL not found!")
+          }
+        }
         try {
             //
             // Prepare machines
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
new file mode 100644
index 0000000..b984d53
--- /dev/null
+++ b/test-cookiecutter-reclass.groovy
@@ -0,0 +1,151 @@
+common = new com.mirantis.mk.Common()
+gerrit = new com.mirantis.mk.Gerrit()
+git = new com.mirantis.mk.Git()
+python = new com.mirantis.mk.Python()
+saltModelTesting = new com.mirantis.mk.SaltModelTesting()
+
+def generateSaltMaster(modelEnv, clusterDomain, clusterName) {
+    def nodeFile = "${modelEnv}/nodes/cfg01.${clusterDomain}.yml"
+    def nodeString = """classes:
+- cluster.${clusterName}.infra.config
+parameters:
+    _param:
+        linux_system_codename: xenial
+        reclass_data_revision: master
+    linux:
+        system:
+            name: cfg01
+            domain: ${clusterDomain}
+"""
+    sh "mkdir -p ${modelEnv}/nodes/"
+    writeFile(file: nodeFile, text: nodeString)
+}
+
+def generateModel(contextFile, cutterEnv) {
+    def templateEnv = "${env.WORKSPACE}"
+    def modelEnv = "${env.WORKSPACE}/model"
+    def basename = sh(script: "basename ${contextFile} .yml", returnStdout: true).trim()
+    def generatedModel = "${modelEnv}/${basename}"
+    def testEnv = "${env.WORKSPACE}/test"
+    def content = readFile(file: "${templateEnv}/contexts/${contextFile}")
+    def templateContext = readYaml text: content
+    def clusterDomain = templateContext.default_context.cluster_domain
+    def clusterName = templateContext.default_context.cluster_name
+    def outputDestination = "${generatedModel}/classes/cluster/${clusterName}"
+    def targetBranch = "feature/${clusterName}"
+    def templateBaseDir = "${env.WORKSPACE}"
+    def templateDir = "${templateEnv}/dir"
+    def templateOutputDir = templateBaseDir
+    sh "rm -rf ${generatedModel} || true"
+
+    stage("Generate model from ${contextFile}") {
+        def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
+        for (product in productList) {
+            def stagename = (product == "infra") ? "Generate base infrastructure" : "Generate product ${product}"
+            if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+                && templateContext.default_context["${product}_enabled"].toBoolean())) {
+                templateDir = "${templateEnv}/cluster_product/${product}"
+                templateOutputDir = "${env.WORKSPACE}/output/${product}"
+                sh "rm -rf ${templateOutputDir} || true"
+                sh "mkdir -p ${templateOutputDir}"
+                sh "mkdir -p ${outputDestination}"
+                python.buildCookiecutterTemplate(templateDir, content, templateOutputDir, cutterEnv, templateBaseDir)
+                sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+            }
+        }
+        generateSaltMaster(generatedModel, clusterDomain, clusterName)
+    }
+}
+
+def testModel(contextFile, testEnv) {
+    def templateEnv = "${env.WORKSPACE}"
+    def content = readFile(file: "${templateEnv}/contexts/${contextFile}.yml")
+    def templateContext = readYaml text: content
+    def clusterDomain = templateContext.default_context.cluster_domain
+    git.checkoutGitRepository("${testEnv}/classes/system", RECLASS_MODEL_URL, RECLASS_MODEL_BRANCH, CREDENTIALS_ID)
+    saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", "", testEnv)
+}
+
+def gerritRef
+try {
+  gerritRef = GERRIT_REFSPEC
+} catch (MissingPropertyException e) {
+  gerritRef = null
+}
+
+timestamps {
+    node("python&&docker") {
+        def templateEnv = "${env.WORKSPACE}"
+        def cutterEnv = "${env.WORKSPACE}/cutter"
+        def jinjaEnv = "${env.WORKSPACE}/jinja"
+
+        try {
+            stage("Cleanup") {
+                sh("rm -rf * || true")
+            }
+
+            stage ('Download Cookiecutter template') {
+                if (gerritRef) {
+                    def gerritChange = gerrit.getGerritChange(GERRIT_NAME, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID)
+                    merged = gerritChange.status == "MERGED"
+                    if(!merged){
+                        checkouted = gerrit.gerritPatchsetCheckout ([
+                            credentialsId : CREDENTIALS_ID
+                        ])
+                    } else{
+                        common.successMsg("Change ${GERRIT_CHANGE_NUMBER} is already merged, no need to gate them")
+                    }
+                } else {
+                    git.checkoutGitRepository(templateEnv, "ssh://jenkins-mk@gerrit.mcp.mirantis.net:29418/mk/cookiecutter-templates", COOKIECUTTER_TEMPLATE_BRANCH, CREDENTIALS_ID)
+                }
+            }
+
+            stage("Setup") {
+                python.setupCookiecutterVirtualenv(cutterEnv)
+            }
+
+            def contextFiles
+            dir("${templateEnv}/contexts") {
+                contextFiles = findFiles(glob: "*.yml")
+            }
+
+            def contextFileList = []
+            for (int i = 0; i < contextFiles.size(); i++) {
+                //generateModel(contextFiles[i], cutterEnv)
+                contextFileList << contextFiles[i]
+            }
+
+            stage("generate-model") {
+                def buildSteps = [:]
+                for (contextFile in contextFileList) {
+                    buildSteps[contextFile] = { generateModel(contextFile, cutterEnv) }
+                }
+                common.serial(buildSteps)
+            }
+
+            stage("test-nodes") {
+                def partitions = common.partitionList(contextFileList, 3)
+                def buildSteps = [:]
+                for (int i = 0; i < partitions.size(); i++) {
+                    def partition = partitions[i]
+                    buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
+                    for(int k = 0; k < partition.size; k++){
+                        def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
+                        def testEnv = "${env.WORKSPACE}/model/${basename}"
+                        buildSteps.get("partition-${i}").put(basename, { testModel(basename, testEnv) })
+                    }
+                }
+                common.serial(buildSteps)
+            }
+
+        } catch (Throwable e) {
+             currentBuild.result = "FAILURE"
+             throw e
+        } finally {
+            stage ('Clean workspace directories') {
+                sh(returnStatus: true, script: "rm -rfv *")
+            }
+            common.sendNotification(currentBuild.result,"",["slack"])
+        }
+    }
+}
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index 0936113..3ed8f61 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -11,6 +11,7 @@
 def gerrit = new com.mirantis.mk.Gerrit()
 def ssh = new com.mirantis.mk.Ssh()
 def git = new com.mirantis.mk.Git()
+def saltModelTesting = new com.mirantis.mk.SaltModelTesting()
 
 def gerritRef
 try {
@@ -71,6 +72,7 @@
     }
 
     stage("test-nodes") {
+      def workspace = common.getWorkspace()
       def nodes = sh(script: "find ./nodes -type f -name 'cfg*.yml'", returnStdout: true).tokenize()
       def buildSteps = [:]
       if(nodes.size() > 1){
@@ -78,7 +80,7 @@
             common.infoMsg("Found <=3  cfg nodes, running parallel test")
              for(int i=0; i < nodes.size();i++){
                def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
-               buildSteps.put("node-${basename}", { setupAndTestNode(basename) })
+               buildSteps.put("node-${basename}", { saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace) })
              }
              parallel buildSteps
           }else{
@@ -89,7 +91,7 @@
               buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
               for(int k=0; k < partition.size;k++){
                   def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
-                  buildSteps.get("partition-${i}").put(basename, { setupAndTestNode(basename) })
+                  buildSteps.get("partition-${i}").put(basename, { saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace) })
               }
             }
             common.serial(buildSteps)
@@ -97,7 +99,7 @@
       }else{
           common.infoMsg("Found one cfg node, running single test")
           def basename = sh(script: "basename ${nodes[0]} .yml", returnStdout: true).trim()
-          setupAndTestNode(basename)
+          saltModelTesting.setupAndTestNode(basename, EXTRA_FORMULAS, workspace)
       }
     }
 
@@ -110,61 +112,6 @@
   }
 }
 
-def setupAndTestNode(masterName) {
-  def saltOpts = "--retcode-passthrough --force-color"
-  def common = new com.mirantis.mk.Common()
-  def workspace = common.getWorkspace()
-  def imageFound = true
-  def img
-  try {
-    img = docker.image("tcpcloud/salt-models-testing")
-    img.pull()
-  } catch (Throwable e) {
-    img = docker.image("ubuntu:latest")
-    imageFound = false
-  }
-
-  img.inside("-u root:root --hostname=${masterName}") {
-    wrap([$class: 'AnsiColorBuildWrapper']) {
-      if (!imageFound) {
-        sh("apt-get update && apt-get install -y curl subversion git python-pip sudo python-pip python-dev zlib1g-dev git")
-        sh("pip install git+https://github.com/epcim/reclass.git@pr/fix/fix_raise_UndefinedVariableError")
-      }
-      sh("mkdir -p /srv/salt/ || true")
-      sh("cp -r ${workspace} /srv/salt/reclass")
-      sh("svn export --force https://github.com/salt-formulas/salt-formulas/trunk/deploy/scripts /srv/salt/scripts")
-      sh("git config --global user.email || git config --global user.email 'ci@ci.local'")
-      sh("git config --global user.name || git config --global user.name 'CI'")
-
-      // setup iniot and verify salt master and minions
-      withEnv(["FORMULAS_SOURCE=pkg", "EXTRA_FORMULAS=${EXTRA_FORMULAS}", "DEBUG=1", "MASTER_HOSTNAME=${masterName}", "MINION_ID=${masterName}", "HOSTNAME=cfg01", "DOMAIN=mk-ci.local"]){
-          sh("bash -c 'echo $MASTER_HOSTNAME'")
-          sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && system_config'")
-          sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_bootstrap'")
-          sh("bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && saltmaster_init'")
-      }
-
-      def nodes
-      if (DEFAULT_GIT_URL.contains("mk-ci")) {
-        nodes = sh script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
-      } else {
-        nodes = sh script:"find /srv/salt/reclass/nodes/_generated -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
-      }
-      for (minion in nodes.tokenize()) {
-        def basename = sh script: "basename ${minion} .yml", returnStdout: true
-        if (!basename.trim().contains(masterName)) {
-          testMinion(basename.trim())
-        }
-      }
-    }
-  }
-}
-
-def testMinion(minionName)
-{
-  sh("service salt-master restart && service salt-minion restart && sleep 5 && bash -c 'source /srv/salt/scripts/salt-master-init.sh; cd /srv/salt/scripts && verify_salt_minion ${minionName}'")
-}
-
 @NonCPS
 def _getRunningTriggeredTestsBuildNumbers(jobName, gerritChangeNumber, excludePatchsetNumber){
   def gerrit = new com.mirantis.mk.Gerrit()