Implement patch-diff artifacts for test-cookiecutter-reclass

   * test-cookiecutter-reclass:
     - build  in paralell 2 full-contexts - for patched
       and for head CC repos.
     - compare and publish results via diff
     - all env's|models use same system-reclass version
   * Move GetBaseName func to pipeline-lib
   * Misc patches for test-cookiecutter-reclass-chunk.groovy

Closes-Bug: PROD-22114 (PROD:22114)

Change-Id: I29ba901f07275670a62a20297bbcafc09f7720fe
diff --git a/test-cookiecutter-reclass-chunk.groovy b/test-cookiecutter-reclass-chunk.groovy
index 12428ba..9e34cea 100644
--- a/test-cookiecutter-reclass-chunk.groovy
+++ b/test-cookiecutter-reclass-chunk.groovy
@@ -1,23 +1,27 @@
 package com.mirantis.mk
+
 def common = new com.mirantis.mk.Common()
 def saltModelTesting = new com.mirantis.mk.SaltModelTesting()
 
 /**
  * Test CC model wrapper
  *  EXTRA_VARIABLES_YAML: yaml based string, to be directly passed into testCCModel
+ *  SLAVE_NODE:
  */
 
+slaveNode = env.SLAVE_NODE ?: 'python&&docker'
+
 timeout(time: 1, unit: 'HOURS') {
-node() {
-  try {
-    extra_vars = readYaml text: EXTRA_VARIABLES_YAML
-    currentBuild.description = extra_vars.modelFile
-    saltModelTesting.testCCModel(extra_vars)
+  node(slaveNode) {
+    try {
+      extraVars = readYaml text: EXTRA_VARIABLES_YAML
+      currentBuild.description = extraVars.modelFile
+      saltModelTesting.testCCModel(extraVars)
     } catch (Throwable e) {
-          // If there was an error or exception thrown, the build failed
-          currentBuild.result = "FAILURE"
-          currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
-          throw e
-        }
-      }
+      // If there was an error or exception thrown, the build failed
+      currentBuild.result = "FAILURE"
+      currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+      throw e
     }
+  }
+}
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index 2a64990..ac5bc1f 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -2,17 +2,19 @@
 gerrit = new com.mirantis.mk.Gerrit()
 git = new com.mirantis.mk.Git()
 python = new com.mirantis.mk.Python()
-saltModelTesting = new com.mirantis.mk.SaltModelTesting()
 
-slave_node = 'python&&docker'
+gerritRef = env.GERRIT_REFSPEC ?: null
+slaveNode = (env.SLAVE_NODE ?: 'python&&docker')
+def alreadyMerged = false
+
 def reclassVersion = 'v1.5.4'
 if (common.validInputParam('RECLASS_VERSION')) {
-  reclassVersion = RECLASS_VERSION
+    reclassVersion = RECLASS_VERSION
 }
 
 def generateSaltMaster(modEnv, clusterDomain, clusterName) {
-  def nodeFile = "${modEnv}/nodes/cfg01.${clusterDomain}.yml"
-  def nodeString = """classes:
+    def nodeFile = "${modEnv}/nodes/cfg01.${clusterDomain}.yml"
+    def nodeString = """classes:
 - cluster.${clusterName}.infra.config
 parameters:
     _param:
@@ -23,82 +25,70 @@
             name: cfg01
             domain: ${clusterDomain}
 """
-  sh "mkdir -p ${modEnv}/nodes/"
-  println "Create file ${nodeFile}"
-  writeFile(file: nodeFile, text: nodeString)
+    sh "mkdir -p ${modEnv}/nodes/"
+    println "Create file ${nodeFile}"
+    writeFile(file: nodeFile, text: nodeString)
 }
 
-def GetBaseName(line, remove_ext) {
- filename = line.toString().split('/').last()
- if (remove_ext && filename.endsWith(remove_ext.toString())) {
-   filename = filename.take(filename.lastIndexOf(remove_ext.toString()))
- }
- return filename
-}
+def generateModel(contextFile, virtualenv, templateEnvDir) {
+    def modelEnv = "${templateEnvDir}/model"
+    def basename = common.GetBaseName(contextFile, '.yml')
+    def generatedModel = "${modelEnv}/${basename}"
+    def content = readFile(file: "${templateEnvDir}/contexts/${contextFile}")
+    def templateContext = readYaml text: content
+    def clusterDomain = templateContext.default_context.cluster_domain
+    def clusterName = templateContext.default_context.cluster_name
+    def outputDestination = "${generatedModel}/classes/cluster/${clusterName}"
+    def templateBaseDir = templateEnvDir
+    def templateDir = "${templateEnvDir}/dir"
+    def templateOutputDir = templateBaseDir
+    sh(script: "rm -rf ${generatedModel} || true")
 
-def generateModel(modelFile, cutterEnv) {
-  def templateEnv = "${env.WORKSPACE}"
-  def modelEnv = "${env.WORKSPACE}/model"
-  def basename = GetBaseName(modelFile, '.yml')
-  def generatedModel = "${modelEnv}/${basename}"
-  def testEnv = "${env.WORKSPACE}/test"
-  def content = readFile(file: "${templateEnv}/contexts/${modelFile}")
-  def templateContext = readYaml text: content
-  def clusterDomain = templateContext.default_context.cluster_domain
-  def clusterName = templateContext.default_context.cluster_name
-  def outputDestination = "${generatedModel}/classes/cluster/${clusterName}"
-  def targetBranch = "feature/${clusterName}"
-  def templateBaseDir = "${env.WORKSPACE}"
-  def templateDir = "${templateEnv}/dir"
-  def templateOutputDir = templateBaseDir
-  sh(script: "rm -rf ${generatedModel} || true")
+    common.infoMsg("Generating model from context ${contextFile}")
 
-  common.infoMsg("Generating model from context ${modelFile}")
+    def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
+    for (product in productList) {
 
-  def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
-  for (product in productList) {
+        // get templateOutputDir and productDir
+        if (product.startsWith("stacklight")) {
+            templateOutputDir = "${templateEnvDir}/output/stacklight"
+            try {
+                productDir = "stacklight" + templateContext.default_context['stacklight_version']
+            } catch (Throwable e) {
+                productDir = "stacklight1"
+            }
+        } else {
+            templateOutputDir = "${templateEnvDir}/output/${product}"
+            productDir = product
+        }
 
-    // get templateOutputDir and productDir
-    if (product.startsWith("stacklight")) {
-      templateOutputDir = "${env.WORKSPACE}/output/stacklight"
-      try {
-        productDir = "stacklight" + templateContext.default_context['stacklight_version']
-      } catch (Throwable e) {
-        productDir = "stacklight1"
-      }
-    } else {
-      templateOutputDir = "${env.WORKSPACE}/output/${product}"
-      productDir = product
+        if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+            && templateContext.default_context["${product}_enabled"].toBoolean())) {
+
+            templateDir = "${templateEnvDir}/cluster_product/${productDir}"
+            common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
+
+            sh "rm -rf ${templateOutputDir} || true"
+            sh "mkdir -p ${templateOutputDir}"
+            sh "mkdir -p ${outputDestination}"
+
+            python.buildCookiecutterTemplate(templateDir, content, templateOutputDir, virtualenv, templateBaseDir)
+            sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+        } else {
+            common.warningMsg("Product " + product + " is disabled")
+        }
     }
-
-    if (product == "infra" || (templateContext.default_context["${product}_enabled"]
-        && templateContext.default_context["${product}_enabled"].toBoolean())) {
-
-      templateDir = "${templateEnv}/cluster_product/${productDir}"
-      common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
-
-      sh "rm -rf ${templateOutputDir} || true"
-      sh "mkdir -p ${templateOutputDir}"
-      sh "mkdir -p ${outputDestination}"
-
-      python.buildCookiecutterTemplate(templateDir, content, templateOutputDir, cutterEnv, templateBaseDir)
-      sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-    } else {
-      common.warningMsg("Product " + product + " is disabled")
-    }
-  }
-  generateSaltMaster(generatedModel, clusterDomain, clusterName)
+    generateSaltMaster(generatedModel, clusterDomain, clusterName)
 }
 
 
-def testModel(modelFile, testEnv, reclassVersion='v1.5.4') {
-  // modelFile - `modelfiname` from model/modelfiname/modelfiname.yaml
-  // testEnv - path for model (model/modelfilename/)
-  //* Grub all models and send it to check in paralell - by one in thread.
+def testModel(modelFile, reclassVersion = 'v1.5.4') {
+    // modelFile - `modelfiname` from model/modelfiname/modelfiname.yaml
+    //* Grub all models and send it to check in paralell - by one in thread.
 
-  _values_string =  """
+    _values_string = """
   ---
-  MODELS_TARGZ: "${env.BUILD_URL}/artifact/reclass.tar.gz"
+  MODELS_TARGZ: "${env.BUILD_URL}/artifact/patched_reclass.tar.gz"
   DockerCName: "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}_${modelFile.toLowerCase()}"
   testReclassEnv: "model/${modelFile}/"
   modelFile: "contexts/${modelFile}.yml"
@@ -106,143 +96,197 @@
   EXTRA_FORMULAS: "${env.EXTRA_FORMULAS}"
   reclassVersion: "${reclassVersion}"
   """
-  build job: "test-mk-cookiecutter-templates-chunk", parameters: [
-  [$class: 'StringParameterValue', name: 'EXTRA_VARIABLES_YAML', value: _values_string.stripIndent() ],
-  ]
+    build job: "test-mk-cookiecutter-templates-chunk", parameters: [
+        [$class: 'StringParameterValue', name: 'EXTRA_VARIABLES_YAML',
+         value: _values_string.stripIndent()],
+    ]
 }
 
-def gerritRef
-try {
-  gerritRef = GERRIT_REFSPEC
-  } catch (MissingPropertyException e) {
-    gerritRef = null
-  }
-
-def testModelStep(basename,testEnv) {
-  // We need to wrap what we return in a Groovy closure, or else it's invoked
-  // when this method is called, not when we pass it to parallel.
-  // To do this, you need to wrap the code below in { }, and either return
-  // that explicitly, or use { -> } syntax.
-  return {
-    node(slave_node) {
-      testModel(basename, testEnv)
+def StepTestModel(basename) {
+    // We need to wrap what we return in a Groovy closure, or else it's invoked
+    // when this method is called, not when we pass it to parallel.
+    // To do this, you need to wrap the code below in { }, and either return
+    // that explicitly, or use { -> } syntax.
+    // return node object
+    return {
+        node(slaveNode) {
+            testModel(basename)
+        }
     }
-  }
 }
 
-timeout(time: 2, unit: 'HOURS') {
-  node(slave_node) {
-    def templateEnv = "${env.WORKSPACE}"
-    def cutterEnv = "${env.WORKSPACE}/cutter"
-    def jinjaEnv = "${env.WORKSPACE}/jinja"
-
-    try {
-      // Fixme. Just use 'cleanup workspace' option.
-      stage("Cleanup") {
-        sh(script:  'find . -mindepth 1 -delete > /dev/null || true')
-      }
-
-      stage('Download Cookiecutter template') {
-        if (gerritRef) {
-          def gerritChange = gerrit.getGerritChange(GERRIT_NAME, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID)
-          merged = gerritChange.status == "MERGED"
-          if (!merged) {
-            checkouted = gerrit.gerritPatchsetCheckout([
-              credentialsId: CREDENTIALS_ID
-              ])
-            } else {
-              common.successMsg("Change ${GERRIT_CHANGE_NUMBER} is already merged, no need to gate them")
-            }
-            } else {
-              git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, CREDENTIALS_ID)
-            }
-          }
-
-          stage("Setup") {
-            python.setupCookiecutterVirtualenv(cutterEnv)
-          }
-
-          stage("Check workflow_definition") {
-            sh(script: "python ${env.WORKSPACE}/workflow_definition_test.py")
-          }
-
-          def contextFileList = []
-          dir("${templateEnv}/contexts") {
-            for (String x : findFiles(glob: "*.yml")) {
-              contextFileList.add(x)
-            }
-          }
-
-          stage("generate-model") {
-            for (contextFile in contextFileList) {
-              generateModel(contextFile, cutterEnv)
-            }
-          }
-
-          dir("${env.WORKSPACE}") {
-          // Collect only models. For backward compatability - who know, probably someone use it..
-          sh(script: "tar -czf model.tar.gz -C model ../contexts .", returnStatus: true)
-          archiveArtifacts artifacts: "model.tar.gz"
-          // to be able share reclass for all subenvs
-          // Also, makes artifact test more solid - use one reclass for all of sub-models.
-          // Archive Structure will be:
-          // tar.gz
-          // ├── contexts
-          // │   └── ceph.yml
-          // ├── global_reclass <<< reclass system
-          // ├── model
-          // │   └── ceph       <<< from `context basename`
-          // │       ├── classes
-          // │       │   ├── cluster
-          // │       │   └── system -> ../../../global_reclass
-          // │       └── nodes
-          // │           └── cfg01.ceph-cluster-domain.local.yml
-
-          if (SYSTEM_GIT_URL == "") {
-            git.checkoutGitRepository("${env.WORKSPACE}/global_reclass/", RECLASS_MODEL_URL, RECLASS_MODEL_BRANCH, CREDENTIALS_ID)
-            } else {
-              dir("${env.WORKSPACE}/global_reclass/") {
-                if (!gerrit.gerritPatchsetCheckout(SYSTEM_GIT_URL, SYSTEM_GIT_REF, "HEAD", CREDENTIALS_ID)) {
-                  common.errorMsg("Failed to obtain system reclass with url: ${SYSTEM_GIT_URL} and ${SYSTEM_GIT_REF}")
-                  throw new RuntimeException("Failed to obtain system reclass")
+def StepPrepareCCenv(refchange, templateEnvFolder) {
+    // return git clone  object
+    return {
+        // fetch needed sources
+        dir(templateEnvFolder) {
+            if (refchange) {
+                def gerritChange = gerrit.getGerritChange(GERRIT_NAME, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID)
+                merged = gerritChange.status == "MERGED"
+                if (!merged) {
+                    checkouted = gerrit.gerritPatchsetCheckout([
+                        credentialsId: CREDENTIALS_ID
+                    ])
+                } else {
+                    // update global variable for success return from pipeline
+                    //alreadyMerged = true
+                    common.successMsg("Change ${GERRIT_CHANGE_NUMBER} is already merged, no need to gate them")
+                    currentBuild.result = 'ABORTED'
+                    throw new hudson.AbortException('change already merged')
                 }
-              }
+            } else {
+                git.checkoutGitRepository(templateEnvFolder, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, CREDENTIALS_ID)
             }
-            // link all models, to use one global reclass
-            for (String context : contextFileList) {
-              def basename = GetBaseName(context, '.yml')
-              dir("${env.WORKSPACE}/model/${basename}"){
-                sh(script: 'mkdir -p classes/; ln -sfv ../../../global_reclass classes/system ')
-              }
-            }
-            // Save all models and all contexts. Warning! `h` flag has been used.
-            sh(script: "tar -chzf reclass.tar.gz --exclude='*@tmp' model contexts global_reclass", returnStatus: true)
-            archiveArtifacts artifacts: "reclass.tar.gz"
-          }
-
-          stage("test-contexts") {
-            stepsForParallel = [:]
-            common.infoMsg("Found: ${contextFileList.size()} contexts to test.")
-            for (String context : contextFileList) {
-              def basename = GetBaseName(context, '.yml')
-              def testEnv = "${env.WORKSPACE}/model/${basename}"
-              stepsForParallel.put("Test:${basename}", testModelStep(basename, testEnv))
-            }
-            parallel stepsForParallel
-            common.infoMsg('All tests done')
-          }
-
-          stage('Clean workspace directories') {
-            sh(script:  'find . -mindepth 1 -delete > /dev/null || true')
-          }
-
-} catch (Throwable e) {
-  currentBuild.result = "FAILURE"
-  currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
-  throw e
-  } finally {
-    def dummy = "dummy"
-      //FAILING common.sendNotification(currentBuild.result,"",["slack"])
+        }
     }
-  }
+}
+
+def StepGenerateModels(_contextFileList, _virtualenv, _templateEnvDir) {
+    return {
+        for (contextFile in _contextFileList) {
+            generateModel(contextFile, _virtualenv, _templateEnvDir)
+        }
+    }
+}
+
+timeout(time: 1, unit: 'HOURS') {
+    node(slaveNode) {
+        def templateEnvHead = "${env.WORKSPACE}/env_head/"
+        def templateEnvPatched = "${env.WORKSPACE}/env_patched/"
+        def contextFileListHead = []
+        def contextFileListPatched = []
+        def vEnv = "${env.WORKSPACE}/venv"
+
+        try {
+            sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
+            stage('Download and prepare CC env') {
+                // Prepare 2 env - for patchset, and for HEAD
+                paralellEnvs = [:]
+                paralellEnvs.failFast = true
+                paralellEnvs['downloadEnvHead'] = StepPrepareCCenv('', templateEnvHead)
+                paralellEnvs['downloadEnvPatched'] = StepPrepareCCenv(gerritRef, templateEnvPatched)
+                parallel paralellEnvs
+            }
+            stage("Check workflow_definition") {
+                // Check only for patchset
+                python.setupVirtualenv(vEnv, 'python2', [], "${templateEnvPatched}/requirements.txt")
+                common.infoMsg(python.runVirtualenvCommand(vEnv, "python ${templateEnvPatched}/workflow_definition_test.py"))
+            }
+
+            stage("generate models") {
+                dir("${templateEnvHead}/contexts") {
+                    for (String x : findFiles(glob: "*.yml")) {
+                        contextFileListHead.add(x)
+                    }
+                }
+                dir("${templateEnvPatched}/contexts") {
+                    for (String x : findFiles(glob: "*.yml")) {
+                        contextFileListPatched.add(x)
+                    }
+                }
+                // Generate over 2env's - for patchset, and for HEAD
+                paralellEnvs = [:]
+                paralellEnvs.failFast = true
+                paralellEnvs['GenerateEnvHead'] = StepGenerateModels(contextFileListPatched, vEnv, templateEnvPatched)
+                paralellEnvs['GenerateEnvPatched'] = StepGenerateModels(contextFileListHead, vEnv, templateEnvHead)
+                parallel paralellEnvs
+
+                // Collect artifacts
+                dir(templateEnvPatched) {
+                    // Collect only models. For backward comparability - who know, probably someone use it..
+                    sh(script: "tar -czf model.tar.gz -C model ../contexts .", returnStatus: true)
+                    archiveArtifacts artifacts: "model.tar.gz"
+                }
+
+                // to be able share reclass for all subenvs
+                // Also, makes artifact test more solid - use one reclass for all of sub-models.
+                // Archive Structure will be:
+                // tar.gz
+                // ├── contexts
+                // │   └── ceph.yml
+                // ├── global_reclass <<< reclass system
+                // ├── model
+                // │   └── ceph       <<< from `context basename`
+                // │       ├── classes
+                // │       │   ├── cluster
+                // │       │   └── system -> ../../../global_reclass
+                // │       └── nodes
+                // │           └── cfg01.ceph-cluster-domain.local.yml
+
+                if (SYSTEM_GIT_URL == "") {
+                    git.checkoutGitRepository("${env.WORKSPACE}/global_reclass/", RECLASS_MODEL_URL, RECLASS_MODEL_BRANCH, CREDENTIALS_ID)
+                } else {
+                    dir("${env.WORKSPACE}/global_reclass/") {
+                        if (!gerrit.gerritPatchsetCheckout(SYSTEM_GIT_URL, SYSTEM_GIT_REF, "HEAD", CREDENTIALS_ID)) {
+                            common.errorMsg("Failed to obtain system reclass with url: ${SYSTEM_GIT_URL} and ${SYSTEM_GIT_REF}")
+                            throw new RuntimeException("Failed to obtain system reclass")
+                        }
+                    }
+                }
+                // link all models, to use one global reclass
+                // For HEAD
+                dir(templateEnvHead) {
+                    for (String context : contextFileListHead) {
+                        def basename = common.GetBaseName(context, '.yml')
+                        dir("${templateEnvHead}/model/${basename}") {
+                            sh(script: 'mkdir -p classes/; ln -sfv ../../../../global_reclass classes/system ')
+                        }
+                    }
+                    // Save all models and all contexts. Warning! `h` flag must be used.
+                    sh(script: "tar -chzf head_reclass.tar.gz --exclude='*@tmp' model contexts global_reclass", returnStatus: true)
+                    archiveArtifacts artifacts: "head_reclass.tar.gz"
+                    // move for "Compare Pillars" stage
+                    sh(script: "mv -v head_reclass.tar.gz ${env.WORKSPACE}")
+                }
+                // For patched
+                dir(templateEnvPatched) {
+                    for (String context : contextFileListPatched) {
+                        def basename = common.GetBaseName(context, '.yml')
+                        dir("${templateEnvPatched}/model/${basename}") {
+                            sh(script: 'mkdir -p classes/; ln -sfv ../../../../global_reclass classes/system ')
+                        }
+                    }
+                    // Save all models and all contexts. Warning! `h` flag must be used.
+                    sh(script: "tar -chzf patched_reclass.tar.gz --exclude='*@tmp' model contexts global_reclass", returnStatus: true)
+                    archiveArtifacts artifacts: "patched_reclass.tar.gz"
+                    // move for "Compare Pillars" stage
+                    sh(script: "mv -v patched_reclass.tar.gz ${env.WORKSPACE}")
+                }
+            }
+
+            stage("Compare Pillars") {
+                // Compare patched and HEAD reclass pillars
+                compareRoot = "${env.WORKSPACE}/test_compare/"
+                sh(script: """
+                   mkdir -pv ${compareRoot}/new ${compareRoot}/old
+                   tar -xzf patched_reclass.tar.gz  --directory ${compareRoot}/new
+                   tar -xzf head_reclass.tar.gz  --directory ${compareRoot}/old
+                   """)
+                common.warningMsg('infra/secrets.yml has been skipped from compare!')
+                rezult = common.comparePillars(compareRoot, env.BUILD_URL, "-Ev 'infra/secrets.yml'")
+                currentBuild.description = rezult
+            }
+            stage("test-contexts") {
+                // Test contexts for patched only
+                stepsForParallel = [:]
+                common.infoMsg("Found: ${contextFileListPatched.size()} patched contexts to test.")
+                for (String context : contextFileListPatched) {
+                    def basename = common.GetBaseName(context, '.yml')
+                    stepsForParallel.put("ContextPatchTest:${basename}", StepTestModel(basename))
+                }
+                parallel stepsForParallel
+                common.infoMsg('All tests done')
+            }
+
+            sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
+
+        } catch (Throwable e) {
+            currentBuild.result = "FAILURE"
+            currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+            throw e
+        } finally {
+            def dummy = "dummy"
+            //FAILING common.sendNotification(currentBuild.result,"",["slack"])
+        }
+    }
 }