Merge "Add verification of Calico policy setting during upgrade"
diff --git a/cvp-runner.groovy b/cvp-runner.groovy
index 7cf8e28..6b5c0e2 100644
--- a/cvp-runner.groovy
+++ b/cvp-runner.groovy
@@ -10,27 +10,34 @@
  *   TESTS_REPO                      Repo to clone
  *   TESTS_SETTINGS                  Additional environment varibales to apply
  *   PROXY                           Proxy to use for cloning repo or for pip
- *   TEST_IMAGE                      Docker image link or name to use for running container with test framework.
+ *   IMAGE                           Docker image to use for running container with test framework.
  *   DEBUG_MODE                      If you need to debug (keep container after test), please enabled this
- *
+ *  To launch tests from cvp_spt docker images need to set IMAGE and left TESTS_REPO empty
  */
 
 common = new com.mirantis.mk.Common()
 validate = new com.mirantis.mcp.Validate()
 salt = new com.mirantis.mk.Salt()
-def artifacts_dir = 'validation_artifacts/'
-def remote_dir = '/root/qa_results/'
+salt_testing = new com.mirantis.mk.SaltModelTesting()
+def artifacts_dir = "validation_artifacts/"
+def remote_dir = '/root/qa_results'
 def container_workdir = '/var/lib'
+def name = 'cvp-spt'
+def xml_file = "${name}_report.xml"
 def TARGET_NODE = "I@gerrit:client"
 def reinstall_env = false
 def container_name = "${env.JOB_NAME}"
 def saltMaster
 def settings
 
-node() {
+slaveNode = (env.getProperty('SLAVE_NODE')) ?: 'docker'
+imageName = (env.getProperty('IMAGE')) ?: 'docker-prod-local.docker.mirantis.net/mirantis/cvp/cvp-spt:stable'
+
+node(slaveNode) {
     try{
         stage('Initialization') {
             sh "rm -rf ${artifacts_dir}"
+            // TODO collaps TESTS_SETTINGS flow into EXTRA variables map
             if ( TESTS_SETTINGS != "" ) {
                 for (var in TESTS_SETTINGS.tokenize(";")) {
                     key = var.tokenize("=")[0].trim()
@@ -50,11 +57,11 @@
                 validate.prepareVenv(TESTS_REPO, PROXY)
             } else {
                 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
-                salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_dir}")
-                salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_dir}")
+                salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_dir}/")
+                salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_dir}/")
                 validate.runContainer(saltMaster, TARGET_NODE, IMAGE, container_name)
                 if ( TESTS_REPO != "") {
-                    salt.cmdRun(saltMaster, TARGET_NODE, "docker exec ${container_name} rm -rf ${container_workdir}/cvp*")
+                    salt.cmdRun(saltMaster, TARGET_NODE, "docker exec ${container_name} rm -rf ${container_workdir}/${container_name}")
                     salt.cmdRun(saltMaster, TARGET_NODE, "docker exec ${container_name} git clone ${TESTS_REPO} ${container_workdir}/${container_name}")
                     TESTS_SET = container_workdir + '/' + container_name + '/' + TESTS_SET
                     if ( reinstall_env ) {
@@ -66,8 +73,36 @@
         }
 
         stage('Run Tests') {
+            def creds = common.getCredentials(SALT_MASTER_CREDENTIALS)
+            def username = creds.username
+            def password = creds.password
+            def script = "pytest --junitxml ${container_workdir}/${artifacts_dir}/${xml_file} --tb=short -sv ${container_workdir}/${TESTS_SET}"
+
             sh "mkdir -p ${artifacts_dir}"
-            validate.runPyTests(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS, TESTS_SET, TESTS_SETTINGS.tokenize(";"), container_name, TARGET_NODE, remote_dir, artifacts_dir)
+
+            def configRun = [
+                'image': imageName,
+                'baseRepoPreConfig': false,
+                'dockerMaxCpus': 2,
+                'dockerExtraOpts' : [
+                    "-v /root/qa_results/:/root/qa_results/",
+                    "-v ${env.WORKSPACE}/validation_artifacts/:${container_workdir}/validation_artifacts/",
+                    "--entrypoint=''",  // to override ENTRYPOINT=/bin/bash in Dockerfile of image
+                ],
+
+                'envOpts'         : [
+                    "WORKSPACE=${container_workdir}/${name}",
+                    "SALT_USERNAME=${username}",
+                    "SALT_PASSWORD=${password}",
+                    "SALT_URL=${SALT_MASTER_URL}"
+                ] + TESTS_SETTINGS.replaceAll('\\"', '').tokenize(";"),
+                'runCommands'     : [
+                      '010_start_tests'    : {
+                          sh("cd ${container_workdir} && ${script}")
+                      }
+                  ]
+                ]
+            salt_testing.setupDockerAndTest(configRun)
         }
 
         stage ('Publish results') {
diff --git a/test-openscap-pipeline.groovy b/test-openscap-pipeline.groovy
index b886467..7134cfb 100644
--- a/test-openscap-pipeline.groovy
+++ b/test-openscap-pipeline.groovy
@@ -15,6 +15,7 @@
  *                              For OVAL definitions, paths to OVAL definition files separated by semicolon, profile is ignored.
  *  XCCDF_VERSION               The XCCDF version (default 1.2)
  *  XCCDF_TAILORING_ID          The tailoring id (default None)
+ *  XCCDF_CPE                   CPE dictionary or language for applicability checks (default None)
  *
  *  TARGET_SERVERS              The target Salt nodes (default *)
  *
@@ -149,6 +150,7 @@
     def benchmarksAndProfilesArray = XCCDF_BENCHMARKS.tokenize(';')
     def xccdfVersion = XCCDF_VERSION ?: '1.2'
     def xccdfTailoringId = XCCDF_TAILORING_ID ?: 'None'
+    def xccdfCPE = XCCDF_CPE ?: ''
     def targetServers = TARGET_SERVERS ?: '*'
 
     // To have an ability to work in heavy concurrency conditions
@@ -203,7 +205,7 @@
             salt.runSaltProcessStep(pepperEnv, targetServers, 'oscap.eval', [
                 benchmarkType, benchmarkFile, "results_dir=${resultsDir}",
                 "profile=${profileName}", "xccdf_version=${xccdfVersion}",
-                "tailoring_id=${xccdfTailoringId}"
+                "tailoring_id=${xccdfTailoringId}", "cpe=${xccdfCPE}"
             ])
 
             salt.cmdRun(pepperEnv, targetServers, "rm -f /tmp/${scanUUID}.tar.xz; tar -cJf /tmp/${scanUUID}.tar.xz -C ${resultsBaseDir} .")
diff --git a/test-salt-model-wrapper.groovy b/test-salt-model-wrapper.groovy
index 3ef577b..2691081 100644
--- a/test-salt-model-wrapper.groovy
+++ b/test-salt-model-wrapper.groovy
@@ -26,14 +26,26 @@
  */
 
 import groovy.json.JsonOutput
+gerrit = new com.mirantis.mk.Gerrit()
 
 cookiecutterTemplatesRepo='mk/cookiecutter-templates'
 reclassSystemRepo='salt-models/reclass-system'
 slaveNode = env.getProperty('SLAVE_NODE') ?: 'python&&docker'
 
+voteMatrix = [
+  'test-mk-cookiecutter-templates': true,
+  'test-drivetrain': true,
+  'oscore-test-cookiecutter-models': false,
+  'test-salt-model-infra': true,
+  'test-salt-model-mcp-virtual-lab': true,
+]
+
+baseGerritConfig = [:]
+jobResultComments = [:]
+commentLock = false
+
 LinkedHashMap getManualRefParams(LinkedHashMap map) {
     LinkedHashMap manualParams = [:]
-    String defaultGitRef = 'HEAD'
     if (map.containsKey('RECLASS_SYSTEM_GIT_REF') && map.containsKey('RECLASS_SYSTEM_URL')) {
         manualParams[reclassSystemRepo] = [
             'url': map.get('RECLASS_SYSTEM_URL'),
@@ -51,33 +63,60 @@
     return manualParams
 }
 
-def runTests(String jobName, String extraVars, Boolean propagateStatus=true) {
-    return {
-        try {
-            build job: "${jobName}", parameters: [
-                [$class: 'TextParameterValue', name: 'EXTRA_VARIABLES_YAML', value: extraVars ]
-            ]
-        } catch (Exception e) {
-            if (propagateStatus) {
-                throw e
+def setGerritReviewComment(Boolean initComment = false) {
+    if (baseGerritConfig) {
+        while(commentLock) {
+            sleep 5
+        }
+        commentLock = true
+        LinkedHashMap config = baseGerritConfig.clone()
+        String jobResultComment = ''
+        jobResultComments.each { job, info ->
+            String skipped = ''
+            if (!initComment) {
+                skipped = voteMatrix.get(job, 'true') ? '' : '(skipped)'
             }
+            jobResultComment += "- ${job} ${info.url}console : ${info.status} ${skipped}".trim() + '\n'
+        }
+        config['message'] = sh(script: "echo '${jobResultComment}'", returnStdout: true).trim()
+        gerrit.postGerritComment(config)
+        commentLock = false
+    }
+}
+
+def runTests(String jobName, String extraVars) {
+    def propagateStatus = voteMatrix.get(jobName, true)
+    return {
+        def jobBuild = build job: jobName, propagate: false, parameters: [
+            [$class: 'TextParameterValue', name: 'EXTRA_VARIABLES_YAML', value: extraVars ]
+        ]
+        jobResultComments[jobName] = [ 'url': jobBuild.absoluteUrl, 'status': jobBuild.result ]
+        setGerritReviewComment()
+        if (propagateStatus && jobBuild.result == 'FAILURE') {
+            throw new Exception("Build ${jobName} is failed!")
         }
     }
 }
 
-def runTestSaltModelReclass(String cluster, String defaultGitUrl, String clusterGitUrl, String refSpec) {
+def runTestSaltModelReclass(String jobName, String defaultGitUrl, String clusterGitUrl, String refSpec) {
+    def propagateStatus = voteMatrix.get(jobName, true)
     return {
-        build job: "test-salt-model-${cluster}", parameters: [
+        def jobBuild = build job: jobName, propagate: false, parameters: [
             [$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: clusterGitUrl],
             [$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: "HEAD"],
             [$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: defaultGitUrl],
             [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: refSpec ],
         ]
+        jobResultComments[jobName] = [ 'url': jobBuild.absoluteUrl, 'status': jobBuild.result ]
+        setGerritReviewComment()
+        if (propagateStatus && jobBuild.result == 'FAILURE') {
+            throw new Exception("Build ${jobName} is failed!")
+        }
     }
 }
 
-def checkReclassSystemDocumentationCommit(gerritLib, gerritCredentials) {
-    gerritLib.gerritPatchsetCheckout([
+def checkReclassSystemDocumentationCommit(gerritCredentials) {
+    gerrit.gerritPatchsetCheckout([
         credentialsId: gerritCredentials
     ])
 
@@ -90,14 +129,16 @@
 timeout(time: 12, unit: 'HOURS') {
     node(slaveNode) {
         def common = new com.mirantis.mk.Common()
-        def gerrit = new com.mirantis.mk.Gerrit()
         def git = new com.mirantis.mk.Git()
         def python = new com.mirantis.mk.Python()
 
         // Var TEST_PARAMETERS_YAML contains any additional parameters for tests,
         // like manually specified Gerrit Refs/URLs, additional parameters and so on
-        if (env.getProperty('TEST_PARAMETERS_YAML')) {
-            common.mergeEnv(env, env.getProperty('TEST_PARAMETERS_YAML'))
+        def buildTestParams = [:]
+        def buildTestParamsYaml = env.getProperty('TEST_PARAMETERS_YAML')
+        if (buildTestParamsYaml) {
+            common.mergeEnv(env, buildTestParamsYaml)
+            buildTestParams = readYaml text: buildTestParamsYaml
         }
 
         // init required job variables
@@ -128,6 +169,7 @@
                 gerritHost = job_env.get('GERRIT_HOST')
                 gerritPort = job_env.get('GERRIT_PORT')
                 gerritChangeNumber = job_env.get('GERRIT_CHANGE_NUMBER')
+                gerritPatchSetNumber = job_env.get('GERRIT_PATCHSET_NUMBER')
                 gerritBranch = job_env.get('GERRIT_BRANCH')
 
                 // check if change aren't already merged
@@ -144,6 +186,14 @@
                     'branch': gerritBranch,
                 ]
                 buildType = 'Gerrit Trigger'
+                buildTestParams << job_env.findAll { k,v -> k ==~ /GERRIT_.+/ }
+                baseGerritConfig = [
+                    'gerritName': gerritName,
+                    'gerritHost': gerritHost,
+                    'gerritChangeNumber': gerritChangeNumber,
+                    'credentialsId': gerritCredentials,
+                    'gerritPatchSetNumber': gerritPatchSetNumber,
+                ]
             } else {
                 projectsMap = getManualRefParams(job_env)
                 if (!projectsMap) {
@@ -157,41 +207,49 @@
                 descriptionMsgs.add("Branch for ${project} => ${projectsMap[project]['branch']}")
             }
             descriptionMsgs.add("Distrib revision => ${distribRevision}")
-            currentBuild.description = descriptionMsgs.join('\n')
+            currentBuild.description = descriptionMsgs.join('<br/>')
         }
 
         stage("Run tests") {
             def branches = [:]
-            branches.failFast = true
+            String branchJobName = ''
 
             if (projectsMap.containsKey(reclassSystemRepo)) {
-                def documentationOnly = checkReclassSystemDocumentationCommit(gerrit, gerritCredentials)
+                def documentationOnly = checkReclassSystemDocumentationCommit(gerritCredentials)
                 if (['master'].contains(gerritBranch) && !documentationOnly) {
                     for (int i = 0; i < testModels.size(); i++) {
                         def cluster = testModels[i]
-                        //def clusterGitUrl = projectsMap[reclassSystemRepo]['url'].substring(0, defaultGitUrl.lastIndexOf("/") + 1) + cluster
-                        def clusterGitUrl = ''
-                        branches["reclass-system-${cluster}"] = runTestSaltModelReclass(cluster, projectsMap[reclassSystemRepo]['url'], clusterGitUrl, projectsMap[reclassSystemRepo]['ref'])
+                        def clusterGitUrl = projectsMap[reclassSystemRepo]['url'].substring(0, projectsMap[reclassSystemRepo]['url'].lastIndexOf("/") + 1) + cluster
+                        branchJobName = "test-salt-model-${cluster}"
+                        branches[branchJobName] = runTestSaltModelReclass(branchJobName, projectsMap[reclassSystemRepo]['url'], clusterGitUrl, projectsMap[reclassSystemRepo]['ref'])
                     }
                 } else {
                     common.warningMsg("Tests for ${testModels} skipped!")
                 }
             }
             if (projectsMap.containsKey(reclassSystemRepo) || projectsMap.containsKey(cookiecutterTemplatesRepo)) {
-                branches['cookiecutter-templates'] = runTests('test-mk-cookiecutter-templates', JsonOutput.toJson(job_env))
+                branchJobName = 'test-mk-cookiecutter-templates'
+                branches[branchJobName] = runTests(branchJobName, JsonOutput.toJson(buildTestParams))
             }
             if (projectsMap.containsKey(cookiecutterTemplatesRepo)) {
-                branches['test-drivetrain'] = runTests('test-drivetrain', JsonOutput.toJson(job_env))
-                branches['oscore-test-cookiecutter-models'] = runTests('oscore-test-cookiecutter-models', JsonOutput.toJson(job_env))
+                branchJobName = 'test-drivetrain'
+                branches[branchJobName] = runTests(branchJobName, JsonOutput.toJson(buildTestParams))
+                branchJobName = 'oscore-test-cookiecutter-models'
+                branches[branchJobName] = runTests(branchJobName, JsonOutput.toJson(buildTestParams))
             }
 
-            // temp block to disable test run until job is stable
-            print branches.keySet()
-            currentBuild.result = 'SUCCESS'
-            return
-            // ----
-
-            parallel branches
+            branches.keySet().each { key ->
+                if (branches[key] instanceof Closure) {
+                    jobResultComments[key] = [ 'url': job_env.get('BUILD_URL'), 'status': 'WAITING' ]
+                }
+            }
+            setGerritReviewComment(true)
+            try {
+                parallel branches
+            } catch (Exception e) {
+                println e
+                println 'Job is in non-voting mode for now. Skipping fails.'
+            }
         }
     }
 }