Merge "Revert "commitMessage is provided in the job from the EXTRA_VARIABLES_YAML""
diff --git a/src/com/mirantis/mk/Gerrit.groovy b/src/com/mirantis/mk/Gerrit.groovy
index 4d04906..51771cb 100644
--- a/src/com/mirantis/mk/Gerrit.groovy
+++ b/src/com/mirantis/mk/Gerrit.groovy
@@ -188,7 +188,7 @@
     if(includeCurrentPatchset){
         curPatchset = "--current-patch-set"
     }
-    return common.parseJSON(ssh.agentSh(String.format("ssh -p %s %s@%s gerrit query ${curPatchset} --format=JSON change:%s", gerritPort, gerritName, gerritHost, gerritChangeNumber)))
+    return common.parseJSON(ssh.agentSh(String.format("ssh -p %s %s@%s gerrit query ${curPatchset} --commit-message --format=JSON change:%s", gerritPort, gerritName, gerritHost, gerritChangeNumber)))
 }
 
 /**
@@ -552,7 +552,8 @@
         }
         common.infoMsg("GERRIT_EVENT_COMMENT_TEXT is ${gerritEventCommentTextStr}")
     }
-    if (gerritEventType == 'comment-added' && gerritEventCommentTextStr =~ /^Patch Set \d+:\s.*Workflow\+1$/) {
+    def gateComment = gerritEventCommentTextStr.split('\n')[0]
+    if (gerritEventType == 'comment-added' && gateComment =~ /^Patch Set \d+:\s.*Workflow\+1.*$/) {
         common.infoMsg("Running in gate mode")
         res = true
     }
diff --git a/src/com/mirantis/mk/KaasUtils.groovy b/src/com/mirantis/mk/KaasUtils.groovy
index a96bd2d..0782f54 100644
--- a/src/com/mirantis/mk/KaasUtils.groovy
+++ b/src/com/mirantis/mk/KaasUtils.groovy
@@ -80,6 +80,9 @@
     def runMgmtDeleteMasterTest = env.RUN_MGMT_DELETE_MASTER_TEST ? env.RUN_MGMT_DELETE_MASTER_TEST.toBoolean() : false
     def runRgnlDeleteMasterTest = env.RUN_RGNL_DELETE_MASTER_TEST ? env.RUN_RGNL_DELETE_MASTER_TEST.toBoolean() : false
     def runChildDeleteMasterTest = env.RUN_CHILD_DELETE_MASTER_TEST ? env.RUN_CHILD_DELETE_MASTER_TEST.toBoolean() : false
+    def runGracefulRebootTest = env.RUN_GRACEFUL_REBOOT_TEST ? env.RUN_GRACEFUL_REBOOT_TEST.toBoolean() : false
+    def pauseForDebug = env.PAUSE_FOR_DEBUG ? env.PAUSE_FOR_DEBUG.toBoolean() : false
+    def runChildMachineDeletionPolicyTest = env.RUN_CHILD_MACHINE_DELETION_POLICY_TEST ? env.RUN_CHILD_MACHINE_DELETION_POLICY_TEST.toBoolean() : false
     // multiregion configuration from env variable: comma-separated string in form $mgmt_provider,$regional_provider
     def multiregionalMappings = env.MULTIREGION_SETUP ? multiregionWorkflowParser(env.MULTIREGION_SETUP) : [
         enabled: false,
@@ -110,6 +113,8 @@
     def childOsBootFromVolume = env.OPENSTACK_BOOT_FROM_VOLUME ? env.OPENSTACK_BOOT_FROM_VOLUME.toBoolean() : false
     def bootstrapV2Scenario = env.BOOTSTRAP_V2_ENABLED ? env.BOOTSTRAP_V2_ENABLED.toBoolean() : false
     def equinixMetalV2Metro = env.EQUINIX_MGMT_METRO ? env.EQUINIX_MGMT_METRO : ''
+    def enableFips = env.ENABLE_FIPS ? env.ENABLE_FIPS.toBoolean() : false
+    def aioCluster = env.AIO_CLUSTER ? env.AIO_CLUSTER.toBoolean() : false
 
     def commitMsg = env.GERRIT_CHANGE_COMMIT_MESSAGE ? new String(env.GERRIT_CHANGE_COMMIT_MESSAGE.decodeBase64()) : ''
     if (commitMsg ==~ /(?s).*\[mgmt-proxy\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*mgmt-proxy.*/) {
@@ -230,6 +235,19 @@
     }
     if (commitMsg ==~ /(?s).*\[child-delete-master-test\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*child-delete-master-test.*/) {
         runChildDeleteMasterTest = true
+        deployChild = true
+        common.infoMsg('Child cluster deployment will be enabled since delete child master node test suite will be executed')
+    }
+    if (commitMsg ==~ /(?s).*\[child-machine-deletion-policy-test\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*child-machine-deletion-policy-test.*/) {
+        runChildMachineDeletionPolicyTest = true
+        deployChild = true
+        common.infoMsg('Child cluster deployment will be enabled since machine deletion child policy test suite will be executed')
+    }
+    if (commitMsg ==~ /(?s).*\[graceful-reboot-test\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*graceful-reboot-test.*/) {
+        runGracefulRebootTest = true
+    }
+    if (commitMsg ==~ /(?s).*\[pause-for-debug\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*pause-for-debug.*/) {
+        pauseForDebug = true
     }
     if (commitMsg ==~ /(?s).*\[child-offline\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*child-offline.*/) {
         proxyConfig['childOffline'] = true
@@ -316,6 +334,10 @@
         upgradeChild = false
     }
 
+    if (commitMsg ==~ /(?s).*\[aio-cluster\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*aio-cluster.*/) {
+        aioCluster = true
+    }
+
     // multiregional tests
     def multiRegionalMatches = (commitMsg =~ /(\[multiregion\s*.*?\])/)
     if (multiRegionalMatches.size() > 0) {
@@ -388,6 +410,10 @@
         bootstrapV2Scenario = true
     }
 
+    if (commitMsg ==~ /(?s).*\[enable-fips\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*enable-fips\.*/) {
+        enableFips = true
+    }
+
     // parse equinixmetalv2-metro trigger
     def equinixMetalV2MetroMatcher = (commitMsg =~ /\[equinixmetalv2-metro(\s+.*)?\]/)
     if (equinixMetalV2OnDemandDemo && equinixMetalV2MetroMatcher.size() > 0) {
@@ -423,9 +449,11 @@
         Maintenance test: ${runMaintenanceTest}
         Container Registry test: ${runContainerregistryTest}
         Child proxy test: ${runProxyChildTest}
+        Graceful reboot test: ${runGracefulRebootTest}
         Delete mgmt master node test: ${runMgmtDeleteMasterTest}
         Delete rgnl master node test: ${runRgnlDeleteMasterTest}
         Delete child master node test: ${runChildDeleteMasterTest}
+        Child machine deletion policy test: ${runChildMachineDeletionPolicyTest}
         AWS provider deployment scheduled: ${awsOnDemandDemo}
         Equinix provider deployment scheduled: ${equinixOnDemandDemo}
         EquinixmetalV2 provider deployment scheduled: ${equinixMetalV2OnDemandDemo}
@@ -444,57 +472,65 @@
         Service binaries fetching scheduled: ${fetchServiceBinaries}
         Current weight of the demo run: ${demoWeight} (Used to manage lockable resources)
         Bootstrap v2 scenario enabled: ${bootstrapV2Scenario}
+        FIPS enabled: ${enableFips}
+        Pause for debug enabled: ${pauseForDebug}
+        AIO cluster: ${aioCluster}
         Triggers: https://gerrit.mcp.mirantis.com/plugins/gitiles/kaas/core/+/refs/heads/master/hack/ci-gerrit-keywords.md""")
     return [
-        osCloudLocation                      : openstackIMC,
-        cdnConfig                            : cdnConfig,
-        proxyConfig                          : proxyConfig,
-        useMacOsSeedNode                     : seedMacOs,
-        deployChildEnabled                   : deployChild,
-        childDeployCustomRelease             : customChildRelease,
-        upgradeChildEnabled                  : upgradeChild,
-        fullUpgradeChildEnabled              : fullUpgradeChild,
-        mosDeployChildEnabled                : mosDeployChild,
-        mosUpgradeChildEnabled               : mosUpgradeChild,
-        runChildConformanceEnabled           : runChildConformance,
-        attachBYOEnabled                     : attachBYO,
-        upgradeBYOEnabled                    : upgradeBYO,
-        runBYOMatrixEnabled                  : runBYOMatrix,
-        defaultBYOOs                         : defaultBYOOs,
-        upgradeMgmtEnabled                   : upgradeMgmt,
-        autoUpgradeMgmtEnabled               : autoUpgradeMgmt,
-        enableLMALoggingEnabled              : enableLMALogging,
-        deployOsOnMosEnabled                 : deployOsOnMos,
-        runUie2eEnabled                      : runUie2e,
-        runUie2eNewEnabled                   : runUie2eNew,
-        runMgmtConformanceEnabled            : runMgmtConformance,
-        runMaintenanceTestEnabled            : runMaintenanceTest,
-        runContainerregistryTestEnabled      : runContainerregistryTest,
-        runMgmtDeleteMasterTestEnabled       : runMgmtDeleteMasterTest,
-        runRgnlDeleteMasterTestEnabled       : runRgnlDeleteMasterTest,
-        runChildDeleteMasterTestEnabled      : runChildDeleteMasterTest,
-        runLMATestEnabled                    : runLMATest,
-        runMgmtUserControllerTestEnabled     : runMgmtUserControllerTest,
-        runProxyChildTestEnabled             : runProxyChildTest,
-        fetchServiceBinariesEnabled          : fetchServiceBinaries,
-        awsOnDemandDemoEnabled               : awsOnDemandDemo,
-        equinixOnDemandDemoEnabled           : equinixOnDemandDemo,
-        equinixMetalV2OnDemandDemoEnabled    : equinixMetalV2OnDemandDemo,
-        equinixMetalV2ChildDiffMetroEnabled  : equinixMetalV2ChildDiffMetro,
-        equinixOnAwsDemoEnabled              : equinixOnAwsDemo,
-        azureOnDemandDemoEnabled             : azureOnDemandDemo,
-        azureOnAwsDemoEnabled                : azureOnAwsDemo,
-        vsphereDemoEnabled                   : enableVsphereDemo,
-        vsphereOnDemandDemoEnabled           : enableVsphereDemo, // TODO: remove after MCC 2.7 is out
-        bmDemoEnabled                        : enableBMDemo,
-        osDemoEnabled                        : enableOSDemo,
-        vsphereUbuntuEnabled                 : enableVsphereUbuntu,
-        artifactsBuildEnabled                : enableArtifactsBuild,
-        childOsBootFromVolume                : childOsBootFromVolume,
-        multiregionalConfiguration           : multiregionalMappings,
-        demoWeight                           : demoWeight,
-        bootstrapV2Scenario                  : bootstrapV2Scenario,
-        equinixMetalV2Metro                  : equinixMetalV2Metro]
+        osCloudLocation                          : openstackIMC,
+        cdnConfig                                : cdnConfig,
+        proxyConfig                              : proxyConfig,
+        useMacOsSeedNode                         : seedMacOs,
+        deployChildEnabled                       : deployChild,
+        childDeployCustomRelease                 : customChildRelease,
+        upgradeChildEnabled                      : upgradeChild,
+        fullUpgradeChildEnabled                  : fullUpgradeChild,
+        mosDeployChildEnabled                    : mosDeployChild,
+        mosUpgradeChildEnabled                   : mosUpgradeChild,
+        runChildConformanceEnabled               : runChildConformance,
+        attachBYOEnabled                         : attachBYO,
+        upgradeBYOEnabled                        : upgradeBYO,
+        runBYOMatrixEnabled                      : runBYOMatrix,
+        defaultBYOOs                             : defaultBYOOs,
+        upgradeMgmtEnabled                       : upgradeMgmt,
+        autoUpgradeMgmtEnabled                   : autoUpgradeMgmt,
+        enableLMALoggingEnabled                  : enableLMALogging,
+        deployOsOnMosEnabled                     : deployOsOnMos,
+        runUie2eEnabled                          : runUie2e,
+        runUie2eNewEnabled                       : runUie2eNew,
+        runMgmtConformanceEnabled                : runMgmtConformance,
+        runMaintenanceTestEnabled                : runMaintenanceTest,
+        runContainerregistryTestEnabled          : runContainerregistryTest,
+        runGracefulRebootTestEnabled             : runGracefulRebootTest,
+        pauseForDebugEnabled                     : pauseForDebug,
+        runMgmtDeleteMasterTestEnabled           : runMgmtDeleteMasterTest,
+        runRgnlDeleteMasterTestEnabled           : runRgnlDeleteMasterTest,
+        runChildDeleteMasterTestEnabled          : runChildDeleteMasterTest,
+        runChildMachineDeletionPolicyTestEnabled : runChildMachineDeletionPolicyTest,
+        runLMATestEnabled                        : runLMATest,
+        runMgmtUserControllerTestEnabled         : runMgmtUserControllerTest,
+        runProxyChildTestEnabled                 : runProxyChildTest,
+        fetchServiceBinariesEnabled              : fetchServiceBinaries,
+        awsOnDemandDemoEnabled                   : awsOnDemandDemo,
+        equinixOnDemandDemoEnabled               : equinixOnDemandDemo,
+        equinixMetalV2OnDemandDemoEnabled        : equinixMetalV2OnDemandDemo,
+        equinixMetalV2ChildDiffMetroEnabled      : equinixMetalV2ChildDiffMetro,
+        equinixOnAwsDemoEnabled                  : equinixOnAwsDemo,
+        azureOnDemandDemoEnabled                 : azureOnDemandDemo,
+        azureOnAwsDemoEnabled                    : azureOnAwsDemo,
+        vsphereDemoEnabled                       : enableVsphereDemo,
+        vsphereOnDemandDemoEnabled               : enableVsphereDemo, // TODO: remove after MCC 2.7 is out
+        bmDemoEnabled                            : enableBMDemo,
+        osDemoEnabled                            : enableOSDemo,
+        vsphereUbuntuEnabled                     : enableVsphereUbuntu,
+        artifactsBuildEnabled                    : enableArtifactsBuild,
+        childOsBootFromVolume                    : childOsBootFromVolume,
+        multiregionalConfiguration               : multiregionalMappings,
+        demoWeight                               : demoWeight,
+        bootstrapV2Scenario                      : bootstrapV2Scenario,
+        equinixMetalV2Metro                      : equinixMetalV2Metro,
+        enableFips                               : enableFips,
+        aioCluster                               : aioCluster]
 }
 
 /**
@@ -771,9 +807,11 @@
         booleanParam(name: 'RUN_MGMT_CFM', value: triggers.runMgmtConformanceEnabled),
         booleanParam(name: 'RUN_MAINTENANCE_TEST', value: triggers.runMaintenanceTestEnabled),
         booleanParam(name: 'RUN_CONTAINER_REGISTRY_TEST', value: triggers.runContainerregistryTestEnabled),
+        booleanParam(name: 'RUN_GRACEFUL_REBOOT_TEST', value: triggers.runGracefulRebootTestEnabled),
         booleanParam(name: 'RUN_MGMT_DELETE_MASTER_TEST', value: triggers.runMgmtDeleteMasterTestEnabled),
         booleanParam(name: 'RUN_RGNL_DELETE_MASTER_TEST', value: triggers.runRgnlDeleteMasterTestEnabled),
         booleanParam(name: 'RUN_CHILD_DELETE_MASTER_TEST', value: triggers.runChildDeleteMasterTestEnabled),
+        booleanParam(name: 'RUN_CHILD_MACHINE_DELETION_POLICY_TEST', value: triggers.runChildMachineDeletionPolicyTestEnabled),
         booleanParam(name: 'RUN_LMA_TEST', value: triggers.runLMATestEnabled),
         booleanParam(name: 'RUN_MGMT_USER_CONTROLLER_TEST', value: triggers.runMgmtUserControllerTestEnabled),
         booleanParam(name: 'DEPLOY_CHILD_CLUSTER', value: triggers.deployChildEnabled),
@@ -792,6 +830,9 @@
         booleanParam(name: 'ALLOW_AZURE_ON_DEMAND', value: triggers.azureOnDemandDemoEnabled),
         booleanParam(name: 'AZURE_ON_AWS_DEMO', value: triggers.azureOnAwsDemoEnabled),
         booleanParam(name: 'VSPHERE_DEPLOY_UBUNTU', value: triggers.vsphereUbuntuEnabled),
+        booleanParam(name: 'PAUSE_FOR_DEBUG', value: triggers.pauseForDebugEnabled),
+        booleanParam(name: 'ENABLE_FIPS', value: triggers.enableFips),
+        booleanParam(name: 'AIO_CLUSTER', value: triggers.aioCluster),
     ]
 
     // customize multiregional demo
@@ -1273,7 +1314,8 @@
         while (!freeNodes) {
             freeNodes = jenkins.model.Jenkins.instance.computers.findAll { node ->
                 label in node.getAssignedLabels().collect { it.name } &&
-                        node.isPartiallyIdle()
+                        node.isPartiallyIdle() &&
+                        node.isOnline()
             }
             if (!freeNodes) {
                 echo 'No nodes available for scheduling, retrying...'
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index f552fc3..74a1ce9 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -60,6 +60,7 @@
         'warlock>1.3.1;python_version=="3.5"',
         // NOTE: pin client packages to current latest to prevent
         // downloading packages which are not support Python 2.7
+        'keystoneauth1<=5.0.0',
         'python-openstackclient==4.0.0',
         'python-ironicclient==3.1.2',
         'openstacksdk<0.44.0',
diff --git a/src/com/mirantis/mk/ReleaseWorkflow.groovy b/src/com/mirantis/mk/ReleaseWorkflow.groovy
index da17fb9..63a3675 100644
--- a/src/com/mirantis/mk/ReleaseWorkflow.groovy
+++ b/src/com/mirantis/mk/ReleaseWorkflow.groovy
@@ -62,6 +62,10 @@
     checkoutReleaseMetadataRepo(params)
 
     docker.image(appDockerImage).inside("--volume ${repoDir}:/workspace") {
+        checkResult = sh(script: "metadata-app --path /workspace/metadata ${opts} validate --structure", returnStatus: true)
+        if (checkResult != 0) {
+            throw new Exception("Invalid metadata structure, see errors in the metadata.log file")
+        }
         result = sh(script: "metadata-app --path /workspace/metadata ${opts} get --key ${key}", returnStdout: true).trim()
     }
     common.infoMsg("""
@@ -106,6 +110,7 @@
     String changeAuthorName     = params.get('crAuthorName', 'MCP-CI')
     String changeAuthorEmail    = params.get('crAuthorEmail', 'mcp-ci-jenkins@ci.mcp.mirantis.net')
     Boolean valuesFromFile      = params.get('valuesFromFile', false)
+    Boolean catMetadataLog       = params.get('catMetadataLog', false)
 
     def cred = common.getCredentials(gitCredentialsId, 'key')
     String gerritUser = cred.username
@@ -161,9 +166,19 @@
                     }
                     try {
                         sh "metadata-app --path /workspace/metadata update --create --key '${keyArr[i]}' ${valueExpression}"
+                        checkResult = sh(script: "metadata-app --path /workspace/metadata validate --structure", returnStatus: true)
+                        def MetadataLogFile = readFile("metadata.log")
+                        if (MetadataLogFile.size() > 0) {
+                            if (catMetadataLog) {
+                                common.errorMsg("Invalid metadata structure, metadata.log file content:\n${MetadataLogFile}")
+                            }
+                        }
+                        if (checkResult != 0) {
+                            throw new Exception("Invalid metadata structure, see errors in the metadata.log file")
+                        }
                     } finally {
-                        if (valuesFromFile){
-                            sh "rm ${tmpFile}"
+                        if (valuesFromFile) {
+                            sh "rm -v ${tmpFile}"
                         }
                     }
                 }
diff --git a/src/com/mirantis/mk/Workflow.groovy b/src/com/mirantis/mk/Workflow.groovy
index 44203ab..f231337 100644
--- a/src/com/mirantis/mk/Workflow.groovy
+++ b/src/com/mirantis/mk/Workflow.groovy
@@ -542,6 +542,24 @@
         list_id += 1
     }
 
+    def pause_step_id = list_id
+    for (step in scenario['pause']) {
+        def display_name = step['job']
+        if (step['description'] != null && step['description'].toString() != "") {
+            display_name = step['description']
+        }
+        jobs_data.add([list_id   : "$list_id",
+                       type      : "pause",
+                       name      : "$display_name",
+                       build_url : "0",
+                       build_id  : "-",
+                       status    : "-",
+                       desc      : "",
+                       child_desc: "",
+                       duration  : '-'])
+        list_id += 1
+    }
+
     def finally_step_id = list_id
     for (step in scenario['finally']) {
         def display_name = step['job']
@@ -559,18 +577,31 @@
                        duration  : '-'])
         list_id += 1
     }
-
+    def job_failed_flag = false
     try {
         // Run the 'workflow' jobs
         runSteps(scenario['workflow'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
     } catch (InterruptedException x) {
+        job_failed_flag = true
         error "The job was aborted"
     } catch (e) {
+        job_failed_flag = true
         error("Build failed: " + e.toString())
+
     } finally {
-        // Switching to 'finally' step index
+        flag_pause_variable = (env.PAUSE_FOR_DEBUG) != null
+        // Run the 'finally' or 'pause' jobs
+        common.infoMsg(failed_jobs)
+        if (flag_pause_variable && (PAUSE_FOR_DEBUG && (job_failed_flag || failed_jobs))) {
+            // Switching to 'pause' step index
+            common.infoMsg("FINALLY BLOCK - PAUSE")
+            step_id = pause_step_id
+            runSteps(scenario['pause'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
+
+        }
+         // Switching to 'finally' step index
+        common.infoMsg("FINALLY BLOCK - CLEAR")
         step_id = finally_step_id
-        // Run the 'finally' jobs
         runSteps(scenario['finally'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
 
         if (failed_jobs) {