Merge "Methods for execute openstack commands in docker container have been added"
diff --git a/src/com/mirantis/mcp/UpgradeChecks.groovy b/src/com/mirantis/mcp/UpgradeChecks.groovy
new file mode 100644
index 0000000..60ab9a6
--- /dev/null
+++ b/src/com/mirantis/mcp/UpgradeChecks.groovy
@@ -0,0 +1,154 @@
+package com.mirantis.mcp
+
+/**
+ * Run check with parameters
+ *
+ * @param salt                com.mirantis.mk.Salt object
+ * @param venvPepper          venvPepper
+ * @param cluster_name        MCP cluster name
+ * @param raise_exc           Raise exception or return status of check
+**/
+
+def check_34406(salt, venvPepper, String cluster_name, Boolean raise_exc) {
+    def sphinxpasswordPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:sphinx_proxy_password_generated').get("return")[0].values()[0]
+    def waStatus = [prodId: "PROD-34406", isFixed: "", waInfo: ""]
+    if (sphinxpasswordPillar == '' || sphinxpasswordPillar == 'null' || sphinxpasswordPillar == null) {
+        waStatus.isFixed = "Work-around should be applied manually"
+        waStatus.waInfo = "See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-9/mu-9-addressed/mu-9-dtrain/mu-9-dt-manual.html#i-34406 for more info"
+        if (raise_exc) {
+            error('Sphinx password is not defined.\n' +
+            waStatus.waInfo)
+        }
+        return waStatus
+    }
+    waStatus.isFixed = "Work-around for PROD-34406 already applied, nothing todo"
+    return waStatus
+}
+
+def check_34645(salt, venvPepper, String cluster_name, Boolean raise_exc) {
+    def updatecellsPillar = salt.getPillar(venvPepper, 'I@nova:controller', 'nova:controller:update_cells').get("return")[0].values()[0]
+    def waStatus = [prodId: "PROD-34645", isFixed: "", waInfo: ""]
+    if (updatecellsPillar.toString().toLowerCase() == 'false') {
+        waStatus.isFixed = "Work-around should be applied manually"
+        waStatus.waInfo = "See https://docs.mirantis.com/mcp/q4-18/mcp-operations-guide/openstack-operations/disable-nova-cell-mapping.html for more info"
+        if (raise_exc) {
+            error('Update cells disabled.\n' +
+            waStatus.waInfo)
+        }
+        return waStatus
+    }
+    waStatus.isFixed = "Work-around for PROD-34645 already applied, nothing todo"
+    return waStatus
+}
+
+def check_35705(salt, venvPepper, String cluster_name, Boolean raise_exc) {
+    def galeracheckpasswordPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:galera_clustercheck_password').get("return")[0].values()[0]
+    def waStatus = [prodId: "PROD-35705", isFixed: "", waInfo: ""]
+    if (galeracheckpasswordPillar == '' || galeracheckpasswordPillar == 'null' || galeracheckpasswordPillar == null) {
+        waStatus.isFixed = "Work-around should be applied manually"
+        waStatus.waInfo = "See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#improper-operation-of-galera-ha for more info"
+        if (raise_exc) {
+            error('Galera clustercheck password is not defined.\n' +
+            waStatus.waInfo)
+        }
+        return waStatus
+    }
+    waStatus.isFixed = "Work-around for PROD-35705 already applied, nothing todo"
+    return waStatus
+}
+
+def check_35884(salt, venvPepper, String cluster_name, Boolean raise_exc) {
+    def waStatus = [prodId: "PROD-35884", isFixed: "", waInfo: ""]
+    if (salt.getMinions(venvPepper, 'I@prometheus:alerta or I@prometheus:alertmanager')) {
+        def alertaApiKeyGenPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:alerta_admin_api_key_generated').get("return")[0].values()[0]
+        def alertaApiKeyPillar = salt.getPillar(venvPepper, 'I@prometheus:alerta or I@prometheus:alertmanager', '_param:alerta_admin_key').get("return")[0].values()[0]
+        if (alertaApiKeyGenPillar == '' || alertaApiKeyGenPillar == 'null' || alertaApiKeyGenPillar == null || alertaApiKeyPillar == '' || alertaApiKeyPillar == 'null' || alertaApiKeyPillar == null) {
+            waStatus.isFixed = "Work-around should be applied manually"
+            waStatus.waInfo = "See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#i-35884 for more info"
+            if (raise_exc) {
+                error('Alerta admin API key not defined.\n' +
+                waStatus.waInfo)
+            }
+            return waStatus
+        }
+    }
+    waStatus.isFixed = "Work-around for PROD-35884 already applied, nothing todo"
+    return waStatus
+}
+
+def check_36461(salt, venvPepper, String cluster_name, Boolean raise_exc){
+    def common = new com.mirantis.mk.Common()
+    def waStatus = [prodId: "PROD-36461", isFixed: "", waInfo: ""]
+    if (!salt.testTarget(venvPepper, 'I@ceph:radosgw')) {
+        return
+    }
+    def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
+    def checkFile = "${clusterModelPath}/ceph/rgw.yml"
+    def saltTarget = "I@salt:master"
+    try {
+        salt.cmdRun(venvPepper, saltTarget, "test -f ${checkFile}")
+    }
+    catch (Exception e) {
+        waStatus.isFixed = "Check skipped"
+        waStatus.waInfo = "Unable to check ordering of RadosGW imports, file ${checkFile} not found, skipping"
+        if (raise_exc) {
+            common.warningMsg(waStatus.waInfo)
+            return
+        }
+        return waStatus
+    }
+    def fileContent = salt.cmdRun(venvPepper, saltTarget, "cat ${checkFile}").get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+    def yamlData = readYaml text: fileContent
+    def infraClassImport = "cluster.${cluster_name}.infra"
+    def cephClassImport = "cluster.${cluster_name}.ceph"
+    def cephCommonClassImport = "cluster.${cluster_name}.ceph.common"
+    def infraClassFound = false
+    def importErrorDetected = false
+    def importErrorMessage = """Ceph classes in '${checkFile}' are used in wrong order! Please reorder it:
+'${infraClassImport}' should be placed before '${cephClassImport}' and '${cephCommonClassImport}'.
+For additional information please see https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-15/mu-15-addressed/mu-15-dtrain/mu-15-dtrain-manual.html"""
+    for (yamlClass in yamlData.classes) {
+        switch(yamlClass){
+          case infraClassImport:
+            infraClassFound = true;
+            break;
+          case cephClassImport:
+            if (!infraClassFound) {
+              importErrorDetected = true
+            };
+            break;
+          case cephCommonClassImport:
+            if (!infraClassFound) {
+              importErrorDetected = true
+            };
+            break;
+        }
+    }
+    if (importErrorDetected) {
+        waStatus.isFixed = "Work-around should be applied manually"
+        waStatus.waInfo = importErrorMessage
+        if (raise_exc) {
+            common.errorMsg(importErrorMessage)
+            error(importErrorMessage)
+        }
+        return waStatus
+    }
+    waStatus.isFixed = "Work-around for PROD-36461 already applied, nothing todo"
+    return waStatus
+}
+
+def check_36461_2 (salt, venvPepper, String cluster_name, Boolean raise_exc) {
+    def cephMonPillar = salt.getPillar(venvPepper, 'I@ceph:mon', 'ceph:common:config:mon:auth_allow_insecure_global_id_reclaim').get("return")[0].values()[0]
+    def waStatus = [prodId: "PROD-36461_2", isFixed: "", waInfo: ""]
+    if (cephMonPillar.toString().toLowerCase() != 'false') {
+        waStatus.isFixed = "Work-around should be applied manually"
+        waStatus.waInfo = "See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/single/index.html#i-cve-2021-20288 for more info"
+        if (raise_exc) {
+            error('Needed option is not set.\n' +
+            waStatus.waInfo)
+        }
+        return waStatus
+    }
+    waStatus.isFixed = "Work-around for PROD-36461_2 already applied, nothing todo"
+    return waStatus
+}
diff --git a/src/com/mirantis/mk/KaasUtils.groovy b/src/com/mirantis/mk/KaasUtils.groovy
index 3f1b1c6..29d3cff 100644
--- a/src/com/mirantis/mk/KaasUtils.groovy
+++ b/src/com/mirantis/mk/KaasUtils.groovy
@@ -54,6 +54,7 @@
     def customChildRelease = env.KAAS_CHILD_CLUSTER_RELEASE_NAME ? env.KAAS_CHILD_CLUSTER_RELEASE_NAME : ''
     def attachBYO = env.ATTACH_BYO ? env.ATTACH_BYO.toBoolean() : false
     def upgradeBYO = env.UPGRADE_BYO ? env.UPGRADE_BYO.toBoolean() : false
+    def runBYOMatrix = env.RUN_BYO_MATRIX ? env.RUN_BYO_MATRIX.toBoolean() : false
     def upgradeMgmt = env.UPGRADE_MGMT_CLUSTER ? env.UPGRADE_MGMT_CLUSTER.toBoolean() : false
     def enableLMALogging = env.ENABLE_LMA_LOGGING ? env.ENABLE_LMA_LOGGING.toBoolean(): false
     def runUie2e = env.RUN_UI_E2E ? env.RUN_UI_E2E.toBoolean() : false
@@ -62,6 +63,7 @@
     def runMgmtUserControllerTest = env.RUN_MGMT_USER_CONTROLLER_TEST ? env.RUN_MGMT_USER_CONTROLLER_TEST.toBoolean() : false
     def runChildConformance = env.RUN_CHILD_CFM ? env.RUN_CHILD_CFM.toBoolean() : false
     def fetchServiceBinaries = env.FETCH_BINARIES_FROM_UPSTREAM ? env.FETCH_BINARIES_FROM_UPSTREAM.toBoolean() : false
+    def equinixMetalV2ChildDiffMetro = env.EQUINIXMETALV2_CHILD_DIFF_METRO ? env.EQUINIXMETALV2_CHILD_DIFF_METRO.toBoolean() : false
     // multiregion configuration from env variable: comma-separated string in form $mgmt_provider,$regional_provider
     def multiregionalMappings = env.MULTIREGION_SETUP ? multiregionWorkflowParser(env.MULTIREGION_SETUP) : [
         enabled: false,
@@ -80,7 +82,6 @@
     def awsOnDemandDemo = env.ALLOW_AWS_ON_DEMAND ? env.ALLOW_AWS_ON_DEMAND.toBoolean() : false
     def equinixOnDemandDemo = env.ALLOW_EQUINIX_ON_DEMAND ? env.ALLOW_EQUINIX_ON_DEMAND.toBoolean() : false
     def equinixMetalV2OnDemandDemo = env.ALLOW_EQUINIXMETALV2_ON_DEMAND ? env.ALLOW_EQUINIXMETALV2_ON_DEMAND.toBoolean() : false
-    def equinixPrivateDemo = env.EQUINIX_PRIVATE_DEMO ? env.EQUINIX_PRIVATE_DEMO.toBoolean() : false
     def equinixOnAwsDemo = env.EQUINIX_ON_AWS_DEMO ? env.EQUINIX_ON_AWS_DEMO.toBoolean() : false
     def azureOnAwsDemo = env.AZURE_ON_AWS_DEMO ? env.AZURE_ON_AWS_DEMO.toBoolean() : false
     def azureOnDemandDemo = env.ALLOW_AZURE_ON_DEMAND ? env.ALLOW_AZURE_ON_DEMAND.toBoolean() : false
@@ -117,6 +118,13 @@
         attachBYO = true
         upgradeBYO = true
     }
+    if (commitMsg ==~ /(?s).*\[run-byo-matrix\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*run-byo-matrix\.*/) {
+        runBYOMatrix = true
+
+        common.warningMsg('Forced byo matrix test via run-byo-matrix, all other byo triggers will be skipped')
+        attachBYO = false
+        upgradeBYO = false
+    }
     if (commitMsg ==~ /(?s).*\[mgmt-upgrade\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*mgmt-upgrade.*/) {
         upgradeMgmt = true
     }
@@ -165,6 +173,7 @@
         env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*aws-demo.*/ ||
         attachBYO                                            ||
         upgradeBYO                                           ||
+        runBYOMatrix                                         ||
         seedMacOs                                            ||
         equinixOnAwsDemo                                     ||
         azureOnAwsDemo) {
@@ -178,10 +187,9 @@
     if (commitMsg ==~ /(?s).*\[equinixmetalv2-demo\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*equinixmetalv2-demo\.*/) {
         equinixMetalV2OnDemandDemo = true
     }
-    // below will be deprecated after v2
-    if (commitMsg ==~ /(?s).*\[equinix-private-demo\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*equinix-private-demo\.*/) {
-        equinixOnDemandDemo = true
-        equinixPrivateDemo = true
+    if (commitMsg ==~ /(?s).*\[equinixmetalv2-child-diff-metro\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*equinixmetalv2-child-diff-metro\.*/) {
+        equinixMetalV2OnDemandDemo = true
+        equinixMetalV2ChildDiffMetro = true
     }
     if (commitMsg ==~ /(?s).*\[azure-demo\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*azure-demo\.*/) {
         azureOnDemandDemo = true
@@ -268,8 +276,9 @@
         Custom child cluster release: ${customChildRelease}
         Child cluster release upgrade scheduled: ${upgradeChild}
         Child conformance testing scheduled: ${runChildConformance}
-        BYO cluster attachment scheduled: ${attachBYO}
-        Attached BYO cluster upgrade test scheduled: ${upgradeBYO}
+        Single BYO cluster attachment scheduled: ${attachBYO}
+        Single Attached BYO cluster upgrade test scheduled: ${upgradeBYO}
+        BYO test matrix whole suite scheduled: ${runBYOMatrix}
         Mgmt cluster release upgrade scheduled: ${upgradeMgmt}
         Mgmt LMA logging enabled: ${enableLMALogging}
         Mgmt conformance testing scheduled: ${runMgmtConformance}
@@ -279,7 +288,7 @@
         AWS provider deployment scheduled: ${awsOnDemandDemo}
         Equinix provider deployment scheduled: ${equinixOnDemandDemo}
         EquinixmetalV2 provider deployment scheduled: ${equinixMetalV2OnDemandDemo}
-        Equinix provider private MCC deployment: ${equinixPrivateDemo}
+        EquinixmetalV2 child deploy in a separate metro scheduled: ${equinixMetalV2ChildDiffMetro}
         Equinix@AWS child cluster deployment scheduled: ${equinixOnAwsDemo}
         Azure provider deployment scheduled: ${azureOnDemandDemo}
         Azure@AWS child cluster deployment scheduled: ${azureOnAwsDemo}
@@ -291,35 +300,36 @@
         Current weight of the demo run: ${demoWeight} (Used to manage lockable resources)
         Triggers: https://gerrit.mcp.mirantis.com/plugins/gitiles/kaas/core/+/refs/heads/master/hack/ci-gerrit-keywords.md""")
     return [
-        cdnConfig                        : cdnConfig,
-        proxyConfig                      : proxyConfig,
-        useMacOsSeedNode                 : seedMacOs,
-        deployChildEnabled               : deployChild,
-        childDeployCustomRelease         : customChildRelease,
-        upgradeChildEnabled              : upgradeChild,
-        runChildConformanceEnabled       : runChildConformance,
-        attachBYOEnabled                 : attachBYO,
-        upgradeBYOEnabled                : upgradeBYO,
-        upgradeMgmtEnabled               : upgradeMgmt,
-        enableLMALoggingEnabled          : enableLMALogging,
-        runUie2eEnabled                  : runUie2e,
-        runMgmtConformanceEnabled        : runMgmtConformance,
-        runLMATestEnabled                : runLMATest,
-        runMgmtUserControllerTestEnabled : runMgmtUserControllerTest,
-        fetchServiceBinariesEnabled      : fetchServiceBinaries,
-        awsOnDemandDemoEnabled           : awsOnDemandDemo,
-        equinixOnDemandDemoEnabled       : equinixOnDemandDemo,
-        equinixMetalV2OnDemandDemoEnabled: equinixMetalV2OnDemandDemo,
-        equinixPrivateDemoEnabled        : equinixPrivateDemo,
-        equinixOnAwsDemoEnabled          : equinixOnAwsDemo,
-        azureOnDemandDemoEnabled         : azureOnDemandDemo,
-        azureOnAwsDemoEnabled            : azureOnAwsDemo,
-        vsphereDemoEnabled               : enableVsphereDemo,
-        vsphereOnDemandDemoEnabled       : enableVsphereDemo, // TODO: remove after MCC 2.7 is out
-        bmDemoEnabled                    : enableBMDemo,
-        osDemoEnabled                    : enableOSDemo,
-        multiregionalConfiguration       : multiregionalMappings,
-        demoWeight                       : demoWeight]
+        cdnConfig                            : cdnConfig,
+        proxyConfig                          : proxyConfig,
+        useMacOsSeedNode                     : seedMacOs,
+        deployChildEnabled                   : deployChild,
+        childDeployCustomRelease             : customChildRelease,
+        upgradeChildEnabled                  : upgradeChild,
+        runChildConformanceEnabled           : runChildConformance,
+        attachBYOEnabled                     : attachBYO,
+        upgradeBYOEnabled                    : upgradeBYO,
+        runBYOMatrixEnabled                  : runBYOMatrix,
+        upgradeMgmtEnabled                   : upgradeMgmt,
+        enableLMALoggingEnabled              : enableLMALogging,
+        runUie2eEnabled                      : runUie2e,
+        runMgmtConformanceEnabled            : runMgmtConformance,
+        runLMATestEnabled                    : runLMATest,
+        runMgmtUserControllerTestEnabled     : runMgmtUserControllerTest,
+        fetchServiceBinariesEnabled          : fetchServiceBinaries,
+        awsOnDemandDemoEnabled               : awsOnDemandDemo,
+        equinixOnDemandDemoEnabled           : equinixOnDemandDemo,
+        equinixMetalV2OnDemandDemoEnabled    : equinixMetalV2OnDemandDemo,
+        equinixMetalV2ChildDiffMetroEnabled  : equinixMetalV2ChildDiffMetro,
+        equinixOnAwsDemoEnabled              : equinixOnAwsDemo,
+        azureOnDemandDemoEnabled             : azureOnDemandDemo,
+        azureOnAwsDemoEnabled                : azureOnAwsDemo,
+        vsphereDemoEnabled                   : enableVsphereDemo,
+        vsphereOnDemandDemoEnabled           : enableVsphereDemo, // TODO: remove after MCC 2.7 is out
+        bmDemoEnabled                        : enableBMDemo,
+        osDemoEnabled                        : enableOSDemo,
+        multiregionalConfiguration           : multiregionalMappings,
+        demoWeight                           : demoWeight]
 }
 
 /**
@@ -583,11 +593,12 @@
         booleanParam(name: 'UPGRADE_CHILD_CLUSTER', value: triggers.upgradeChildEnabled),
         booleanParam(name: 'ATTACH_BYO', value: triggers.attachBYOEnabled),
         booleanParam(name: 'UPGRADE_BYO', value: triggers.upgradeBYOEnabled),
+        booleanParam(name: 'RUN_BYO_MATRIX', value: triggers.runBYOMatrixEnabled),
         booleanParam(name: 'RUN_CHILD_CFM', value: triggers.runChildConformanceEnabled),
         booleanParam(name: 'ALLOW_AWS_ON_DEMAND', value: triggers.awsOnDemandDemoEnabled),
         booleanParam(name: 'ALLOW_EQUINIX_ON_DEMAND', value: triggers.equinixOnDemandDemoEnabled),
         booleanParam(name: 'ALLOW_EQUINIXMETALV2_ON_DEMAND', value: triggers.equinixMetalV2OnDemandDemoEnabled),
-        booleanParam(name: 'EQUINIX_PRIVATE_DEMO', value: triggers.equinixPrivateDemoEnabled),
+        booleanParam(name: 'EQUINIXMETALV2_CHILD_DIFF_METRO', value: triggers.equinixMetalV2ChildDiffMetroEnabled),
         booleanParam(name: 'EQUINIX_ON_AWS_DEMO', value: triggers.equinixOnAwsDemoEnabled),
         booleanParam(name: 'ALLOW_AZURE_ON_DEMAND', value: triggers.azureOnDemandDemoEnabled),
         booleanParam(name: 'AZURE_ON_AWS_DEMO', value: triggers.azureOnAwsDemoEnabled),
@@ -607,21 +618,6 @@
         parameters.addAll(additionalParameters)
     }
 
-    if (triggers.awsOnDemandDemoEnabled     ||
-        triggers.equinixOnDemandDemoEnabled ||
-        triggers.azureOnDemandDemoEnabled) {
-
-        common.infoMsg('Public provider demo triggered, need to sync artifacts in the public-ci cdn..')
-        switch (component) {
-            case 'iam':
-                build job: 'cdn-binary-dev-replication-iam', propagate: true, wait: true
-                break
-            case 'lcm':
-                build job: 'cdn-binary-dev-replication-lcm', propagate: true, wait: true
-                break
-        }
-    }
-
     def jobResults = []
 
     platforms.each { platform ->