Merge "Switch keystone_chek param in false in restore pipeline for maas Related-Prod:PROD-36455" into release/proposed/2019.2.0
diff --git a/openstack-database-cleanup.groovy b/openstack-database-cleanup.groovy
index 0654d54..879780b 100644
--- a/openstack-database-cleanup.groovy
+++ b/openstack-database-cleanup.groovy
@@ -13,7 +13,7 @@
 def salt = new com.mirantis.mk.Salt()
 def python = new com.mirantis.mk.Python()
 
-def os_services = [ 'nova:controller', 'heat:server', 'cinder:controller' ]
+def os_services = [ 'nova:controller', 'heat:server', 'cinder:controller', 'glance:server' ]
 
 def slave_node = 'python'
 
diff --git a/pre-upgrade-verify.groovy b/pre-upgrade-verify.groovy
new file mode 100644
index 0000000..d4e8e3b
--- /dev/null
+++ b/pre-upgrade-verify.groovy
@@ -0,0 +1,58 @@
+salt = new com.mirantis.mk.Salt()
+python = new com.mirantis.mk.Python()
+common = new com.mirantis.mk.Common()
+venvPepper = "venvPepper"
+upgradeChecks = new com.mirantis.mcp.UpgradeChecks()
+
+reportDir = 'reportDir/'
+waList =['check_34406', 'check_34645', 'check_35705', 'check_35884', 'check_36461', 'check_36461_2']
+
+timeout(time: PIPELINE_TIMEOUT, unit: 'HOURS') {
+    node('python') {
+        try {
+            python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            def clusterName = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
+            def reportContent = ""
+            stage('Start checking work-arounds') {
+                for (wa in waList) {
+                    //false - return status of check, true - raise exception
+                    def waData = upgradeChecks."$wa"(salt, venvPepper, clusterName, false)
+                    def reportTemplate = """
+      <tr>
+        <td class='row'>${waData.prodId}</td>
+        <td class='row'>${waData.isFixed}</td>
+        <td class='row'>${waData.waInfo}</td>
+      </tr>"""
+                    reportContent = "${reportContent}${reportTemplate}"
+                }
+            }
+            stage('Generate report') {
+                sh "rm -rf ${reportDir}"
+                sh "mkdir -p ${reportDir}"
+                def reportHead = """<html>
+  <head>
+    <title>WA verify report</title>
+  </head>
+  <body>
+    <h1>WA verify report</h1>
+    <table border='1' cellpadding='5' cellspacing='0' style='border-collapse:collapse'>
+      <tr>
+        <th class='row'>Prod id</th>
+        <th class='row'>Status</th>
+        <th class='row'>Doc link</th>
+      </tr>"""
+                def reportTail = """
+    </table>
+  </body>
+</html>"""
+                writeFile file: "${reportDir}report.html", text: "${reportHead}${reportContent}${reportTail}"
+                archiveArtifacts artifacts: "${reportDir}/*"
+            }
+        } catch (Throwable e) {
+            // If there was an error or exception thrown, the build failed
+            currentBuild.result = "FAILURE"
+            currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+            throw e
+        }
+    }
+}
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 93389a0..d0f1a97 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -29,6 +29,7 @@
 def saltMastCreds = ''
 def packageUpgradeMode = ''
 batchSize = ''
+upgradeChecks = new com.mirantis.mcp.UpgradeChecks()
 
 def fullRefreshOneByOne(venvPepper, minions) {
     for (minion in minions) {
@@ -239,90 +240,6 @@
     }
 }
 
-def check_34406(String cluster_name) {
-    def sphinxpasswordPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:sphinx_proxy_password_generated').get("return")[0].values()[0]
-    if (sphinxpasswordPillar == '' || sphinxpasswordPillar == 'null' || sphinxpasswordPillar == null) {
-        error('Sphinx password is not defined.\n' +
-        'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-9/mu-9-addressed/mu-9-dtrain/mu-9-dt-manual.html#i-34406 for more info')
-    }
-}
-
-def check_34645(String cluster_name) {
-    def updatecellsPillar = salt.getPillar(venvPepper, 'I@nova:controller', 'nova:controller:update_cells').get("return")[0].values()[0]
-    if (updatecellsPillar.toString().toLowerCase() == 'false') {
-        error('Update cells disabled.\n' +
-        'See https://docs.mirantis.com/mcp/q4-18/mcp-operations-guide/openstack-operations/disable-nova-cell-mapping.html')
-    }
-}
-
-def check_35705(String cluster_name) {
-    def galeracheckpasswordPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:galera_clustercheck_password').get("return")[0].values()[0]
-    if (galeracheckpasswordPillar == '' || galeracheckpasswordPillar == 'null' || galeracheckpasswordPillar == null) {
-        error('Galera clustercheck password is not defined.\n' +
-        'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#improper-operation-of-galera-ha for more info')
-    }
-}
-
-def check_35884(String cluster_name) {
-    if (salt.getMinions(venvPepper, 'I@prometheus:alerta or I@prometheus:alertmanager')) {
-        def alertaApiKeyGenPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:alerta_admin_api_key_generated').get("return")[0].values()[0]
-        def alertaApiKeyPillar = salt.getPillar(venvPepper, 'I@prometheus:alerta or I@prometheus:alertmanager', '_param:alerta_admin_key').get("return")[0].values()[0]
-
-        if (alertaApiKeyGenPillar == '' || alertaApiKeyGenPillar == 'null' || alertaApiKeyGenPillar == null || alertaApiKeyPillar == '' || alertaApiKeyPillar == 'null' || alertaApiKeyPillar == null) {
-            error('Alerta admin API key not defined.\n' +
-            'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#i-35884 for more info')
-        }
-    }
-}
-
-// ceph cluster class ordering for radosgw
-def check_36461(String cluster_name){
-    if (!salt.testTarget(venvPepper, 'I@ceph:radosgw')) {
-        return
-    }
-    def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
-    def checkFile = "${clusterModelPath}/ceph/rgw.yml"
-    def saltTarget = "I@salt:master"
-    try {
-        salt.cmdRun(venvPepper, saltTarget, "test -f ${checkFile}")
-    }
-    catch (Exception e) {
-        common.warningMsg("Unable to check ordering of RadosGW imports, file ${checkFile} not found, skipping")
-        return
-    }
-    def fileContent = salt.cmdRun(venvPepper, saltTarget, "cat ${checkFile}").get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
-    def yamlData = readYaml text: fileContent
-    def infraClassImport = "cluster.${cluster_name}.infra"
-    def cephClassImport = "cluster.${cluster_name}.ceph"
-    def cephCommonClassImport = "cluster.${cluster_name}.ceph.common"
-    def infraClassFound = false
-    def importErrorDetected = false
-    def importErrorMessage = """Ceph classes in '${checkFile}' are used in wrong order! Please reorder it:
-'${infraClassImport}' should be placed before '${cephClassImport}' and '${cephCommonClassImport}'.
-For additional information please see https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-15/mu-15-addressed/mu-15-dtrain/mu-15-dtrain-manual.html"""
-    for (yamlClass in yamlData.classes) {
-        switch(yamlClass){
-          case infraClassImport:
-            infraClassFound = true;
-            break;
-          case cephClassImport:
-            if (!infraClassFound) {
-              importErrorDetected = true
-            };
-            break;
-          case cephCommonClassImport:
-            if (!infraClassFound) {
-              importErrorDetected = true
-            };
-            break;
-        }
-    }
-    if (importErrorDetected) {
-        common.errorMsg(importErrorMessage)
-        error(importErrorMessage)
-    }
-}
-
 def wa32182(String cluster_name) {
     if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
         def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
@@ -734,11 +651,12 @@
                 common.infoMsg('Perform: Full salt sync')
                 fullRefreshOneByOne(venvPepper, allMinions)
 
-                check_34406(cluster_name)
-                check_34645(cluster_name)
-                check_35705(cluster_name)
-                check_35884(cluster_name)
-                check_36461(cluster_name)
+                upgradeChecks.check_34406(salt, venvPepper, cluster_name, true)
+                upgradeChecks.check_34645(salt, venvPepper, cluster_name, true)
+                upgradeChecks.check_35705(salt, venvPepper, cluster_name, true)
+                upgradeChecks.check_35884(salt, venvPepper, cluster_name, true)
+                upgradeChecks.check_36461(salt, venvPepper, cluster_name, true)
+                upgradeChecks.check_36461_2(salt, venvPepper, cluster_name, true)
 
                 common.infoMsg('Perform: Validate reclass medata before processing')
                 validateReclassModel(minions, 'before')