Merge "Fix partition removal in Ceph pipelines, make cleaning data partition optional"
diff --git a/ceph-remove-node.groovy b/ceph-remove-node.groovy
index 18b03fb..771a399 100644
--- a/ceph-remove-node.groovy
+++ b/ceph-remove-node.groovy
@@ -19,6 +19,8 @@
 def python = new com.mirantis.mk.Python()
 def pepperEnv = "pepperEnv"
 
+def cleanDisk = CLEANDISK
+
 timeout(time: 12, unit: 'HOURS') {
     node("python") {
 
@@ -79,7 +81,8 @@
                 salt.cmdRun(pepperEnv, "${targetProvider}", "virsh destroy ${target}.${domain}")
                 salt.cmdRun(pepperEnv, "${targetProvider}", "virsh undefine ${target}.${domain}")
             }
-        } else if (HOST_TYPE.toLowerCase() == 'osd') {
+        }
+        else if (HOST_TYPE.toLowerCase() == 'osd') {
             def osd_ids = []
             def device_grain_name =  "ceph_disk"
             // get list of osd disks of the host
@@ -132,65 +135,107 @@
             }
 
             for (osd_id in osd_ids) {
-
                 id = osd_id.replaceAll('osd.', '')
-                def dmcrypt = ""
-                try {
-                    dmcrypt = salt.cmdRun(pepperEnv, HOST, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep dmcrypt")['return'][0].values()[0]
-                } catch (Exception e) {
-                    common.warningMsg(e)
-                }
 
-                if (dmcrypt?.trim()) {
-                    mount = salt.cmdRun(pepperEnv, HOST, "lsblk -rp | grep /var/lib/ceph/osd/ceph-${id} -B1")['return'][0].values()[0]
-                    dev = mount.split()[0].replaceAll("[0-9]", "")
-
-                    // remove partition tables
-                    stage("dd part table on ${dev}") {
-                        salt.cmdRun(pepperEnv, HOST, "dd if=/dev/zero of=${dev} bs=512 count=1 conv=notrunc")
-                    }
-
-                }
                 // remove journal, block_db, block_wal partition `parted /dev/sdj rm 3`
                 stage('Remove journal / block_db / block_wal partition') {
                     def partition_uuid = ""
                     def journal_partition_uuid = ""
                     def block_db_partition_uuid = ""
                     def block_wal_partition_uuid = ""
-                    try {
-                        journal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep journal | grep partuuid")
-                        journal_partition_uuid = journal_partition_uuid.toString().trim().split("\n")[0].substring(journal_partition_uuid.toString().trim().lastIndexOf("/") + 1)
-                    } catch (Exception e) {
-                        common.infoMsg(e)
+                    def ceph_version = salt.getPillar(pepperEnv, HOST, 'ceph:common:ceph_version').get('return')[0].values()[0]
+
+                    if (ceph_version == "luminous") {
+                        try {
+                            journal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/journal_uuid")['return'][0].values()[0].split("\n")[0]
+                        }
+                        catch(Exception e) {
+                            common.infoMsg(e)
+                        }
+                        try {
+                            block_db_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.db_uuid")['return'][0].values()[0].split("\n")[0]
+                        }
+                        catch(Exception e) {
+                            common.infoMsg(e)
+                        }
+                        try {
+                            block_wal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.wal_uuid")['return'][0].values()[0].split("\n")[0]
+                        }
+                        catch(Exception e) {
+                            common.infoMsg(e)
+                        }
                     }
-                    try {
-                        block_db_partition_uuid = salt.cmdRun(pepperEnv, HOST, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep 'block.db' | grep partuuid")
-                        block_db_partition_uuid = block_db_partition_uuid.toString().trim().split("\n")[0].substring(block_db_partition_uuid.toString().trim().lastIndexOf("/") + 1)
-                    } catch (Exception e) {
-                        common.infoMsg(e)
+                    else {
+                        def volumes = salt.cmdRun(pepperEnv, HOST, "ceph-volume lvm list --format=json", checkResponse=true, batch=null, output=false)
+                        volumes = new groovy.json.JsonSlurperClassic().parseText(volumes['return'][0].values()[0])
+
+                        block_db_partition_uuid = volumes[id][0]['tags'].get('ceph.db_uuid')
+                        block_wal_partition_uuid = volumes[id][0]['tags'].get('ceph.wal_uuid')
                     }
 
-                    try {
-                        block_wal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep 'block.wal' | grep partuuid")
-                        block_wal_partition_uuid = block_wal_partition_uuid.toString().trim().split("\n")[0].substring(block_wal_partition_uuid.toString().trim().lastIndexOf("/") + 1)
-                    } catch (Exception e) {
-                        common.infoMsg(e)
-                    }
 
-                    // set partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
                     if (journal_partition_uuid?.trim()) {
-                        partition_uuid = journal_partition_uuid
-                    } else if (block_db_partition_uuid?.trim()) {
-                        partition_uuid = block_db_partition_uuid
+                        ceph.removePartition(pepperEnv, HOST, journal_partition_uuid)
                     }
-
-                    // if disk has journal, block_db or block_wal on different disk, then remove the partition
-                    if (partition_uuid?.trim()) {
-                        ceph.removePartition(pepperEnv, HOST, partition_uuid)
+                    if (block_db_partition_uuid?.trim()) {
+                        ceph.removePartition(pepperEnv, HOST, block_db_partition_uuid)
                     }
                     if (block_wal_partition_uuid?.trim()) {
                         ceph.removePartition(pepperEnv, HOST, block_wal_partition_uuid)
                     }
+
+                    try {
+                        salt.cmdRun(pepperEnv, HOST, "partprobe")
+                    } catch (Exception e) {
+                        common.warningMsg(e)
+                    }
+                }
+
+                if (cleanDisk) {
+                // remove data / block / lockbox partition `parted /dev/sdj rm 3`
+                    stage('Remove data / block / lockbox partition') {
+                        def data_partition_uuid = ""
+                        def block_partition_uuid = ""
+                        def osd_fsid = ""
+                        def lvm = ""
+                        def lvm_enabled= salt.getPillar(pepperEnv,"I@ceph:osd","ceph:osd:lvm_enabled")['return'].first().containsValue(true)
+                        try {
+                            osd_fsid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/fsid")['return'][0].values()[0].split("\n")[0]
+                            if (lvm_enabled) {
+                                lvm = salt.runSaltCommand(pepperEnv, 'local', ['expression': HOST, 'type': 'compound'], 'cmd.run', null, "salt-call lvm.lvdisplay --output json -l quiet")['return'][0].values()[0]
+                                lvm = new groovy.json.JsonSlurperClassic().parseText(lvm)
+                                lvm["local"].each { lv, params ->
+                                    if (params["Logical Volume Name"].contains(osd_fsid)) {
+                                        data_partition_uuid = params["Logical Volume Name"].minus("/dev/")
+                                    }
+                                }
+                            } else {
+                                data_partition_uuid = osd_fsid
+                            }
+                        } catch (Exception e) {
+                            common.infoMsg(e)
+                        }
+                        try {
+                            block_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block_uuid")['return'][0].values()[0].split("\n")[0]
+                        }
+                        catch (Exception e) {
+                            common.infoMsg(e)
+                        }
+
+                        // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
+                        if (block_partition_uuid?.trim()) {
+                            ceph.removePartition(pepperEnv, HOST, block_partition_uuid)
+                            try {
+                                salt.cmdRun(pepperEnv, HOST, "ceph-volume lvm zap `readlink /var/lib/ceph/osd/ceph-${id}/block` --destroy")
+                            }
+                            catch (Exception e) {
+                                common.infoMsg(e)
+                            }
+                        }
+                        if (data_partition_uuid?.trim()) {
+                            ceph.removePartition(pepperEnv, HOST, data_partition_uuid, 'data', id)
+                        }
+                    }
                 }
             }
 
@@ -213,37 +258,18 @@
                 salt.cmdRun(pepperEnv, HOST, "mv /etc/salt/minion.d/minion.conf minion.conf")
                 salt.runSaltProcessStep(pepperEnv, HOST, 'service.stop', ['salt-minion'], [], null, true, 5)
             }
-        }
 
-        stage('Remove salt-key') {
-            try {
-                salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key -d ${target}.${domain} -y")
-            } catch (Exception e) {
-                common.warningMsg(e)
-            }
-            try {
-                salt.cmdRun(pepperEnv, 'I@salt:master', "rm /srv/salt/reclass/nodes/_generated/${target}.${domain}.yml")
-            } catch (Exception e) {
-                common.warningMsg(e)
-            }
-        }
-
-        stage('Remove keyring') {
-            def keyring = ""
-            def keyring_lines = ""
-            try {
-                keyring_lines = salt.cmdRun(pepperEnv, ADMIN_HOST, "ceph auth list | grep ${target}")['return'][0].values()[0].split('\n')
-            } catch (Exception e) {
-                common.warningMsg(e)
-            }
-            for (line in keyring_lines) {
-                if (line.toLowerCase().contains(target.toLowerCase())) {
-                    keyring = line
-                    break
+            stage('Remove salt-key') {
+                try {
+                    salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key -d ${target}.${domain} -y")
+                } catch (Exception e) {
+                    common.warningMsg(e)
                 }
-            }
-            if (keyring?.trim()) {
-                salt.cmdRun(pepperEnv, ADMIN_HOST, "ceph auth del ${keyring}")
+                try {
+                    salt.cmdRun(pepperEnv, 'I@salt:master', "rm /srv/salt/reclass/nodes/_generated/${target}.${domain}.yml")
+                } catch (Exception e) {
+                    common.warningMsg(e)
+                }
             }
         }
 
@@ -260,7 +286,6 @@
             }
 
             def target_hosts = salt.getMinions(pepperEnv, 'I@ceph:common')
-            print target_hosts
 
             // Update configs
             stage('Update Ceph configs') {
diff --git a/ceph-remove-osd.groovy b/ceph-remove-osd.groovy
index 2788dcb..0dbb8d9 100644
--- a/ceph-remove-osd.groovy
+++ b/ceph-remove-osd.groovy
@@ -22,6 +22,7 @@
 def pepperEnv = "pepperEnv"
 def flags = CLUSTER_FLAGS.tokenize(',')
 def osds = OSD.tokenize(',')
+def cleanDisk = CLEANDISK
 
 timeout(time: 12, unit: 'HOURS') {
     node("python") {
@@ -122,18 +123,21 @@
                 def block_wal_partition_uuid = ""
                 try {
                     journal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/journal_uuid")['return'][0].values()[0].split("\n")[0]
-                } catch (Exception e) {
+                }
+                 catch (Exception e) {
                     common.infoMsg(e)
                 }
                 try {
                     block_db_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.db_uuid")['return'][0].values()[0].split("\n")[0]
-                } catch (Exception e) {
+                }
+                catch (Exception e) {
                     common.infoMsg(e)
                 }
 
                 try {
                     block_wal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.wal_uuid")['return'][0].values()[0].split("\n")[0]
-                } catch (Exception e) {
+                }
+                catch (Exception e) {
                     common.infoMsg(e)
                 }
 
@@ -150,44 +154,54 @@
 
                 try {
                     salt.cmdRun(pepperEnv, HOST, "partprobe")
-                } catch (Exception e) {
+                }
+                catch (Exception e) {
                     common.warningMsg(e)
                 }
             }
-
-            // remove data / block / lockbox partition `parted /dev/sdj rm 3`
-            stage('Remove data / block / lockbox partition') {
-                def data_partition_uuid = ""
-                def block_partition_uuid = ""
-                def osd_fsid = ""
-                def lvm = ""
-                def lvm_enabled= salt.getPillar(pepperEnv,"I@ceph:osd","ceph:osd:lvm_enabled")['return'].first().containsValue(true)
-                try {
-                    osd_fsid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/fsid")['return'][0].values()[0].split("\n")[0]
-                    if (lvm_enabled) {
-                        lvm = salt.runSaltCommand(pepperEnv, 'local', ['expression': HOST, 'type': 'compound'], 'cmd.run', null, "salt-call lvm.lvdisplay --output json -l quiet")['return'][0].values()[0]
-                        lvm = new groovy.json.JsonSlurperClassic().parseText(lvm)
-                        lvm["local"].each { lv, params ->
-                            if (params["Logical Volume Name"].contains(osd_fsid)) {
-                                data_partition_uuid = params["Logical Volume Name"].minus("/dev/")
+            if (cleanDisk) {
+                // remove data / block / lockbox partition `parted /dev/sdj rm 3`
+                stage('Remove data / block / lockbox partition') {
+                    def data_partition_uuid = ""
+                    def block_partition_uuid = ""
+                    def osd_fsid = ""
+                    def lvm = ""
+                    def lvm_enabled= salt.getPillar(pepperEnv,"I@ceph:osd","ceph:osd:lvm_enabled")['return'].first().containsValue(true)
+                    try {
+                        osd_fsid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/fsid")['return'][0].values()[0].split("\n")[0]
+                        if (lvm_enabled) {
+                            lvm = salt.runSaltCommand(pepperEnv, 'local', ['expression': HOST, 'type': 'compound'], 'cmd.run', null, "salt-call lvm.lvdisplay --output json -l quiet")['return'][0].values()[0]
+                            lvm = new groovy.json.JsonSlurperClassic().parseText(lvm)
+                            lvm["local"].each { lv, params ->
+                                if (params["Logical Volume Name"].contains(osd_fsid)) {
+                                    data_partition_uuid = params["Logical Volume Name"].minus("/dev/")
+                                }
                             }
                         }
                     }
-                } catch (Exception e) {
-                    common.infoMsg(e)
-                }
-                try {
-                    block_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block_uuid")['return'][0].values()[0].split("\n")[0]
-                } catch (Exception e) {
-                    common.infoMsg(e)
-                }
+                    catch (Exception e) {
+                        common.infoMsg(e)
+                    }
+                    try {
+                        block_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block_uuid")['return'][0].values()[0].split("\n")[0]
+                    }
+                    catch (Exception e) {
+                        common.infoMsg(e)
+                    }
 
-                // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
-                if (block_partition_uuid?.trim()) {
-                    ceph.removePartition(pepperEnv, HOST, block_partition_uuid)
-                }
-                if (data_partition_uuid?.trim()) {
-                    ceph.removePartition(pepperEnv, HOST, data_partition_uuid, 'data', id)
+                    // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
+                    if (block_partition_uuid?.trim()) {
+                        ceph.removePartition(pepperEnv, HOST, block_partition_uuid)
+                        try{
+                            salt.cmdRun(pepperEnv, HOST, "ceph-volume lvm zap `readlink /var/lib/ceph/osd/ceph-${id}/block` --destroy")
+                        }
+                        catch (Exception e) {
+                            common.infoMsg(e)
+                        }
+                    }
+                    if (data_partition_uuid?.trim()) {
+                        ceph.removePartition(pepperEnv, HOST, data_partition_uuid, 'data', id)
+                    }
                 }
             }
         }