Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 1 | /** |
| 2 | * |
| 3 | * Remove OSD from existing cluster |
| 4 | * |
| 5 | * Requred parameters: |
| 6 | * SALT_MASTER_URL URL of Salt master |
| 7 | * SALT_MASTER_CREDENTIALS Credentials to the Salt API |
| 8 | * |
| 9 | * HOST Host (minion id) to be removed |
Jiri Broulik | 2c00f4c | 2017-10-26 13:23:11 +0200 | [diff] [blame] | 10 | * OSD Comma separated list of osd ids to be removed |
Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 11 | * ADMIN_HOST Host (minion id) with admin keyring |
| 12 | * CLUSTER_FLAGS Comma separated list of tags to apply to cluster |
| 13 | * WAIT_FOR_HEALTHY Wait for cluster rebalance before stoping daemons |
| 14 | * |
| 15 | */ |
| 16 | |
Ivan Berezovskiy | 2325dcb | 2019-11-05 17:42:57 +0400 | [diff] [blame] | 17 | def common = new com.mirantis.mk.Common() |
| 18 | def salt = new com.mirantis.mk.Salt() |
| 19 | def ceph = new com.mirantis.mk.Ceph() |
chnyda | 625f4b4 | 2017-10-11 14:10:31 +0200 | [diff] [blame] | 20 | def python = new com.mirantis.mk.Python() |
Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 21 | |
chnyda | 625f4b4 | 2017-10-11 14:10:31 +0200 | [diff] [blame] | 22 | def pepperEnv = "pepperEnv" |
Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 23 | def flags = CLUSTER_FLAGS.tokenize(',') |
Tomáš Kukrál | 9d6228b | 2017-08-15 16:54:55 +0200 | [diff] [blame] | 24 | def osds = OSD.tokenize(',') |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 25 | def cleanDisk = CLEANDISK |
Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 26 | |
Jakub Josef | a63f986 | 2018-01-11 17:58:38 +0100 | [diff] [blame] | 27 | timeout(time: 12, unit: 'HOURS') { |
| 28 | node("python") { |
Jiri Broulik | 96c867a | 2017-11-07 16:14:10 +0100 | [diff] [blame] | 29 | |
Jakub Josef | a63f986 | 2018-01-11 17:58:38 +0100 | [diff] [blame] | 30 | // create connection to salt master |
| 31 | python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS) |
Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 32 | |
Jakub Josef | a63f986 | 2018-01-11 17:58:38 +0100 | [diff] [blame] | 33 | def osd_ids = [] |
Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 34 | |
Mateusz Los | b5663e6 | 2020-05-11 09:51:13 +0200 | [diff] [blame] | 35 | def checknode = salt.runSaltProcessStep(pepperEnv, HOST, 'test.ping') |
| 36 | if (checknode['return'][0].values().isEmpty()) { |
| 37 | common.errorMsg("Host not found") |
| 38 | throw new InterruptedException() |
| 39 | } |
| 40 | |
Jakub Josef | a63f986 | 2018-01-11 17:58:38 +0100 | [diff] [blame] | 41 | // get list of osd disks of the host |
| 42 | salt.runSaltProcessStep(pepperEnv, HOST, 'saltutil.sync_grains', [], null, true, 5) |
Jiri Broulik | a5bc8f6 | 2018-01-31 15:04:40 +0100 | [diff] [blame] | 43 | def cephGrain = salt.getGrain(pepperEnv, HOST, 'ceph') |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 44 | def ceph_disks = cephGrain['return'][0].values()[0].values()[0]['ceph_disk'] |
Jiri Broulik | a5bc8f6 | 2018-01-31 15:04:40 +0100 | [diff] [blame] | 45 | |
Ivan Berezovskiy | 2325dcb | 2019-11-05 17:42:57 +0400 | [diff] [blame] | 46 | if (cephGrain['return'].isEmpty()) { |
Jakub Josef | ed670ca | 2018-01-18 14:22:20 +0100 | [diff] [blame] | 47 | throw new Exception("Ceph salt grain cannot be found!") |
| 48 | } |
Jiri Broulik | adc7ecd | 2017-10-18 06:59:27 +0200 | [diff] [blame] | 49 | |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 50 | if (flags.size() > 0) { |
| 51 | stage('Set cluster flags') { |
| 52 | for (flag in flags) { |
| 53 | salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd set ' + flag) |
Jiri Broulik | a5bc8f6 | 2018-01-31 15:04:40 +0100 | [diff] [blame] | 54 | } |
| 55 | } |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 56 | } |
| 57 | |
| 58 | try { |
| 59 | for (i in ceph_disks) { |
| 60 | def osd_id = i.getKey().toString() |
| 61 | if (osd_id in osds || OSD == '*') { |
| 62 | osd_ids.add('osd.' + osd_id) |
| 63 | print("Will delete " + osd_id) |
| 64 | } else { |
| 65 | print("Skipping " + osd_id) |
| 66 | } |
| 67 | } |
| 68 | |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 69 | // `ceph osd out <id> <id>` |
| 70 | stage('Set OSDs out') { |
Tomek Jaroszyk | 7c836ba | 2020-09-07 15:02:03 +0200 | [diff] [blame^] | 71 | if ( !osd_ids.isEmpty() ) { |
| 72 | salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd out ' + osd_ids.join(' ')) |
| 73 | } |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 74 | } |
| 75 | |
| 76 | // wait for healthy cluster |
| 77 | if (WAIT_FOR_HEALTHY.toBoolean()) { |
| 78 | sleep(5) |
| 79 | ceph.waitForHealthy(pepperEnv, ADMIN_HOST) |
| 80 | } |
| 81 | |
| 82 | // stop osd daemons |
| 83 | stage('Stop OSD daemons') { |
| 84 | for (i in osd_ids) { |
| 85 | salt.runSaltProcessStep(pepperEnv, HOST, 'service.stop', ['ceph-osd@' + i.replaceAll('osd.', '')], null, true) |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | // `ceph osd crush remove osd.2` |
| 90 | stage('Remove OSDs from CRUSH') { |
| 91 | for (i in osd_ids) { |
| 92 | salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd crush remove ' + i) |
| 93 | } |
| 94 | } |
| 95 | |
| 96 | // remove keyring `ceph auth del osd.3` |
| 97 | stage('Remove OSD keyrings from auth') { |
| 98 | for (i in osd_ids) { |
| 99 | salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph auth del ' + i) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // remove osd `ceph osd rm osd.3` |
| 104 | stage('Remove OSDs') { |
| 105 | for (i in osd_ids) { |
| 106 | salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd rm ' + i) |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | for (osd_id in osd_ids) { |
| 111 | id = osd_id.replaceAll('osd.', '') |
| 112 | |
| 113 | // remove journal, block_db, block_wal partition `parted /dev/sdj rm 3` |
| 114 | stage('Remove journal / block_db / block_wal partition') { |
| 115 | def partition_uuid = "" |
| 116 | def journal_partition_uuid = "" |
| 117 | def block_db_partition_uuid = "" |
| 118 | def block_wal_partition_uuid = "" |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 119 | try { |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 120 | journal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/journal_uuid")['return'][0].values()[0].split("\n")[0] |
| 121 | } |
| 122 | catch (Exception e) { |
| 123 | common.infoMsg(e) |
| 124 | } |
| 125 | try { |
| 126 | block_db_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.db_uuid")['return'][0].values()[0].split("\n")[0] |
mjedynski | af258b0 | 2019-12-09 15:17:58 +0100 | [diff] [blame] | 127 | } |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 128 | catch (Exception e) { |
| 129 | common.infoMsg(e) |
| 130 | } |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 131 | |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 132 | try { |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 133 | block_wal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.wal_uuid")['return'][0].values()[0].split("\n")[0] |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 134 | } |
| 135 | catch (Exception e) { |
| 136 | common.infoMsg(e) |
| 137 | } |
Jiri Broulik | a5bc8f6 | 2018-01-31 15:04:40 +0100 | [diff] [blame] | 138 | |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 139 | // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932 |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 140 | if (journal_partition_uuid?.trim()) { |
| 141 | ceph.removePartition(pepperEnv, HOST, journal_partition_uuid) |
| 142 | } |
| 143 | if (block_db_partition_uuid?.trim()) { |
| 144 | ceph.removePartition(pepperEnv, HOST, block_db_partition_uuid) |
| 145 | } |
| 146 | if (block_wal_partition_uuid?.trim()) { |
| 147 | ceph.removePartition(pepperEnv, HOST, block_wal_partition_uuid) |
| 148 | } |
| 149 | |
| 150 | try { |
| 151 | salt.cmdRun(pepperEnv, HOST, "partprobe") |
| 152 | } |
| 153 | catch (Exception e) { |
| 154 | common.warningMsg(e) |
| 155 | } |
| 156 | } |
| 157 | if (cleanDisk) { |
| 158 | // remove data / block / lockbox partition `parted /dev/sdj rm 3` |
| 159 | stage('Remove data / block / lockbox partition') { |
| 160 | def data_partition_uuid = "" |
| 161 | def block_partition_uuid = "" |
| 162 | def osd_fsid = "" |
| 163 | def lvm = "" |
| 164 | def lvm_enabled= salt.getPillar(pepperEnv,"I@ceph:osd","ceph:osd:lvm_enabled")['return'].first().containsValue(true) |
| 165 | try { |
| 166 | osd_fsid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/fsid")['return'][0].values()[0].split("\n")[0] |
| 167 | if (lvm_enabled) { |
| 168 | lvm = salt.runSaltCommand(pepperEnv, 'local', ['expression': HOST, 'type': 'compound'], 'cmd.run', null, "salt-call lvm.lvdisplay --output json -l quiet")['return'][0].values()[0] |
| 169 | lvm = new groovy.json.JsonSlurperClassic().parseText(lvm) |
| 170 | lvm["local"].each { lv, params -> |
| 171 | if (params["Logical Volume Name"].contains(osd_fsid)) { |
| 172 | data_partition_uuid = params["Logical Volume Name"].minus("/dev/") |
| 173 | } |
| 174 | } |
| 175 | } |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 176 | } |
| 177 | catch (Exception e) { |
| 178 | common.infoMsg(e) |
| 179 | } |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 180 | try { |
| 181 | block_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block_uuid")['return'][0].values()[0].split("\n")[0] |
| 182 | } |
| 183 | catch (Exception e) { |
| 184 | common.infoMsg(e) |
| 185 | } |
| 186 | |
| 187 | // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932 |
| 188 | if (block_partition_uuid?.trim()) { |
| 189 | ceph.removePartition(pepperEnv, HOST, block_partition_uuid) |
| 190 | try{ |
| 191 | salt.cmdRun(pepperEnv, HOST, "ceph-volume lvm zap `readlink /var/lib/ceph/osd/ceph-${id}/block` --destroy") |
| 192 | } |
| 193 | catch (Exception e) { |
| 194 | common.infoMsg(e) |
| 195 | } |
| 196 | } |
| 197 | if (data_partition_uuid?.trim()) { |
| 198 | ceph.removePartition(pepperEnv, HOST, data_partition_uuid, 'data', id) |
| 199 | } |
Mateusz Los | 1535aa2 | 2020-05-11 10:33:58 +0200 | [diff] [blame] | 200 | } |
Jiri Broulik | a5bc8f6 | 2018-01-31 15:04:40 +0100 | [diff] [blame] | 201 | } |
Jiri Broulik | eb7b82f | 2017-11-30 13:55:40 +0100 | [diff] [blame] | 202 | } |
| 203 | } |
Tomek Jaroszyk | e46eb69 | 2020-07-07 12:07:40 +0200 | [diff] [blame] | 204 | finally { |
| 205 | // remove cluster flags |
| 206 | if (flags.size() > 0) { |
| 207 | stage('Unset cluster flags') { |
| 208 | for (flag in flags) { |
| 209 | common.infoMsg('Removing flag ' + flag) |
| 210 | salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd unset ' + flag) |
| 211 | } |
Jakub Josef | a63f986 | 2018-01-11 17:58:38 +0100 | [diff] [blame] | 212 | } |
Tomáš Kukrál | f72096d | 2017-08-11 12:58:03 +0200 | [diff] [blame] | 213 | } |
| 214 | } |
| 215 | } |
Tomek Jaroszyk | 0cb8392 | 2020-04-07 11:20:31 +0200 | [diff] [blame] | 216 | } |