blob: 098fb9833b0d51b52acd37d553bb8ea0c98c4054 [file] [log] [blame]
Tomáš Kukrálf72096d2017-08-11 12:58:03 +02001/**
2 *
3 * Remove OSD from existing cluster
4 *
5 * Requred parameters:
6 * SALT_MASTER_URL URL of Salt master
7 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
8 *
9 * HOST Host (minion id) to be removed
Jiri Broulik2c00f4c2017-10-26 13:23:11 +020010 * OSD Comma separated list of osd ids to be removed
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020011 * ADMIN_HOST Host (minion id) with admin keyring
12 * CLUSTER_FLAGS Comma separated list of tags to apply to cluster
13 * WAIT_FOR_HEALTHY Wait for cluster rebalance before stoping daemons
14 *
15 */
16
17common = new com.mirantis.mk.Common()
18salt = new com.mirantis.mk.Salt()
chnyda625f4b42017-10-11 14:10:31 +020019def python = new com.mirantis.mk.Python()
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020020
chnyda625f4b42017-10-11 14:10:31 +020021def pepperEnv = "pepperEnv"
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020022def flags = CLUSTER_FLAGS.tokenize(',')
Tomáš Kukrál9d6228b2017-08-15 16:54:55 +020023def osds = OSD.tokenize(',')
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020024
Jiri Broulikeb7b82f2017-11-30 13:55:40 +010025
Jiri Broulika5bc8f62018-01-31 15:04:40 +010026def removePartition(master, target, partition_uuid, type='', id=-1) {
27 def partition = ""
28 if (type == 'lockbox') {
29 try {
30 // umount - partition = /dev/sdi2
31 partition = runCephCommand(master, target, "lsblk -rp | grep -v mapper | grep ${partition_uuid} ")['return'][0].values()[0].split()[0]
32 runCephCommand(master, target, "umount ${partition}")
33 } catch (Exception e) {
34 common.warningMsg(e)
35 }
36 } else if (type == 'data') {
37 try {
38 // umount - partition = /dev/sdi2
39 partition = runCephCommand(master, target, "df | grep /var/lib/ceph/osd/ceph-${id}")['return'][0].values()[0].split()[0]
40 runCephCommand(master, target, "umount ${partition}")
41 } catch (Exception e) {
42 common.warningMsg(e)
43 }
44 try {
45 // partition = /dev/sdi2
46 partition = runCephCommand(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
47 } catch (Exception e) {
48 common.warningMsg(e)
49 }
50 } else {
51 try {
52 // partition = /dev/sdi2
53 partition = runCephCommand(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split(":")[0]
54 } catch (Exception e) {
55 common.warningMsg(e)
56 }
57 }
Jiri Broulikeb7b82f2017-11-30 13:55:40 +010058 if (partition?.trim()) {
Mateusz Los06659072019-09-02 18:28:55 +020059 if (partition.contains("nvme")) {
60 // dev = /dev/nvme1n1p1
61 def dev = partition.replaceAll('\\d+$', "")
62 print("Skipping " + dev)
63 // part_id = 2
64 def part_id = partition.substring(partition.lastIndexOf("p")+1).replaceAll("[^0-9]+", "")
65 print("Skipping" + part_id)
66 runCephCommand(master, target, "Ignore | parted ${dev} rm ${part_id}")
67 }
68 else {
69 // dev = /dev/sdi
70 def dev = partition.replaceAll('\\d+$', "")
71 // part_id = 2
72 def part_id = partition.substring(partition.lastIndexOf("/")+1).replaceAll("[^0-9]+", "")
73 runCephCommand(master, target, "Ignore | parted ${dev} rm ${part_id}")
74 }
Jiri Broulikeb7b82f2017-11-30 13:55:40 +010075 }
76 return
77}
78
79def runCephCommand(master, target, cmd) {
80 return salt.cmdRun(master, target, cmd)
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020081}
82
Jiri Broulik96c867a2017-11-07 16:14:10 +010083def waitForHealthy(master, count=0, attempts=300) {
84 // wait for healthy cluster
85 while (count<attempts) {
86 def health = runCephCommand(master, ADMIN_HOST, 'ceph health')['return'][0].values()[0]
87 if (health.contains('HEALTH_OK')) {
88 common.infoMsg('Cluster is healthy')
89 break;
90 }
91 count++
92 sleep(10)
93 }
94}
Jakub Josefa63f9862018-01-11 17:58:38 +010095timeout(time: 12, unit: 'HOURS') {
96 node("python") {
Jiri Broulik96c867a2017-11-07 16:14:10 +010097
Jakub Josefa63f9862018-01-11 17:58:38 +010098 // create connection to salt master
99 python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
Tomáš Kukrálf72096d2017-08-11 12:58:03 +0200100
Jakub Josefa63f9862018-01-11 17:58:38 +0100101 if (flags.size() > 0) {
102 stage('Set cluster flags') {
103 for (flag in flags) {
104 runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd set ' + flag)
105 }
Tomáš Kukrálf72096d2017-08-11 12:58:03 +0200106 }
107 }
Tomáš Kukrálf72096d2017-08-11 12:58:03 +0200108
Jakub Josefa63f9862018-01-11 17:58:38 +0100109 def osd_ids = []
Tomáš Kukrálf72096d2017-08-11 12:58:03 +0200110
Jakub Josefa63f9862018-01-11 17:58:38 +0100111 // get list of osd disks of the host
112 salt.runSaltProcessStep(pepperEnv, HOST, 'saltutil.sync_grains', [], null, true, 5)
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100113 def cephGrain = salt.getGrain(pepperEnv, HOST, 'ceph')
114
Jakub Josefed670ca2018-01-18 14:22:20 +0100115 if(cephGrain['return'].isEmpty()){
116 throw new Exception("Ceph salt grain cannot be found!")
117 }
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100118 common.print(cephGrain)
Jakub Josefed670ca2018-01-18 14:22:20 +0100119 def ceph_disks = cephGrain['return'][0].values()[0].values()[0]['ceph_disk']
Jakub Josefa63f9862018-01-11 17:58:38 +0100120 common.prettyPrint(ceph_disks)
Jiri Broulikadc7ecd2017-10-18 06:59:27 +0200121
Jakub Josefa63f9862018-01-11 17:58:38 +0100122 for (i in ceph_disks) {
123 def osd_id = i.getKey().toString()
124 if (osd_id in osds || OSD == '*') {
125 osd_ids.add('osd.' + osd_id)
126 print("Will delete " + osd_id)
127 } else {
128 print("Skipping " + osd_id)
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100129 }
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100130 }
Jakub Josefa63f9862018-01-11 17:58:38 +0100131
132 // wait for healthy cluster
Mateusz Los63811962019-03-28 14:12:23 +0100133 // if (WAIT_FOR_HEALTHY.toBoolean()) {
134 // waitForHealthy(pepperEnv)
135 // }
136
137 if ( osd_ids == [] )
138 {
139 currentBuild.result = 'SUCCESS'
140 return
Jakub Josefa63f9862018-01-11 17:58:38 +0100141 }
142
143 // `ceph osd out <id> <id>`
144 stage('Set OSDs out') {
145 runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd out ' + osd_ids.join(' '))
146 }
147
148 // wait for healthy cluster
Jakub Josefed670ca2018-01-18 14:22:20 +0100149 if (WAIT_FOR_HEALTHY.toBoolean()) {
Jakub Josefa63f9862018-01-11 17:58:38 +0100150 sleep(5)
151 waitForHealthy(pepperEnv)
152 }
153
154 // stop osd daemons
155 stage('Stop OSD daemons') {
156 for (i in osd_ids) {
157 salt.runSaltProcessStep(pepperEnv, HOST, 'service.stop', ['ceph-osd@' + i.replaceAll('osd.', '')], null, true)
158 }
159 }
160
161 // `ceph osd crush remove osd.2`
162 stage('Remove OSDs from CRUSH') {
163 for (i in osd_ids) {
164 runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd crush remove ' + i)
165 }
166 }
167
168 // remove keyring `ceph auth del osd.3`
169 stage('Remove OSD keyrings from auth') {
170 for (i in osd_ids) {
171 runCephCommand(pepperEnv, ADMIN_HOST, 'ceph auth del ' + i)
172 }
173 }
174
175 // remove osd `ceph osd rm osd.3`
176 stage('Remove OSDs') {
177 for (i in osd_ids) {
178 runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd rm ' + i)
179 }
180 }
181
182 for (osd_id in osd_ids) {
Jakub Josefa63f9862018-01-11 17:58:38 +0100183 id = osd_id.replaceAll('osd.', '')
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100184 /*
185
Jakub Josefa63f9862018-01-11 17:58:38 +0100186 def dmcrypt = ""
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100187 try {
Jakub Josefa63f9862018-01-11 17:58:38 +0100188 dmcrypt = runCephCommand(pepperEnv, HOST, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep dmcrypt")['return'][0].values()[0]
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100189 } catch (Exception e) {
Jakub Josefa63f9862018-01-11 17:58:38 +0100190 common.warningMsg(e)
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100191 }
192
Jakub Josefa63f9862018-01-11 17:58:38 +0100193 if (dmcrypt?.trim()) {
194 mount = runCephCommand(pepperEnv, HOST, "lsblk -rp | grep /var/lib/ceph/osd/ceph-${id} -B1")['return'][0].values()[0]
195 dev = mount.split()[0].replaceAll("[0-9]","")
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100196
Jakub Josefa63f9862018-01-11 17:58:38 +0100197 // remove partition tables
198 stage("dd part table on ${dev}") {
199 runCephCommand(pepperEnv, HOST, "dd if=/dev/zero of=${dev} bs=512 count=1 conv=notrunc")
200 }
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100201
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100202 }
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100203 */
204
Jakub Josefa63f9862018-01-11 17:58:38 +0100205 // remove journal, block_db, block_wal partition `parted /dev/sdj rm 3`
206 stage('Remove journal / block_db / block_wal partition') {
207 def partition_uuid = ""
208 def journal_partition_uuid = ""
209 def block_db_partition_uuid = ""
210 def block_wal_partition_uuid = ""
211 try {
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100212 journal_partition_uuid = runCephCommand(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/journal_uuid")['return'][0].values()[0].split("\n")[0]
Jakub Josefa63f9862018-01-11 17:58:38 +0100213 } catch (Exception e) {
214 common.infoMsg(e)
215 }
216 try {
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100217 block_db_partition_uuid = runCephCommand(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.db_uuid")['return'][0].values()[0].split("\n")[0]
Jakub Josefa63f9862018-01-11 17:58:38 +0100218 } catch (Exception e) {
219 common.infoMsg(e)
220 }
221
222 try {
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100223 block_wal_partition_uuid = runCephCommand(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.wal_uuid")['return'][0].values()[0].split("\n")[0]
Jakub Josefa63f9862018-01-11 17:58:38 +0100224 } catch (Exception e) {
225 common.infoMsg(e)
226 }
227
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100228 // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
Jakub Josefa63f9862018-01-11 17:58:38 +0100229 if (journal_partition_uuid?.trim()) {
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100230 removePartition(pepperEnv, HOST, journal_partition_uuid)
Jakub Josefa63f9862018-01-11 17:58:38 +0100231 }
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100232 if (block_db_partition_uuid?.trim()) {
233 removePartition(pepperEnv, HOST, block_db_partition_uuid)
Jakub Josefa63f9862018-01-11 17:58:38 +0100234 }
235 if (block_wal_partition_uuid?.trim()) {
236 removePartition(pepperEnv, HOST, block_wal_partition_uuid)
237 }
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100238
239 try {
240 runCephCommand(pepperEnv, HOST, "partprobe")
241 } catch (Exception e) {
242 common.warningMsg(e)
243 }
244 }
245
246 // remove data / block / lockbox partition `parted /dev/sdj rm 3`
247 stage('Remove data / block / lockbox partition') {
248 def data_partition_uuid = ""
249 def block_partition_uuid = ""
250 def lockbox_partition_uuid = ""
251 try {
252 data_partition_uuid = runCephCommand(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/fsid")['return'][0].values()[0].split("\n")[0]
253 common.print(data_partition_uuid)
254 } catch (Exception e) {
255 common.infoMsg(e)
256 }
257 try {
258 block_partition_uuid = runCephCommand(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block_uuid")['return'][0].values()[0].split("\n")[0]
259 } catch (Exception e) {
260 common.infoMsg(e)
261 }
262
263 try {
264 lockbox_partition_uuid = data_partition_uuid
265 } catch (Exception e) {
266 common.infoMsg(e)
267 }
268
269 // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
270 if (block_partition_uuid?.trim()) {
271 removePartition(pepperEnv, HOST, block_partition_uuid)
272 }
273 if (data_partition_uuid?.trim()) {
274 removePartition(pepperEnv, HOST, data_partition_uuid, 'data', id)
275 }
276 if (lockbox_partition_uuid?.trim()) {
277 removePartition(pepperEnv, HOST, lockbox_partition_uuid, 'lockbox')
278 }
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100279 }
280 }
Jakub Josefa63f9862018-01-11 17:58:38 +0100281 // remove cluster flags
282 if (flags.size() > 0) {
283 stage('Unset cluster flags') {
284 for (flag in flags) {
285 common.infoMsg('Removing flag ' + flag)
286 runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd unset ' + flag)
287 }
Tomáš Kukrálf72096d2017-08-11 12:58:03 +0200288 }
289 }
290 }
Tomáš Kukrálf72096d2017-08-11 12:58:03 +0200291}