blob: b39434b6839dd5bf9f9414beb8af35ae9b2870fa [file] [log] [blame]
Tomáš Kukrálf72096d2017-08-11 12:58:03 +02001/**
2 *
3 * Remove OSD from existing cluster
4 *
5 * Requred parameters:
6 * SALT_MASTER_URL URL of Salt master
7 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
8 *
9 * HOST Host (minion id) to be removed
Jiri Broulik2c00f4c2017-10-26 13:23:11 +020010 * OSD Comma separated list of osd ids to be removed
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020011 * ADMIN_HOST Host (minion id) with admin keyring
12 * CLUSTER_FLAGS Comma separated list of tags to apply to cluster
13 * WAIT_FOR_HEALTHY Wait for cluster rebalance before stoping daemons
14 *
15 */
16
Ivan Berezovskiy2325dcb2019-11-05 17:42:57 +040017def common = new com.mirantis.mk.Common()
18def salt = new com.mirantis.mk.Salt()
19def ceph = new com.mirantis.mk.Ceph()
chnyda625f4b42017-10-11 14:10:31 +020020def python = new com.mirantis.mk.Python()
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020021
chnyda625f4b42017-10-11 14:10:31 +020022def pepperEnv = "pepperEnv"
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020023def flags = CLUSTER_FLAGS.tokenize(',')
Tomáš Kukrál9d6228b2017-08-15 16:54:55 +020024def osds = OSD.tokenize(',')
Mateusz Los1535aa22020-05-11 10:33:58 +020025def cleanDisk = CLEANDISK
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020026
Jakub Josefa63f9862018-01-11 17:58:38 +010027timeout(time: 12, unit: 'HOURS') {
28 node("python") {
Jiri Broulik96c867a2017-11-07 16:14:10 +010029
Jakub Josefa63f9862018-01-11 17:58:38 +010030 // create connection to salt master
31 python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020032
Jakub Josefa63f9862018-01-11 17:58:38 +010033 def osd_ids = []
Tomáš Kukrálf72096d2017-08-11 12:58:03 +020034
Mateusz Losb5663e62020-05-11 09:51:13 +020035 def checknode = salt.runSaltProcessStep(pepperEnv, HOST, 'test.ping')
36 if (checknode['return'][0].values().isEmpty()) {
37 common.errorMsg("Host not found")
38 throw new InterruptedException()
39 }
40
Jakub Josefa63f9862018-01-11 17:58:38 +010041 // get list of osd disks of the host
42 salt.runSaltProcessStep(pepperEnv, HOST, 'saltutil.sync_grains', [], null, true, 5)
Jiri Broulika5bc8f62018-01-31 15:04:40 +010043 def cephGrain = salt.getGrain(pepperEnv, HOST, 'ceph')
Tomek Jaroszyke46eb692020-07-07 12:07:40 +020044 def ceph_disks = cephGrain['return'][0].values()[0].values()[0]['ceph_disk']
Jiri Broulika5bc8f62018-01-31 15:04:40 +010045
Ivan Berezovskiy2325dcb2019-11-05 17:42:57 +040046 if (cephGrain['return'].isEmpty()) {
Jakub Josefed670ca2018-01-18 14:22:20 +010047 throw new Exception("Ceph salt grain cannot be found!")
48 }
Jiri Broulikadc7ecd2017-10-18 06:59:27 +020049
Tomek Jaroszyke46eb692020-07-07 12:07:40 +020050 if (flags.size() > 0) {
51 stage('Set cluster flags') {
52 for (flag in flags) {
53 salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd set ' + flag)
Jiri Broulika5bc8f62018-01-31 15:04:40 +010054 }
55 }
Tomek Jaroszyke46eb692020-07-07 12:07:40 +020056 }
57
58 try {
59 for (i in ceph_disks) {
60 def osd_id = i.getKey().toString()
61 if (osd_id in osds || OSD == '*') {
62 osd_ids.add('osd.' + osd_id)
63 print("Will delete " + osd_id)
64 } else {
65 print("Skipping " + osd_id)
66 }
67 }
68
Tomek Jaroszyke46eb692020-07-07 12:07:40 +020069 // `ceph osd out <id> <id>`
70 stage('Set OSDs out') {
Tomek Jaroszyk7c836ba2020-09-07 15:02:03 +020071 if ( !osd_ids.isEmpty() ) {
72 salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd out ' + osd_ids.join(' '))
73 }
Tomek Jaroszyke46eb692020-07-07 12:07:40 +020074 }
75
76 // wait for healthy cluster
77 if (WAIT_FOR_HEALTHY.toBoolean()) {
78 sleep(5)
79 ceph.waitForHealthy(pepperEnv, ADMIN_HOST)
80 }
81
82 // stop osd daemons
83 stage('Stop OSD daemons') {
84 for (i in osd_ids) {
85 salt.runSaltProcessStep(pepperEnv, HOST, 'service.stop', ['ceph-osd@' + i.replaceAll('osd.', '')], null, true)
86 }
87 }
88
89 // `ceph osd crush remove osd.2`
90 stage('Remove OSDs from CRUSH') {
91 for (i in osd_ids) {
92 salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd crush remove ' + i)
93 }
94 }
95
96 // remove keyring `ceph auth del osd.3`
97 stage('Remove OSD keyrings from auth') {
98 for (i in osd_ids) {
99 salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph auth del ' + i)
100 }
101 }
102
103 // remove osd `ceph osd rm osd.3`
104 stage('Remove OSDs') {
105 for (i in osd_ids) {
106 salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd rm ' + i)
107 }
108 }
109
110 for (osd_id in osd_ids) {
111 id = osd_id.replaceAll('osd.', '')
112
113 // remove journal, block_db, block_wal partition `parted /dev/sdj rm 3`
114 stage('Remove journal / block_db / block_wal partition') {
115 def partition_uuid = ""
116 def journal_partition_uuid = ""
117 def block_db_partition_uuid = ""
118 def block_wal_partition_uuid = ""
Mateusz Los1535aa22020-05-11 10:33:58 +0200119 try {
Tomek Jaroszyke46eb692020-07-07 12:07:40 +0200120 journal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/journal_uuid")['return'][0].values()[0].split("\n")[0]
121 }
122 catch (Exception e) {
123 common.infoMsg(e)
124 }
125 try {
126 block_db_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.db_uuid")['return'][0].values()[0].split("\n")[0]
mjedynskiaf258b02019-12-09 15:17:58 +0100127 }
Mateusz Los1535aa22020-05-11 10:33:58 +0200128 catch (Exception e) {
129 common.infoMsg(e)
130 }
Tomek Jaroszyke46eb692020-07-07 12:07:40 +0200131
Mateusz Los1535aa22020-05-11 10:33:58 +0200132 try {
Tomek Jaroszyke46eb692020-07-07 12:07:40 +0200133 block_wal_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block.wal_uuid")['return'][0].values()[0].split("\n")[0]
Mateusz Los1535aa22020-05-11 10:33:58 +0200134 }
135 catch (Exception e) {
136 common.infoMsg(e)
137 }
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100138
Mateusz Los1535aa22020-05-11 10:33:58 +0200139 // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
Tomek Jaroszyke46eb692020-07-07 12:07:40 +0200140 if (journal_partition_uuid?.trim()) {
141 ceph.removePartition(pepperEnv, HOST, journal_partition_uuid)
142 }
143 if (block_db_partition_uuid?.trim()) {
144 ceph.removePartition(pepperEnv, HOST, block_db_partition_uuid)
145 }
146 if (block_wal_partition_uuid?.trim()) {
147 ceph.removePartition(pepperEnv, HOST, block_wal_partition_uuid)
148 }
149
150 try {
151 salt.cmdRun(pepperEnv, HOST, "partprobe")
152 }
153 catch (Exception e) {
154 common.warningMsg(e)
155 }
156 }
157 if (cleanDisk) {
158 // remove data / block / lockbox partition `parted /dev/sdj rm 3`
159 stage('Remove data / block / lockbox partition') {
160 def data_partition_uuid = ""
161 def block_partition_uuid = ""
162 def osd_fsid = ""
163 def lvm = ""
164 def lvm_enabled= salt.getPillar(pepperEnv,"I@ceph:osd","ceph:osd:lvm_enabled")['return'].first().containsValue(true)
165 try {
166 osd_fsid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/fsid")['return'][0].values()[0].split("\n")[0]
167 if (lvm_enabled) {
168 lvm = salt.runSaltCommand(pepperEnv, 'local', ['expression': HOST, 'type': 'compound'], 'cmd.run', null, "salt-call lvm.lvdisplay --output json -l quiet")['return'][0].values()[0]
169 lvm = new groovy.json.JsonSlurperClassic().parseText(lvm)
170 lvm["local"].each { lv, params ->
171 if (params["Logical Volume Name"].contains(osd_fsid)) {
172 data_partition_uuid = params["Logical Volume Name"].minus("/dev/")
173 }
174 }
175 }
Mateusz Los1535aa22020-05-11 10:33:58 +0200176 }
177 catch (Exception e) {
178 common.infoMsg(e)
179 }
Tomek Jaroszyke46eb692020-07-07 12:07:40 +0200180 try {
181 block_partition_uuid = salt.cmdRun(pepperEnv, HOST, "cat /var/lib/ceph/osd/ceph-${id}/block_uuid")['return'][0].values()[0].split("\n")[0]
182 }
183 catch (Exception e) {
184 common.infoMsg(e)
185 }
186
187 // remove partition_uuid = 2c76f144-f412-481e-b150-4046212ca932
188 if (block_partition_uuid?.trim()) {
189 ceph.removePartition(pepperEnv, HOST, block_partition_uuid)
190 try{
191 salt.cmdRun(pepperEnv, HOST, "ceph-volume lvm zap `readlink /var/lib/ceph/osd/ceph-${id}/block` --destroy")
192 }
193 catch (Exception e) {
194 common.infoMsg(e)
195 }
196 }
197 if (data_partition_uuid?.trim()) {
198 ceph.removePartition(pepperEnv, HOST, data_partition_uuid, 'data', id)
199 }
Mateusz Los1535aa22020-05-11 10:33:58 +0200200 }
Jiri Broulika5bc8f62018-01-31 15:04:40 +0100201 }
Jiri Broulikeb7b82f2017-11-30 13:55:40 +0100202 }
203 }
Tomek Jaroszyke46eb692020-07-07 12:07:40 +0200204 finally {
205 // remove cluster flags
206 if (flags.size() > 0) {
207 stage('Unset cluster flags') {
208 for (flag in flags) {
209 common.infoMsg('Removing flag ' + flag)
210 salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd unset ' + flag)
211 }
Jakub Josefa63f9862018-01-11 17:58:38 +0100212 }
Tomáš Kukrálf72096d2017-08-11 12:58:03 +0200213 }
214 }
215 }
Tomek Jaroszyk0cb83922020-04-07 11:20:31 +0200216}