blob: 859550f81685259ec158b3c6a8817174dd181eb8 [file] [log] [blame]
Jiri Broulik99887c82017-10-31 09:27:52 +01001/**
2 *
3 * Remove Ceph node from existing cluster
4 *
5 * Requred parameters:
6 * SALT_MASTER_URL URL of Salt master
7 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
8 * HOST Host (minion id) to be removed
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +01009 * WAIT_FOR_HEALTHY Wait for cluster rebalance after a osd was removed
10 * CLUSTER_FLAGS Expected flags on the cluster during job run
11 * FAST_WIPE Clean only partition table insted of full wipe
12 * CLEAN_ORPHANS Clean ceph partition which are no longer part of the cluster
13 * OSD Coma separated list of OSDs to remove while keep the rest intact
Denis Egorenkofe8d7af2021-03-18 15:13:59 +040014 * OSD_NODE_IS_DOWN Remove unavailable (offline) osd node from cluster, provided in HOST parameter
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010015 * GENERATE_CRUSHMAP Generate new crush map. Excludes OSD
Jiri Broulik99887c82017-10-31 09:27:52 +010016 *
17 */
18
Ivan Berezovskiy19c685a2019-11-05 17:42:57 +040019def common = new com.mirantis.mk.Common()
20def salt = new com.mirantis.mk.Salt()
21def ceph = new com.mirantis.mk.Ceph()
Jiri Broulik99887c82017-10-31 09:27:52 +010022def python = new com.mirantis.mk.Python()
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010023
Jiri Broulik99887c82017-10-31 09:27:52 +010024def pepperEnv = "pepperEnv"
25
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010026def osds = OSD.tokenize(',').toSet()
27def flags = CLUSTER_FLAGS.tokenize(',').toSet()
28def cleanOrphans = CLEAN_ORPHANS.toBoolean()
29def fullWipe = !FAST_WIPE.toBoolean()
30def safeRemove = WAIT_FOR_HEALTHY.toBoolean()
31
32def osdOnly = OSD.trim() as Boolean
33def generateCrushmap = osdOnly ? false : GENERATE_CRUSHMAP.toBoolean()
Denis Egorenkofe8d7af2021-03-18 15:13:59 +040034def osdNodeUnavailable = OSD_NODE_IS_DOWN.toBoolean()
Mateusz Los8b64b812020-05-11 10:33:58 +020035
Jakub Josefa63f9862018-01-11 17:58:38 +010036timeout(time: 12, unit: 'HOURS') {
37 node("python") {
Jiri Broulik96c867a2017-11-07 16:14:10 +010038
Jakub Josefa63f9862018-01-11 17:58:38 +010039 // create connection to salt master
40 python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
Jiri Broulik99887c82017-10-31 09:27:52 +010041
Denis Egorenkofe8d7af2021-03-18 15:13:59 +040042 if (osdNodeUnavailable) {
43 stage('Remove unavailable OSD node') {
44 osdHostName = salt.stripDomainName("${HOST}")
45 osdTreeString = ceph.cmdRun(pepperEnv, "ceph osd tree --format json-pretty")
46 osdTree = common.parseJSON(osdTreeString)
47 osdIDs = []
48 for(osd in osdTree["nodes"]) {
49 if (osd["type"] == "host" && osd["name"] == osdHostName) {
50 osdIDs = osd["children"]
51 break
52 }
53 }
54 if (osdIDs.size() == 0) {
55 common.warningMsg("Can't find any OSDs placed on host ${HOST} (${osdHostName}). Is it correct name?")
56 currentBuild.result = "UNSTABLE"
57 } else {
58 common.infoMsg("Found next OSDs for host ${HOST} (${osdHostName}): ${osdIDs}")
59 input message: "Do you want to continue node remove?"
60 for (osdId in osdIDs) {
61 ceph.cmdRun(master, "ceph osd purge ${osdId} --yes-i-really-mean-it", true, true)
62 }
63 salt.cmdRun(pepperEnv, "I@salt:master", "salt-key -d ${HOST} --include-all -y")
64 }
65 }
66 return
67 }
68
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010069 def target = salt.getMinions(pepperEnv, HOST)
70 if(target.isEmpty()) {
Mateusz Los13d05202020-05-11 09:51:13 +020071 common.errorMsg("Host not found")
72 throw new InterruptedException()
73 }
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010074 else if(target.size() > 1) {
75 common.errorMsg("$HOST targeted more than one minion")
76 throw new InterruptedException()
Jakub Josefa63f9862018-01-11 17:58:38 +010077 }
78
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010079 salt.fullRefresh(pepperEnv, HOST)
Jakub Josefa63f9862018-01-11 17:58:38 +010080
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010081 stage('Set flags') {
82 ceph.setFlags(pepperEnv, flags)
Jiri Broulik99887c82017-10-31 09:27:52 +010083 }
84
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010085 try {
Jakub Josefa63f9862018-01-11 17:58:38 +010086 stage('Remove OSDs') {
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010087 if(salt.testTarget(pepperEnv, "$HOST and I@ceph:osd")) {
88 // get list of osd disks of the host
89 def cephGrain = ceph.getGrain(pepperEnv, HOST, 'ceph')
90 def cephDisks = cephGrain.get('ceph_disk',[:]).keySet()
91 if (cephGrain.isEmpty()) {
92 throw new Exception("Ceph salt grains cannot be found on $HOST")
93 }
94
Tomek Jaroszyk1f9a0c52021-02-17 15:25:06 +010095 // glob for OSD input or whole node is going to be removed
96 if(OSD == '*' || !osdOnly) {
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +010097 osds = cephDisks
98 }
99
100 // discard all osds which aren't deployed on target HOST
101 osds = osds.intersect(cephDisks)
102
103 if(!osds.isEmpty()) {
104 common.infoMsg("The following osds will be removed: ${osds.join(', ')}")
105 }
106 if(osds != cephDisks) {
107 common.infoMsg("The following osds will be skiped: ${cephDisks.removeAll(osds).join(', ')}")
108 }
109
110 ceph.removeOsd(pepperEnv, HOST, osds, flags, safeRemove, fullWipe)
111
112 if(cleanOrphans) {
113 ceph.removeOrphans(pepperEnv, HOST, fullWipe)
114 }
115 }
116 else {
117 common.infoMsg('Stage skipped.')
Jakub Josefa63f9862018-01-11 17:58:38 +0100118 }
119 }
120
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +0100121 stage('Remove keyring') {
122 // only non-osd nodes as keyrings for osds was removed already in previous step
123 if(salt.testTarget(pepperEnv, "$HOST and not I@ceph:osd")) {
124 ceph.deleteKeyrings(pepperEnv, HOST)
125 }
126 else {
127 common.infoMsg('Stage skipped.')
128 }
129 }
Jakub Josefa63f9862018-01-11 17:58:38 +0100130
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +0100131 stage('Update monmap') {
132 if(salt.testTarget(pepperEnv, "$HOST and I@ceph:mon")) {
133 def hostname = ceph.getGrain(pepperEnv, HOST, 'host')
134 ceph.cmdRun(pepperEnv, 'ceph mon getmap -o monmap.backup')
135 ceph.cmdRun(pepperEnv, "ceph mon remove $hostname")
136 salt.cmdRun(pepperEnv, 'I@ceph:mon', "monmaptool /tmp/monmap --rm $hostname")
137 }
138 else {
139 common.infoMsg('Stage skipped.')
140 }
141 }
Mateusz Los8b64b812020-05-11 10:33:58 +0200142
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +0100143 stage('Update Ceph configs/crushmap') {
144 //TODO: it won't remove removed mon from config
145 if(salt.testTarget(pepperEnv, "$HOST and I@ceph:mon")) {
146 salt.enforceState(pepperEnv, 'I@ceph:common', 'ceph.common', true)
147 }
148 else if (salt.testTarget(pepperEnv, "$HOST and I@ceph:osd") && salt.testTarget(pepperEnv, "I@ceph:setup:crush and not $HOST") && generateCrushmap) {
149 salt.enforceState(pepperEnv, 'I@ceph:setup:crush', 'ceph.setup.crush', true)
150 }
151 else {
152 common.infoMsg('Stage skipped.')
153 }
154 }
155
156 stage('Purge Ceph components') {
157 Set pkgs = ['ceph-base','ceph-common']
158 if(salt.testTarget(pepperEnv, "$HOST and I@ceph:osd")) {
159 pkgs.addAll(['ceph-osd','ceph-fuse','ceph-mds','python-cephfs','librados2','python-rados','python-rbd','python-rgw'])
160 }
161 //TODO: why removed pkgs on vm which will be remved as whole in next stage
162 if(salt.testTarget(pepperEnv, "$HOST and I@ceph:radosgw")) {
163 ceph.removeRgw(pepperEnv, HOST)
164 pkgs.addAll(['radosgw','libcephfs2','python-cephfs','python-rados','python-rbd','python-rgw'])
165 }
166 if(salt.testTarget(pepperEnv, "$HOST and I@ceph:mon")) {
167 pkgs.addAll(['ceph-mon','ceph-mgr','libcephfs2','python-cephfs','python-rbd','python-rgw'])
168 }
169
170 if(!osdOnly) {
Tomek Jaroszyk6a4bf642021-03-09 12:22:37 +0100171 salt.runSaltProcessStep(pepperEnv, HOST, 'pkg.purge', "pkgs='$pkgs'")
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +0100172 }
173 else {
174 common.infoMsg('Stage skipped.')
175 }
176 }
177
178 stage('Remove salt minion and destroy VM') {
179 if(!osdOnly) {
180 if(salt.testTarget(pepperEnv, "$HOST and I@ceph:osd")) {
181 ceph.removeSalt(pepperEnv, HOST)
Jakub Josefa63f9862018-01-11 17:58:38 +0100182 }
Mateusz Los8b64b812020-05-11 10:33:58 +0200183 else {
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +0100184 ceph.removeVm(pepperEnv, HOST)
Mateusz Los8b64b812020-05-11 10:33:58 +0200185 }
186 }
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +0100187 else {
188 common.infoMsg('Stage skipped.')
Mateusz Los8b64b812020-05-11 10:33:58 +0200189 }
Jiri Broulik3485b2c2017-11-28 15:06:12 +0100190 }
Jiri Broulik99887c82017-10-31 09:27:52 +0100191 }
Tomek Jaroszyk08ad67e2020-11-09 13:58:02 +0100192 finally {
193 stage('Unset cluster flags') {
194 ceph.unsetFlags(pepperEnv, flags)
Jakub Josefa63f9862018-01-11 17:58:38 +0100195 }
Jiri Broulik99887c82017-10-31 09:27:52 +0100196 }
197 }
198}