remove ceph nodes pipeline
Change-Id: Ib90616baa8fdfe164797e595043d19fed9c7ce21
diff --git a/ceph-add-node.groovy b/ceph-add-node.groovy
index c6d2ea2..8757ca1 100644
--- a/ceph-add-node.groovy
+++ b/ceph-add-node.groovy
@@ -31,7 +31,7 @@
}
if (!found) {
- common.errorMsg("No such HOST_TYPE was found. Please insert one of these types: mon/osd/rgw")
+ common.errorMsg("No such HOST_TYPE was found. Please insert one of the following types: mon/osd/rgw")
break
}
diff --git a/ceph-remove-node.groovy b/ceph-remove-node.groovy
new file mode 100644
index 0000000..21671bf
--- /dev/null
+++ b/ceph-remove-node.groovy
@@ -0,0 +1,182 @@
+/**
+ *
+ * Remove Ceph node from existing cluster
+ *
+ * Requred parameters:
+ * SALT_MASTER_URL URL of Salt master
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API
+ * HOST Host (minion id) to be removed
+ * HOST_TYPE Type of Ceph node to be removed. Valid values are mon/osd/rgw
+ * ADMIN_HOST Host (minion id) with admin keyring
+ * WAIT_FOR_HEALTHY Wait for cluster rebalance before stoping daemons
+ * GENERATE_CRUSHMAP Set to true if the crush map should be generated
+ *
+ */
+
+common = new com.mirantis.mk.Common()
+salt = new com.mirantis.mk.Salt()
+orchestrate = new com.mirantis.mk.Orchestrate()
+def python = new com.mirantis.mk.Python()
+
+def pepperEnv = "pepperEnv"
+
+def runCephCommand(master, target, cmd) {
+ return salt.cmdRun(master, target, cmd)
+}
+
+node("python") {
+
+ // create connection to salt master
+ python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+
+ matches = ["osd", "mon", "rgw"]
+ def found = false
+ for (s in matches) {
+ if (HOST_TYPE.toLowerCase() == s) {
+ found = true
+ }
+ }
+
+ if (!found) {
+ common.errorMsg("No such HOST_TYPE was found. Please insert one of the following types: mon/osd/rgw")
+ break
+ }
+
+ stage('Refresh_pillar') {
+ salt.runSaltProcessStep(pepperEnv, '*', 'saltutil.refresh_pillar', [], null, true, 5)
+ }
+
+ // split minion id on '.' and remove '*'
+ def target = HOST.split("\\.")[0].replace("*", "")
+
+ def _pillar = salt.getGrain(pepperEnv, 'I@salt:master', 'domain')
+ domain = _pillar['return'][0].values()[0].values()[0]
+
+ if (HOST_TYPE.toLowerCase() == 'rgw') {
+ // Remove Ceph rgw
+ stage('Remove Ceph RGW') {
+ salt.enforceState(pepperEnv, 'I@ceph:radosgw', ['keepalived', 'haproxy'], true)
+ }
+ }
+
+ if (HOST_TYPE.toLowerCase() != 'osd') {
+
+ // virsh destroy rgw04.deploy-name.local; virsh undefine rgw04.deploy-name.local;
+ stage('Destroy VM') {
+ _pillar = salt.getGrain(pepperEnv, 'I@salt:control', 'id')
+ def kvm01 = _pillar['return'][0].values()[0].values()[0]
+
+ _pillar = salt.getPillar(pepperEnv, "${kvm01}", "salt:control:cluster:internal:node:${target}:provider")
+ def targetProvider = _pillar['return'][0].values()[0]
+
+ salt.cmdRun(pepperEnv, "${targetProvider}", "virsh destroy ${target}.${domain}")
+ salt.cmdRun(pepperEnv, "${targetProvider}", "virsh undefine ${target}.${domain}")
+ }
+ } else if (HOST_TYPE.toLowerCase() == 'osd') {
+ def osd_ids = []
+
+ // get list of osd disks of the host
+ def ceph_disks = salt.getGrain(pepperEnv, HOST, 'ceph')['return'][0].values()[0].values()[0]['ceph_disk']
+
+ for (i in ceph_disks) {
+ def osd_id = i.getKey().toString()
+ osd_ids.add('osd.' + osd_id)
+ print("Will delete " + osd_id)
+ }
+
+ // `ceph osd out <id> <id>`
+ stage('Set OSDs out') {
+ runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd out ' + osd_ids.join(' '))
+ }
+
+ // wait for healthy cluster
+ if (WAIT_FOR_HEALTHY.toBoolean() == true) {
+ stage('Waiting for healthy cluster') {
+ while (true) {
+ def health = runCephCommand(pepperEnv, ADMIN_HOST, 'ceph health')['return'][0].values()[0]
+ if (health.contains('HEALTH_OK')) {
+ common.infoMsg('Cluster is healthy')
+ break;
+ }
+ sleep(10)
+ }
+ }
+ }
+
+ // stop osd daemons
+ stage('Stop OSD daemons') {
+ for (i in osd_ids) {
+ salt.runSaltProcessStep(pepperEnv, HOST, 'service.stop', ['ceph-osd@' + i.replaceAll('osd.', '')], null, true)
+ }
+ }
+
+ // `ceph osd crush remove osd.2`
+ stage('Remove OSDs from CRUSH') {
+ for (i in osd_ids) {
+ runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd crush remove ' + i)
+ }
+ }
+
+ // remove keyring `ceph auth del osd.3`
+ stage('Remove OSD keyrings from auth') {
+ for (i in osd_ids) {
+ runCephCommand(pepperEnv, ADMIN_HOST, 'ceph auth del ' + i)
+ }
+ }
+
+ // remove osd `ceph osd rm osd.3`
+ stage('Remove OSDs') {
+ for (i in osd_ids) {
+ runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd rm ' + i)
+ }
+ }
+
+ // purge Ceph pkgs
+ stage('Purge Ceph OSD pkgs') {
+ runCephCommand(pepperEnv, HOST, 'apt purge ceph-base ceph-common ceph-fuse ceph-mds ceph-osd libcephfs2 python-cephfs librados2 python-rados -y')
+ }
+
+ // stop salt-minion service and move its configuration
+ stage('Stop salt-minion') {
+ salt.cmdRun(pepperEnv, HOST, "mv /etc/salt/minion.d/minion.conf minion.conf")
+ salt.runSaltProcessStep(pepperEnv, HOST, 'service.stop', ['salt-minion'], [], null, true, 5)
+ }
+ }
+
+ stage('Remove salt-key') {
+ try {
+ salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key -d ${target}.${domain} -y")
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ try {
+ salt.cmdRun(pepperEnv, 'I@salt:master', "rm /srv/salt/reclass/nodes/_generated/${target}.${domain}.yml")
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ }
+
+ if (HOST_TYPE.toLowerCase() == 'mon') {
+ // Update Monmap
+ stage('Update monmap') {
+ runCephCommand(pepperEnv, 'I@ceph:mon', "ceph mon getmap -o monmap.backup")
+ try {
+ runCephCommand(pepperEnv, 'I@ceph:mon', "ceph mon remove ${target}")
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+ runCephCommand(pepperEnv, 'I@ceph:mon', "monmaptool /tmp/monmap --rm ${target}")
+ }
+
+ // Update configs
+ stage('Update Ceph configs') {
+ salt.enforceState(pepperEnv, 'I@ceph:common', 'ceph.common', true)
+ }
+ }
+
+ if (HOST_TYPE.toLowerCase() == 'osd' && GENERATE_CRUSHMAP.toBoolean() == true) {
+ stage('Generate CRUSHMAP') {
+ salt.enforceState(pepperEnv, 'I@ceph:setup:crush', 'ceph.setup.crush', true)
+ }
+ }
+}
diff --git a/ceph-remove-osd.groovy b/ceph-remove-osd.groovy
index 747615f..18c5525 100644
--- a/ceph-remove-osd.groovy
+++ b/ceph-remove-osd.groovy
@@ -64,15 +64,15 @@
}
// wait for healthy cluster
- if (common.validInputParam('WAIT_FOR_HEALTHY') && WAIT_FOR_HEALTHY.toBoolean()) {
+ if (WAIT_FOR_HEALTHY.toBoolean() == true) {
stage('Waiting for healthy cluster') {
while (true) {
def health = runCephCommand(pepperEnv, 'ceph health')['return'][0].values()[0]
- if (health.contains('HEALTH OK')) {
+ if (health.contains('HEALTH_OK')) {
common.infoMsg('Cluster is healthy')
break;
}
- sleep(60)
+ sleep(10)
}
}
}
diff --git a/ceph-replace-failed-osd.groovy b/ceph-replace-failed-osd.groovy
index e00160c..6c6f281 100644
--- a/ceph-replace-failed-osd.groovy
+++ b/ceph-replace-failed-osd.groovy
@@ -71,15 +71,15 @@
}
// wait for healthy cluster
- if (common.validInputParam('WAIT_FOR_HEALTHY') && WAIT_FOR_HEALTHY.toBoolean()) {
+ if (WAIT_FOR_HEALTHY.toBoolean() == true) {
stage('Waiting for healthy cluster') {
while (true) {
def health = runCephCommand(pepperEnv, ADMIN_HOST, 'ceph health')['return'][0].values()[0]
- if (health.contains('HEALTH OK')) {
+ if (health.contains('HEALTH_OK')) {
common.infoMsg('Cluster is healthy')
break;
}
- sleep(60)
+ sleep(10)
}
}
}
@@ -107,7 +107,7 @@
common.infoMsg('PGs rebalanced')
break;
}
- sleep(3)
+ sleep(10)
}
}
}