Use common waitForHealthy function from pipeline-library
Additional improvements:
* small refactoring of common functions
* add check for prometheus nodes before running
prometheus state
* improve usage of waitForHealthy function with flags
PROD-33010
Change-Id: Ic8679fb0178965c9788c12793902c9bc265a6146
diff --git a/ceph-backend-migration.groovy b/ceph-backend-migration.groovy
index 7a5821d..676c236 100644
--- a/ceph-backend-migration.groovy
+++ b/ceph-backend-migration.groovy
@@ -20,6 +20,7 @@
common = new com.mirantis.mk.Common()
salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
+ceph = new com.mirantis.mk.Ceph()
MIGRATION_METHOD = "per-osd"
// TBD: per-host
@@ -28,27 +29,7 @@
def flags = CLUSTER_FLAGS.tokenize(',')
def osds = OSD.tokenize(',')
-def removePartition(master, target, partition_uuid) {
- def partition = ""
- try {
- // partition = /dev/sdi2
- partition = runCephCommand(master, target, "blkid | grep ${partition_uuid} ")['return'][0].values()[0].split("(?<=[0-9])")[0]
- } catch (Exception e) {
- common.warningMsg(e)
- }
-
- if (partition?.trim()) {
- // dev = /dev/sdi
- def dev = partition.replaceAll('\\d+$', "")
- // part_id = 2
- def part_id = partition.substring(partition.lastIndexOf("/")+1).replaceAll("[^0-9]", "")
- runCephCommand(master, target, "parted ${dev} rm ${part_id}")
- }
- return
-}
-
def removeJournalOrBlockPartitions(master, target, id) {
-
// remove journal, block_db, block_wal partition `parted /dev/sdj rm 3`
stage('Remove journal / block_db / block_wal partition') {
def partition_uuid = ""
@@ -56,20 +37,20 @@
def block_db_partition_uuid = ""
def block_wal_partition_uuid = ""
try {
- journal_partition_uuid = runCephCommand(master, target, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep journal | grep partuuid")
+ journal_partition_uuid = salt.cmdRun(master, target, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep journal | grep partuuid")
journal_partition_uuid = journal_partition_uuid.toString().trim().split("\n")[0].substring(journal_partition_uuid.toString().trim().lastIndexOf("/")+1)
} catch (Exception e) {
common.infoMsg(e)
}
try {
- block_db_partition_uuid = runCephCommand(master, target, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep 'block.db' | grep partuuid")
+ block_db_partition_uuid = salt.cmdRun(master, target, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep 'block.db' | grep partuuid")
block_db_partition_uuid = block_db_partition_uuid.toString().trim().split("\n")[0].substring(block_db_partition_uuid.toString().trim().lastIndexOf("/")+1)
} catch (Exception e) {
common.infoMsg(e)
}
try {
- block_wal_partition_uuid = runCephCommand(master, target, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep 'block.wal' | grep partuuid")
+ block_wal_partition_uuid = salt.cmdRun(master, target, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep 'block.wal' | grep partuuid")
block_wal_partition_uuid = block_wal_partition_uuid.toString().trim().split("\n")[0].substring(block_wal_partition_uuid.toString().trim().lastIndexOf("/")+1)
} catch (Exception e) {
common.infoMsg(e)
@@ -84,31 +65,15 @@
// if disk has journal, block_db or block_wal on different disk, then remove the partition
if (partition_uuid?.trim()) {
- removePartition(master, target, partition_uuid)
+ ceph.removePartition(master, target, partition_uuid)
}
if (block_wal_partition_uuid?.trim()) {
- removePartition(master, target, block_wal_partition_uuid)
+ ceph.removePartition(master, target, block_wal_partition_uuid)
}
}
return
}
-def runCephCommand(master, target, cmd) {
- return salt.cmdRun(master, target, cmd)
-}
-
-def waitForHealthy(master, count=0, attempts=300) {
- // wait for healthy cluster
- while (count<attempts) {
- def health = runCephCommand(master, ADMIN_HOST, 'ceph health')['return'][0].values()[0]
- if (health.contains('HEALTH_OK')) {
- common.infoMsg('Cluster is healthy')
- break;
- }
- count++
- sleep(10)
- }
-}
timeout(time: 12, unit: 'HOURS') {
node("python") {
@@ -120,7 +85,7 @@
if (flags.size() > 0) {
stage('Set cluster flags') {
for (flag in flags) {
- runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd set ' + flag)
+ salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd set ' + flag)
}
}
}
@@ -147,23 +112,23 @@
for (osd_id in osd_ids) {
def id = osd_id.replaceAll('osd.', '')
- def backend = runCephCommand(pepperEnv, ADMIN_HOST, "ceph osd metadata ${id} | grep osd_objectstore")['return'][0].values()[0]
+ def backend = salt.cmdRun(pepperEnv, ADMIN_HOST, "ceph osd metadata ${id} | grep osd_objectstore")['return'][0].values()[0]
if (backend.contains(ORIGIN_BACKEND.toLowerCase())) {
// wait for healthy cluster before manipulating with osds
- if (WAIT_FOR_HEALTHY.toBoolean() == true) {
- waitForHealthy(pepperEnv)
+ if (WAIT_FOR_HEALTHY.toBoolean()) {
+ ceph.waitForHealthy(pepperEnv, ADMIN_HOST)
}
// `ceph osd out <id> <id>`
stage('Set OSDs out') {
- runCephCommand(pepperEnv, ADMIN_HOST, "ceph osd out ${osd_id}")
+ salt.cmdRun(pepperEnv, ADMIN_HOST, "ceph osd out ${osd_id}")
}
- if (WAIT_FOR_HEALTHY.toBoolean() == true) {
+ if (WAIT_FOR_HEALTHY.toBoolean()) {
sleep(5)
- waitForHealthy(pepperEnv)
+ ceph.waitForHealthy(pepperEnv, ADMIN_HOST)
}
// stop osd daemons
@@ -173,28 +138,28 @@
// remove keyring `ceph auth del osd.3`
stage('Remove OSD keyrings from auth') {
- runCephCommand(pepperEnv, ADMIN_HOST, 'ceph auth del ' + osd_id)
+ salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph auth del ' + osd_id)
}
// remove osd `ceph osd rm osd.3`
stage('Remove OSDs') {
- runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd rm ' + osd_id)
+ salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd rm ' + osd_id)
}
def dmcrypt = ""
try {
- dmcrypt = runCephCommand(pepperEnv, tgt, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep dmcrypt")['return'][0].values()[0]
+ dmcrypt = salt.cmdRun(pepperEnv, tgt, "ls -la /var/lib/ceph/osd/ceph-${id}/ | grep dmcrypt")['return'][0].values()[0]
} catch (Exception e) {
common.warningMsg(e)
}
if (dmcrypt?.trim()) {
- def mount = runCephCommand(pepperEnv, tgt, "lsblk -rp | grep /var/lib/ceph/osd/ceph-${id} -B1")['return'][0].values()[0]
+ def mount = salt.cmdRun(pepperEnv, tgt, "lsblk -rp | grep /var/lib/ceph/osd/ceph-${id} -B1")['return'][0].values()[0]
dev = mount.split()[0].replaceAll("[0-9]","")
// remove partition tables
stage('dd part tables') {
- runCephCommand(pepperEnv, tgt, "dd if=/dev/zero of=${dev} bs=512 count=1 conv=notrunc")
+ salt.cmdRun(pepperEnv, tgt, "dd if=/dev/zero of=${dev} bs=512 count=1 conv=notrunc")
}
// remove journal, block_db, block_wal partition `parted /dev/sdj rm 3`
@@ -210,16 +175,16 @@
// zap disks `ceph-disk zap /dev/sdi`
stage('Zap devices') {
try {
- runCephCommand(pepperEnv, tgt, 'ceph-disk zap ' + dev)
+ salt.cmdRun(pepperEnv, tgt, 'ceph-disk zap ' + dev)
} catch (Exception e) {
common.warningMsg(e)
}
- runCephCommand(pepperEnv, tgt, 'ceph-disk zap ' + dev)
+ salt.cmdRun(pepperEnv, tgt, 'ceph-disk zap ' + dev)
}
} else {
- def mount = runCephCommand(pepperEnv, tgt, "mount | grep /var/lib/ceph/osd/ceph-${id}")['return'][0].values()[0]
+ def mount = salt.cmdRun(pepperEnv, tgt, "mount | grep /var/lib/ceph/osd/ceph-${id}")['return'][0].values()[0]
dev = mount.split()[0].replaceAll("[0-9]","")
// remove journal, block_db, block_wal partition `parted /dev/sdj rm 3`
@@ -227,12 +192,12 @@
// umount `umount /dev/sdi1`
stage('Umount devices') {
- runCephCommand(pepperEnv, tgt, "umount /var/lib/ceph/osd/ceph-${id}")
+ salt.cmdRun(pepperEnv, tgt, "umount /var/lib/ceph/osd/ceph-${id}")
}
// zap disks `ceph-disk zap /dev/sdi`
stage('Zap device') {
- runCephCommand(pepperEnv, tgt, 'ceph-disk zap ' + dev)
+ salt.cmdRun(pepperEnv, tgt, 'ceph-disk zap ' + dev)
}
}
@@ -245,8 +210,8 @@
if (PER_OSD_CONTROL.toBoolean() == true) {
stage("Verify backend version for osd.${id}") {
sleep(5)
- runCephCommand(pepperEnv, tgt, "ceph osd metadata ${id} | grep osd_objectstore")
- runCephCommand(pepperEnv, tgt, "ceph -s")
+ salt.cmdRun(pepperEnv, tgt, "ceph osd metadata ${id} | grep osd_objectstore")
+ salt.cmdRun(pepperEnv, tgt, "ceph -s")
}
stage('Ask for manual confirmation') {
@@ -258,8 +223,8 @@
if (PER_OSD_HOST_CONTROL.toBoolean() == true) {
stage("Verify backend versions") {
sleep(5)
- runCephCommand(pepperEnv, tgt, "ceph osd metadata | grep osd_objectstore -B2")
- runCephCommand(pepperEnv, tgt, "ceph -s")
+ salt.cmdRun(pepperEnv, tgt, "ceph osd metadata | grep osd_objectstore -B2")
+ salt.cmdRun(pepperEnv, tgt, "ceph -s")
}
stage('Ask for manual confirmation') {
@@ -273,7 +238,7 @@
stage('Unset cluster flags') {
for (flag in flags) {
common.infoMsg('Removing flag ' + flag)
- runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd unset ' + flag)
+ salt.cmdRun(pepperEnv, ADMIN_HOST, 'ceph osd unset ' + flag)
}
}
}