ceph-upgrade.groovy

/**
 *
 * Upgrade Ceph mon/mgr/osd/rgw/client
 *
 * Requred parameters:
 *  SALT_MASTER_URL                 URL of Salt master
 *  SALT_MASTER_CREDENTIALS         Credentials to the Salt API
 *
 *  ADMIN_HOST                      Host (minion id) with admin keyring and /etc/crushmap file present
 *  CLUSTER_FLAGS                   Comma separated list of tags to apply to cluster
 *  WAIT_FOR_HEALTHY                Wait for cluster rebalance before stoping daemons
 *  ORIGIN_RELEASE                  Ceph release version before upgrade
 *  TARGET_RELEASE                  Ceph release version after upgrade
 *  STAGE_UPGRADE_MON               Set to True if Ceph mon nodes upgrade is desired
 *  STAGE_UPGRADE_MGR               Set to True if Ceph mgr nodes upgrade or new deploy is desired
 *  STAGE_UPGRADE_OSD               Set to True if Ceph osd nodes upgrade is desired
 *  STAGE_UPGRADE_RGW               Set to True if Ceph rgw nodes upgrade is desired
 *  STAGE_UPGRADE_CLIENT            Set to True if Ceph client nodes upgrade is desired (includes for example ctl/cmp nodes)
 *
 */

common = new com.mirantis.mk.Common()
salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()

def pepperEnv = "pepperEnv"
def flags = CLUSTER_FLAGS.tokenize(',')

def runCephCommand(master, target, cmd) {
    return salt.cmdRun(master, target, cmd)
}

def waitForHealthy(master, count=0, attempts=300) {
    // wait for healthy cluster
    while (count<attempts) {
        def health = runCephCommand(master, ADMIN_HOST, 'ceph health')['return'][0].values()[0]
        if (health.contains('HEALTH_OK')) {
            common.infoMsg('Cluster is healthy')
            break;
        }
        count++
        sleep(10)
    }
}

def backup(master, target) {
    stage("backup ${target}") {

        def _pillar = salt.getGrain(master, 'I@salt:master', 'domain')
        def domain = _pillar['return'][0].values()[0].values()[0]

        def kvm_pillar = salt.getGrain(master, 'I@salt:control', 'id')
        def kvm01 = kvm_pillar['return'][0].values()[0].values()[0]

        def target_pillar = salt.getGrain(master, "I@ceph:${target}", 'host')
        def minions = target_pillar['return'][0].values()
        for (minion in minions) {
            def minion_name = minion.values()[0]
            def provider_pillar = salt.getPillar(master, "${kvm01}", "salt:control:cluster:internal:node:${minion_name}:provider")
            def minionProvider = provider_pillar['return'][0].values()[0]

            waitForHealthy(master)
            try {
                salt.cmdRun(master, "${minionProvider}", "[ ! -f /root/${minion_name}.${domain}.qcow2.bak ] && virsh destroy ${minion_name}.${domain}")
            } catch (Exception e) {
                common.warningMsg('Backup already exists')
            }
            try {
                salt.cmdRun(master, "${minionProvider}", "[ ! -f /root/${minion_name}.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/${minion_name}.${domain}/system.qcow2 /root/${minion_name}.${domain}.qcow2.bak")
            } catch (Exception e) {
                common.warningMsg('Backup already exists')
            }
            try {
                salt.cmdRun(master, "${minionProvider}", "virsh start ${minion_name}.${domain}")
            } catch (Exception e) {
                common.warningMsg(e)
            }
            salt.minionsReachable(master, 'I@salt:master', "${minion_name}*")
            waitForHealthy(master)
        }
    }
    return
}

def upgrade(master, target) {

    stage("Change ${target} repos") {
        salt.runSaltProcessStep(master, "I@ceph:${target}", 'saltutil.refresh_pillar', [], null, true, 5)
        salt.enforceState(master, "I@ceph:${target}", 'linux.system.repo', true)
    }
    if (target == 'mgr') {
        stage('Run ceph mgr state') {
            salt.enforceState(master, "I@ceph:mgr", "ceph.mgr", true)
        }
    }
    if (target == 'common') {
        stage('Upgrade ceph-common pkgs') {
            runCephCommand(master, "I@ceph:${target}", "apt install ceph-${target} -y")
        }
    } else {
        minions = salt.getMinions(master, "I@ceph:${target}")

        for (minion in minions) {
            // upgrade pkgs
            if (target == 'radosgw') {
                stage('Upgrade radosgw pkgs') {
                    runCephCommand(master, "I@ceph:${target}", "apt install ${target} -y ")
                }
            } else {
                stage("Upgrade ${target} pkgs on ${minion}") {
                    runCephCommand(master, "${minion}", "apt install ceph-${target} -y")
                }
            }
            // restart services
            stage("Restart ${target} services on ${minion}") {
                runCephCommand(master, "${minion}", "systemctl restart ceph-${target}.target")
            }

            stage("Verify services for ${minion}") {
                sleep(10)
                runCephCommand(master, ADMIN_HOST, "ceph -s")
            }

            stage('Ask for manual confirmation') {
                input message: "From the verification command above, please check Ceph ${target} joined the cluster correctly. If so, Do you want to continue to upgrade next node?"
            }
        }
    }
    runCephCommand(master, ADMIN_HOST, "ceph versions")
    sleep(5)
    return
}

node("python") {

    // create connection to salt master
    python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)

    if (BACKUP_ENABLED.toBoolean() == true) {
        if (STAGE_UPGRADE_MON.toBoolean() == true) {
            backup(pepperEnv, 'mon')
        }
        if (STAGE_UPGRADE_RGW.toBoolean() == true) {
            backup(pepperEnv, 'radosgw')
        }
    }

    if (flags.size() > 0) {
        stage('Set cluster flags') {
            for (flag in flags) {
                runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd set ' + flag)
            }
        }
    }

    if (STAGE_UPGRADE_MON.toBoolean() == true) {
        upgrade(pepperEnv, 'mon')
    }

    if (STAGE_UPGRADE_MGR.toBoolean() == true) {
        upgrade(pepperEnv, 'mgr')
    }

    if (STAGE_UPGRADE_OSD.toBoolean() == true) {
        upgrade(pepperEnv, 'osd')
    }

    if (STAGE_UPGRADE_RGW.toBoolean() == true) {
        upgrade(pepperEnv, 'radosgw')
    }

    if (STAGE_UPGRADE_CLIENT.toBoolean() == true) {
        upgrade(pepperEnv, 'common')
    }

    // remove cluster flags
    if (flags.size() > 0) {
        stage('Unset cluster flags') {
            for (flag in flags) {
                if (!flag.contains('sortbitwise')) {
                    common.infoMsg('Removing flag ' + flag)
                    runCephCommand(pepperEnv, ADMIN_HOST, 'ceph osd unset ' + flag)
                }

            }
        }
    }

    if (STAGE_FINALIZE.toBoolean() == true) {
        stage("Finalize ceph version upgrade") {
            runCephCommand(pepperEnv, ADMIN_HOST, "ceph osd require-osd-release ${TARGET_RELEASE}")
            try {
                runCephCommand(pepperEnv, ADMIN_HOST, "ceph osd set-require-min-compat-client ${ORIGIN_RELEASE}")
            } catch (Exception e) {
                common.warningMsg(e)
            }
            try {
                runCephCommand(pepperEnv, ADMIN_HOST, "ceph osd crush tunables optimal")
            } catch (Exception e) {
                common.warningMsg(e)
            }
        }
    }

    // wait for healthy cluster
    if (WAIT_FOR_HEALTHY.toBoolean() == true) {
        waitForHealthy(pepperEnv)
    }
}