| /** |
| * |
| * Upgrade Ceph mon/mgr/osd/rgw/client |
| * |
| * Requred parameters: |
| * SALT_MASTER_URL URL of Salt master |
| * SALT_MASTER_CREDENTIALS Credentials to the Salt API |
| * |
| * CLUSTER_FLAGS Comma separated list of tags to apply to cluster |
| * WAIT_FOR_HEALTHY Wait for cluster rebalance before stoping daemons |
| * ASK_CONFIRMATION Ask for manual confirmation |
| * ORIGIN_RELEASE Ceph release version before upgrade |
| * TARGET_RELEASE Ceph release version after upgrade |
| * STAGE_UPGRADE_MON Set to True if Ceph mon nodes upgrade is desired |
| * STAGE_UPGRADE_MGR Set to True if Ceph mgr nodes upgrade or new deploy is desired |
| * STAGE_UPGRADE_OSD Set to True if Ceph osd nodes upgrade is desired |
| * STAGE_UPGRADE_RGW Set to True if Ceph rgw nodes upgrade is desired |
| * STAGE_UPGRADE_CLIENT Set to True if Ceph client nodes upgrade is desired (includes for example ctl/cmp nodes) |
| * STAGE_FINALIZE Set to True if configs recommended for TARGET_RELEASE should be set after upgrade is done |
| * BACKUP_ENABLED Select to copy the disks of Ceph VMs before upgrade and backup Ceph directories on OSD nodes |
| * BACKUP_DIR Select the target dir to backup to when BACKUP_ENABLED |
| * |
| */ |
| |
| common = new com.mirantis.mk.Common() |
| salt = new com.mirantis.mk.Salt() |
| def python = new com.mirantis.mk.Python() |
| ceph = new com.mirantis.mk.Ceph() |
| upgradeChecks = new com.mirantis.mcp.UpgradeChecks() |
| askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean() |
| |
| pepperEnv = "pepperEnv" |
| flags = CLUSTER_FLAGS.tokenize(',') |
| // sortbitwise is set by default on version >jewel. |
| // For jewel upgrade we will set and keep it while for other cases shouldn't be there |
| flags.removeElement('sortbitwise') |
| |
| def backup(master, target) { |
| stage("backup $target") { |
| |
| if (target == 'osd') { |
| try { |
| salt.enforceState(master, "I@ceph:${target}", "ceph.backup", true) |
| ceph.cmdRunOnTarget(master, "I@ceph:${target}", "su root -c '/usr/local/bin/ceph-backup-runner-call.sh'") |
| } catch (Exception e) { |
| common.errorMsg(e) |
| common.errorMsg("Make sure Ceph backup on OSD nodes is enabled") |
| throw new InterruptedException() |
| } |
| } else { |
| def _pillar = salt.getGrain(master, 'I@salt:master', 'domain') |
| def domain = _pillar['return'][0].values()[0].values()[0] |
| |
| def kvm_pillar = salt.getGrain(master, 'I@salt:control', 'id') |
| def kvm01 = kvm_pillar['return'][0].values()[0].values()[0] |
| |
| def target_pillar = salt.getGrain(master, "I@ceph:${target}", 'host') |
| def minions = target_pillar['return'][0].values() |
| for (minion in minions) { |
| def minion_name = minion.values()[0] |
| def provider_pillar = salt.getPillar(master, "${kvm01}", "salt:control:cluster:internal:node:${minion_name}:provider") |
| def minionProvider = provider_pillar['return'][0].values()[0] |
| |
| ceph.waitForHealthy(master, flags) |
| try { |
| ceph.cmdRunOnTarget(master, minionProvider, "[ ! -f ${BACKUP_DIR}/${minion_name}.${domain}.qcow2.bak ] && virsh destroy ${minion_name}.${domain}") |
| } catch (Exception e) { |
| common.warningMsg('Backup already exists') |
| } |
| try { |
| ceph.cmdRunOnTarget(master, minionProvider, "[ ! -f ${BACKUP_DIR}/${minion_name}.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/${minion_name}.${domain}/system.qcow2 ${BACKUP_DIR}/${minion_name}.${domain}.qcow2.bak") |
| } catch (Exception e) { |
| common.warningMsg('Backup already exists') |
| } |
| try { |
| ceph.cmdRunOnTarget(master, minionProvider, "virsh start ${minion_name}.${domain}") |
| } catch (Exception e) { |
| common.warningMsg(e) |
| } |
| salt.minionsReachable(master, 'I@salt:master', "${minion_name}*") |
| ceph.waitForHealthy(master, flags) |
| } |
| } |
| } |
| return |
| } |
| |
| def upgrade(master, target) { |
| |
| stage("Change $target repos") { |
| salt.runSaltProcessStep(master, "I@ceph:${target}", 'saltutil.refresh_pillar', [], null, true, 5) |
| salt.enforceState(master, "I@ceph:${target}", 'linux.system.repo', true) |
| } |
| if (target == 'mgr') { |
| stage('Run ceph mgr state') { |
| salt.enforceState(master, "I@ceph:mgr", "ceph.mgr", true, failOnError=false, retries=3, retries_wait=10) |
| } |
| } |
| if (target == 'common') { |
| stage('Upgrade ceph-common pkgs') { |
| salt.runSaltProcessStep(master, "I@ceph:common", 'pkg.install', ["ceph-common"], 'only_upgrade=True') |
| } |
| } else { |
| minions = salt.getMinions(master, "I@ceph:${target}") |
| |
| def ignoreDifferentSubversions = false |
| for (minion in minions) { |
| // upgrade pkgs |
| if (target == 'radosgw') { |
| stage("Upgrade radosgw pkgs on $minion") { |
| salt.runSaltProcessStep(master, minion, 'pkg.install', [target], 'only_upgrade=True') |
| } |
| } else { |
| stage("Upgrade $target pkgs on $minion") { |
| salt.runSaltProcessStep(master, minion, 'pkg.install', ["ceph-${target}"], 'only_upgrade=True') |
| } |
| } |
| // check for subversion difference before restart |
| if (ignoreDifferentSubversions) { |
| targetVersion = ceph.cmdRun(master, "ceph versions | grep $TARGET_RELEASE | awk '{print \$3}' | sort -V | tail -1") |
| updatedVersion = ceph.cmdRunOnTarget(master, minion, "ceph version | awk '{print \$3}'") |
| if (targetVersion != updatedVersion) { |
| stage('Version differnce warning') { |
| common.warningMsg("A potential problem has been spotted.") |
| common.warningMsg("Some components already have $targetVersion version while ceph-$target has just been updated to $updatedVersion") |
| input message: "Do you want to proceed with restarts and silence this warning?" |
| ignoreDifferentSubversions = true |
| } |
| } |
| } |
| // restart services |
| stage("Restart ${target} services on ${minion}") { |
| if(target == 'osd') { |
| def ceph_disks = salt.getGrain(master, minion, 'ceph')['return'][0].values()[0].values()[0]['ceph_disk'] |
| ceph_disks.each { osd, param -> |
| ceph.cmdRunOnTarget(master, minion, "systemctl restart ceph-${target}@${osd}") |
| } |
| } |
| else { |
| ceph.cmdRunOnTarget(master, minion, "systemctl restart ceph-${target}.target") |
| } |
| |
| ceph.waitForHealthy(master, flags) |
| } |
| |
| stage("Verify services for $minion") { |
| sleep(10) |
| ceph.cmdRunOnTarget(master, minion, "systemctl status ceph-${target}.target") |
| } |
| |
| stage('Verify Ceph status') { |
| ceph.cmdRun(master, "ceph -s", true, true) |
| if (askConfirmation) { |
| input message: "From the verification command above, please check Ceph $target joined the cluster correctly. If so, Do you want to continue to upgrade next node?" |
| } |
| } |
| } |
| } |
| ceph.cmdRun(master, "ceph versions") |
| sleep(5) |
| return |
| } |
| |
| timeout(time: 12, unit: 'HOURS') { |
| node("python") { |
| |
| // create connection to salt master |
| python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS) |
| |
| stage('Check user choices') { |
| if (STAGE_UPGRADE_RGW.toBoolean()) { |
| // if rgw, check if other stuff has required version |
| def mon_ok = true |
| if (!STAGE_UPGRADE_MON.toBoolean()) { |
| def mon_v = ceph.cmdRun(pepperEnv, "ceph mon versions") |
| mon_ok = mon_v.contains("${TARGET_RELEASE}") && !mon_v.contains("${ORIGIN_RELEASE}") |
| } |
| def mgr_ok = true |
| if (!STAGE_UPGRADE_MGR.toBoolean()) { |
| def mgr_v = ceph.cmdRun(pepperEnv, "ceph mgr versions") |
| mgr_ok = mgr_v.contains("${TARGET_RELEASE}") && !mgr_v.contains("${ORIGIN_RELEASE}") |
| } |
| def osd_ok = true |
| if (!STAGE_UPGRADE_OSD.toBoolean()) { |
| def osd_v = ceph.cmdRun(pepperEnv, "ceph osd versions") |
| osd_ok = osd_v.contains("${TARGET_RELEASE}") && !osd_v.contains("${ORIGIN_RELEASE}") |
| } |
| if (!mon_ok || !osd_ok || !mgr_ok) { |
| common.errorMsg('You may choose stages in any order, but RGW should be upgraded last') |
| throw new InterruptedException() |
| } |
| } |
| } |
| |
| stage('Pre-validate cluster model settings'){ |
| upgradeChecks.check_36461_2(salt, pepperEnv, "", true) |
| } |
| |
| if (BACKUP_ENABLED.toBoolean() == true) { |
| if (STAGE_UPGRADE_MON.toBoolean() == true) { |
| backup(pepperEnv, 'mon') |
| } |
| if (STAGE_UPGRADE_RGW.toBoolean() == true) { |
| backup(pepperEnv, 'radosgw') |
| } |
| if (STAGE_UPGRADE_OSD.toBoolean() == true) { |
| backup(pepperEnv, 'osd') |
| } |
| } |
| |
| stage('Set cluster flags') { |
| ceph.setFlags(pepperEnv, flags) |
| if (ORIGIN_RELEASE == 'jewel') { |
| ceph.setFlags('sortbitwise') |
| } |
| } |
| |
| if (STAGE_UPGRADE_MON.toBoolean() == true) { |
| upgrade(pepperEnv, 'mon') |
| } |
| |
| if (STAGE_UPGRADE_MGR.toBoolean() == true) { |
| upgrade(pepperEnv, 'mgr') |
| } |
| |
| if (STAGE_UPGRADE_OSD.toBoolean() == true) { |
| upgrade(pepperEnv, 'osd') |
| } |
| |
| if (STAGE_UPGRADE_RGW.toBoolean() == true) { |
| upgrade(pepperEnv, 'radosgw') |
| } |
| |
| if (STAGE_UPGRADE_CLIENT.toBoolean() == true) { |
| upgrade(pepperEnv, 'common') |
| } |
| |
| // remove cluster flags |
| stage('Unset cluster flags') { |
| ceph.unsetFlags(flags) |
| } |
| |
| if (STAGE_FINALIZE.toBoolean() == true) { |
| stage("Finalize ceph version upgrade") { |
| ceph.cmdRun(pepperEnv, "ceph osd require-osd-release ${TARGET_RELEASE}") |
| try { |
| ceph.cmdRun(pepperEnv, "ceph osd set-require-min-compat-client ${ORIGIN_RELEASE}") |
| } catch (Exception e) { |
| common.warningMsg(e) |
| } |
| try { |
| ceph.cmdRun(pepperEnv, "ceph osd crush tunables optimal") |
| } catch (Exception e) { |
| common.warningMsg(e) |
| } |
| if (TARGET_RELEASE == 'nautilus' ) { |
| ceph.cmdRun(pepperEnv, "ceph mon enable-msgr2") |
| } |
| salt.enforceState(pepperEnv, "I@ceph:common", "ceph.common") |
| } |
| } |
| |
| // wait for healthy cluster |
| if (WAIT_FOR_HEALTHY.toBoolean()) { |
| ceph.waitForHealthy(pepperEnv, flags) |
| } |
| stage('Post-upgrade cluster model settings validation'){ |
| def checkResult = upgradeChecks.check_36461_2(salt, pepperEnv, "", false) |
| common.warningMsg(checkResult.isFixed) |
| if ( checkResult.waInfo != '') { |
| common.warningMsg(checkResult.waInfo) |
| } |
| } |
| } |
| } |