[ceph] Do not restart Ceph services in packages are not updated
Change-Id: I0617a3fe3f9bba0002ff6e181a153b259a8894d9
Related-Prod: PROD-35727
(cherry picked from commit 82309cacf3afd9ba2faf98cc5bed6615d54c1992)
diff --git a/update-ceph.groovy b/update-ceph.groovy
index 6cef0b3..96c8836 100644
--- a/update-ceph.groovy
+++ b/update-ceph.groovy
@@ -7,6 +7,7 @@
*/
pepperEnv = "pepperEnv"
+def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def ceph = new com.mirantis.mk.Ceph()
def python = new com.mirantis.mk.Python()
@@ -17,29 +18,85 @@
def flags = CLUSTER_FLAGS ? CLUSTER_FLAGS.tokenize(',') : []
def runHighState = RUNHIGHSTATE
+def collectPkgVersion(target, packageName) {
+ def salt = new com.mirantis.mk.Salt()
+ return salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], "pkg.version", true, packageName)
+}
+
+def getChangedPkgs(oldVersions, newVersions) {
+ def common = new com.mirantis.mk.Common()
+ def changedPkgs = [:]
+ def updated = false
+ newVersions.each { k, v ->
+ changedPkgs[k] = [:]
+ if (v == null || !v['return'] || oldVersions[k] == null || !oldVersions[k]['return']) {
+ common.warningMsg("Can't detect package version changes for ceph-${k} packages")
+ changedPkgs[k]["*"] = true
+ updated = true
+ return
+ }
+
+ // since run was not in batch mode, get only 0 element which contains all output
+ v['return'][0].each { tgt, newPgkVersion ->
+ oldVersion = oldVersions[k]['return'][0].get(tgt, "")
+ if (oldVersion == newPgkVersion) {
+ changedPkgs[k][tgt] = false
+ return
+ }
+ common.infoMsg("${tgt} has updated ceph ${k} packages ${oldVersion} -> ${newPgkVersion}")
+ updated = true
+ changedPkgs[k][tgt] = true
+ }
+ }
+ return ["updated": updated, "changed": changedPkgs]
+}
+
+// if some map contains tgt and has true value - restart needed
+def needToRestart(infoPkgs, daemon, tgt) {
+ if (infoPkgs[daemon].get("*", false) || infoPkgs[daemon].get(tgt, false) || infoPkgs["common"].get(tgt, false)) {
+ return true
+ }
+ return false
+}
+
timeout(time: 12, unit: 'HOURS') {
node() {
try {
def targets = ["common": "ceph-common", "osd": "ceph-osd", "mon": "ceph-mon",
"mgr" : "ceph-mgr", "radosgw": "radosgw"]
+ def oldPackageVersions = [:]
+ def newPackageVersions = [:]
+
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
stage('Apply package upgrades on all nodes') {
targets.each { key, value ->
- salt.enforceState(pepperEnv, "I@ceph:${key}", 'linux.system.repo', true)
- command = "pkg.install"
- packages = value
- commandKwargs = ['only_upgrade': 'true', 'force_yes': 'true']
target = "I@ceph:${key}"
+ packages = value
+ // check package versions before upgrade to compare it after update run
+ oldPackageVersions[key] = collectPkgVersion(target, packages)
+ salt.enforceState(pepperEnv, target, 'linux.system.repo', true)
+ command = "pkg.install"
+ commandKwargs = ['only_upgrade': 'true', 'force_yes': 'true']
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, true, packages, commandKwargs)
salt.printSaltCommandResult(out)
+ // check package version after update
+ newPackageVersions[key] = collectPkgVersion(target, packages)
}
}
+ def packageChanges = getChangedPkgs(oldPackageVersions, newPackageVersions)
+
+ if (!packageChanges["updated"]) {
+ common.infoMsg("Ceph packages were not updated, skipping service restart")
+ return
+ }
+
stage('Set cluster flags') {
+ common.infoMsg("Ceph packages update detected, setting cluster noout flag")
if (flags.size() > 0) {
stage('Set cluster flags') {
for (flag in flags) {
@@ -49,23 +106,20 @@
}
}
- stage("Restart MONs") {
- selMinions = salt.getMinions(pepperEnv, "I@ceph:mon")
- for (tgt in selMinions) {
- // runSaltProcessStep 'service.restart' don't work for this services
- salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-mon.target")
- ceph.waitForHealthy(pepperEnv, tgt, flags)
- if (runHighState) {
- salt.enforceHighstate(pepperEnv, tgt)
- }
- }
- selMinions = salt.getMinions(pepperEnv, "I@ceph:mgr")
- for (tgt in selMinions) {
- // runSaltProcessStep 'service.restart' don't work for this services
- salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-mgr.target")
- ceph.waitForHealthy(pepperEnv, tgt, flags)
- if (runHighState) {
- salt.enforceHighstate(pepperEnv, tgt)
+ stage("Restart MONs/MGRs") {
+ ["mon", "mgr"].each {daemon ->
+ selMinions = salt.getMinions(pepperEnv, "I@ceph:${daemon}")
+ for (tgt in selMinions) {
+ if (!needToRestart(packageChanges["changed"], daemon, tgt)) {
+ common.infoMsg("Node ${tgt} has no updated ceph packages, skipping service restart on it")
+ continue
+ }
+ // runSaltProcessStep 'service.restart' don't work for this services
+ salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-${daemon}.target")
+ ceph.waitForHealthy(pepperEnv, tgt, flags)
+ if (runHighState) {
+ salt.enforceHighstate(pepperEnv, tgt)
+ }
}
}
}
@@ -74,6 +128,10 @@
def device_grain_name = "ceph_disk"
selMinions = salt.getMinions(pepperEnv, "I@ceph:osd")
for (tgt in selMinions) {
+ if (!needToRestart(packageChanges["changed"], "osd", tgt)) {
+ common.infoMsg("Node ${tgt} has no updated ceph packages, skipping service restart on it")
+ continue
+ }
salt.runSaltProcessStep(pepperEnv, tgt, 'saltutil.sync_grains', [], null, true, 5)
def ceph_disks = salt.getGrain(pepperEnv, tgt, 'ceph')['return'][0].values()[0].values()[0][device_grain_name]
@@ -103,6 +161,10 @@
stage('Restart RGWs') {
selMinions = salt.getMinions(pepperEnv, "I@ceph:radosgw")
for (tgt in selMinions) {
+ if (!needToRestart(packageChanges["changed"], "radosgw", tgt)) {
+ common.infoMsg("Node ${tgt} has no updated ceph packages, skipping service restart on it")
+ continue
+ }
salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-radosgw.target")
ceph.waitForHealthy(pepperEnv, tgt, flags)
if (runHighState) {