Merge the tip of origin/release/proposed/2019.2.0 into origin/release/2019.2.0
318fc89 Add correct doc link to radosgw class import issue
8b7008f Add check for rgw.yml classes import order
8efa75a Match update repo pattern at end of line
20d6367 Reorder jenkins upgrade steps
5f335e3 [ceph] Do not restart Ceph services in packages are not updated
9a7295a Optimise the backup node choosing method
7a47b7f Fix logic for ceph offline node remove
Change-Id: I3bcbdbf202fdc93139c5b68e91adf284412d33d7
diff --git a/ceph-remove-node.groovy b/ceph-remove-node.groovy
index 42c92db..d8a6d0e 100644
--- a/ceph-remove-node.groovy
+++ b/ceph-remove-node.groovy
@@ -39,9 +39,19 @@
// create connection to salt master
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ def target = salt.getMinions(pepperEnv, HOST)
+ if(target.isEmpty()) {
+ common.errorMsg("Host not found")
+ throw new InterruptedException()
+ }
+ else if(target.size() > 1) {
+ common.errorMsg("$HOST targeted more than one minion")
+ throw new InterruptedException()
+ }
+
if (osdNodeUnavailable) {
stage('Remove unavailable OSD node') {
- osdHostName = salt.stripDomainName("${HOST}")
+ osdHostName = salt.stripDomainName("${target[0]}")
osdTreeString = ceph.cmdRun(pepperEnv, "ceph osd tree --format json-pretty")
osdTree = common.parseJSON(osdTreeString)
osdIDs = []
@@ -58,24 +68,18 @@
common.infoMsg("Found next OSDs for host ${HOST} (${osdHostName}): ${osdIDs}")
input message: "Do you want to continue node remove?"
for (osdId in osdIDs) {
- ceph.cmdRun(master, "ceph osd purge ${osdId} --yes-i-really-mean-it", true, true)
+ ceph.cmdRun(pepperEnv, "ceph osd purge ${osdId} --yes-i-really-mean-it", true, true)
}
salt.cmdRun(pepperEnv, "I@salt:master", "salt-key -d ${HOST} --include-all -y")
+
+ if(safeRemove) {
+ ceph.waitForHealthy(pepperEnv, flags)
+ }
}
}
return
}
- def target = salt.getMinions(pepperEnv, HOST)
- if(target.isEmpty()) {
- common.errorMsg("Host not found")
- throw new InterruptedException()
- }
- else if(target.size() > 1) {
- common.errorMsg("$HOST targeted more than one minion")
- throw new InterruptedException()
- }
-
salt.fullRefresh(pepperEnv, HOST)
stage('Set flags') {
diff --git a/galera-database-backup-pipeline.groovy b/galera-database-backup-pipeline.groovy
index 8239aa6..466c7ea 100644
--- a/galera-database-backup-pipeline.groovy
+++ b/galera-database-backup-pipeline.groovy
@@ -8,6 +8,8 @@
primaryNodes = []
syncedNodes = []
galeraMembers = []
+maxValue = 0
+masterNode = "none"
if (common.validInputParam('OVERRIDE_BACKUP_NODE')) {
backupNode = OVERRIDE_BACKUP_NODE
@@ -37,8 +39,29 @@
}
}
stage('Choose backup node') {
- backupNode = primaryNodes.sort()[0] // STEP 2 - Use node with lowest hostname number (last option if everything previous fails)
- }
+ primaryNodes = primaryNodes.sort()
+ for (node in primaryNodes.reverse()) { // Looking for the node with the highest mysql_queries value
+ try {
+ queriesValue = galera.getWsrepParameters(pepperEnv, node, "Queries", false).get('Queries')
+ common.infoMsg("Queries for ${node} - ${queriesValue}")
+ if (queriesValue >= maxValue) {
+ maxValue = queriesValue
+ masterNode = node
+ common.infoMsg("Current main node in galera cluster: ${masterNode}")
+ }
+ } catch (Exception e) {
+ common.warningMsg("Minion '${node}' unavailable.")
+ }
+ }
+ if (masterNode.equals("none")) {
+ backupNode = primaryNodes[-1] // STEP 2 - Use node with highest hostname number (last option if everything previous fails)
+ } else {
+ common.infoMsg("Master node is ${masterNode}.")
+ primaryNodes = primaryNodes.minus(masterNode)
+ backupNode = primaryNodes[-1]
+ }
+ common.infoMsg("Choose backup node as ${backupNode}.")
+ }
} else {
stage('Choose backup node') {
common.infoMsg("Backup node backup was overriden to ${backupNode}.")
@@ -72,4 +95,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/update-ceph.groovy b/update-ceph.groovy
index 6cef0b3..96c8836 100644
--- a/update-ceph.groovy
+++ b/update-ceph.groovy
@@ -7,6 +7,7 @@
*/
pepperEnv = "pepperEnv"
+def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def ceph = new com.mirantis.mk.Ceph()
def python = new com.mirantis.mk.Python()
@@ -17,29 +18,85 @@
def flags = CLUSTER_FLAGS ? CLUSTER_FLAGS.tokenize(',') : []
def runHighState = RUNHIGHSTATE
+def collectPkgVersion(target, packageName) {
+ def salt = new com.mirantis.mk.Salt()
+ return salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], "pkg.version", true, packageName)
+}
+
+def getChangedPkgs(oldVersions, newVersions) {
+ def common = new com.mirantis.mk.Common()
+ def changedPkgs = [:]
+ def updated = false
+ newVersions.each { k, v ->
+ changedPkgs[k] = [:]
+ if (v == null || !v['return'] || oldVersions[k] == null || !oldVersions[k]['return']) {
+ common.warningMsg("Can't detect package version changes for ceph-${k} packages")
+ changedPkgs[k]["*"] = true
+ updated = true
+ return
+ }
+
+ // since run was not in batch mode, get only 0 element which contains all output
+ v['return'][0].each { tgt, newPgkVersion ->
+ oldVersion = oldVersions[k]['return'][0].get(tgt, "")
+ if (oldVersion == newPgkVersion) {
+ changedPkgs[k][tgt] = false
+ return
+ }
+ common.infoMsg("${tgt} has updated ceph ${k} packages ${oldVersion} -> ${newPgkVersion}")
+ updated = true
+ changedPkgs[k][tgt] = true
+ }
+ }
+ return ["updated": updated, "changed": changedPkgs]
+}
+
+// if some map contains tgt and has true value - restart needed
+def needToRestart(infoPkgs, daemon, tgt) {
+ if (infoPkgs[daemon].get("*", false) || infoPkgs[daemon].get(tgt, false) || infoPkgs["common"].get(tgt, false)) {
+ return true
+ }
+ return false
+}
+
timeout(time: 12, unit: 'HOURS') {
node() {
try {
def targets = ["common": "ceph-common", "osd": "ceph-osd", "mon": "ceph-mon",
"mgr" : "ceph-mgr", "radosgw": "radosgw"]
+ def oldPackageVersions = [:]
+ def newPackageVersions = [:]
+
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
stage('Apply package upgrades on all nodes') {
targets.each { key, value ->
- salt.enforceState(pepperEnv, "I@ceph:${key}", 'linux.system.repo', true)
- command = "pkg.install"
- packages = value
- commandKwargs = ['only_upgrade': 'true', 'force_yes': 'true']
target = "I@ceph:${key}"
+ packages = value
+ // check package versions before upgrade to compare it after update run
+ oldPackageVersions[key] = collectPkgVersion(target, packages)
+ salt.enforceState(pepperEnv, target, 'linux.system.repo', true)
+ command = "pkg.install"
+ commandKwargs = ['only_upgrade': 'true', 'force_yes': 'true']
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, true, packages, commandKwargs)
salt.printSaltCommandResult(out)
+ // check package version after update
+ newPackageVersions[key] = collectPkgVersion(target, packages)
}
}
+ def packageChanges = getChangedPkgs(oldPackageVersions, newPackageVersions)
+
+ if (!packageChanges["updated"]) {
+ common.infoMsg("Ceph packages were not updated, skipping service restart")
+ return
+ }
+
stage('Set cluster flags') {
+ common.infoMsg("Ceph packages update detected, setting cluster noout flag")
if (flags.size() > 0) {
stage('Set cluster flags') {
for (flag in flags) {
@@ -49,23 +106,20 @@
}
}
- stage("Restart MONs") {
- selMinions = salt.getMinions(pepperEnv, "I@ceph:mon")
- for (tgt in selMinions) {
- // runSaltProcessStep 'service.restart' don't work for this services
- salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-mon.target")
- ceph.waitForHealthy(pepperEnv, tgt, flags)
- if (runHighState) {
- salt.enforceHighstate(pepperEnv, tgt)
- }
- }
- selMinions = salt.getMinions(pepperEnv, "I@ceph:mgr")
- for (tgt in selMinions) {
- // runSaltProcessStep 'service.restart' don't work for this services
- salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-mgr.target")
- ceph.waitForHealthy(pepperEnv, tgt, flags)
- if (runHighState) {
- salt.enforceHighstate(pepperEnv, tgt)
+ stage("Restart MONs/MGRs") {
+ ["mon", "mgr"].each {daemon ->
+ selMinions = salt.getMinions(pepperEnv, "I@ceph:${daemon}")
+ for (tgt in selMinions) {
+ if (!needToRestart(packageChanges["changed"], daemon, tgt)) {
+ common.infoMsg("Node ${tgt} has no updated ceph packages, skipping service restart on it")
+ continue
+ }
+ // runSaltProcessStep 'service.restart' don't work for this services
+ salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-${daemon}.target")
+ ceph.waitForHealthy(pepperEnv, tgt, flags)
+ if (runHighState) {
+ salt.enforceHighstate(pepperEnv, tgt)
+ }
}
}
}
@@ -74,6 +128,10 @@
def device_grain_name = "ceph_disk"
selMinions = salt.getMinions(pepperEnv, "I@ceph:osd")
for (tgt in selMinions) {
+ if (!needToRestart(packageChanges["changed"], "osd", tgt)) {
+ common.infoMsg("Node ${tgt} has no updated ceph packages, skipping service restart on it")
+ continue
+ }
salt.runSaltProcessStep(pepperEnv, tgt, 'saltutil.sync_grains', [], null, true, 5)
def ceph_disks = salt.getGrain(pepperEnv, tgt, 'ceph')['return'][0].values()[0].values()[0][device_grain_name]
@@ -103,6 +161,10 @@
stage('Restart RGWs') {
selMinions = salt.getMinions(pepperEnv, "I@ceph:radosgw")
for (tgt in selMinions) {
+ if (!needToRestart(packageChanges["changed"], "radosgw", tgt)) {
+ common.infoMsg("Node ${tgt} has no updated ceph packages, skipping service restart on it")
+ continue
+ }
salt.cmdRun(pepperEnv, tgt, "systemctl restart ceph-radosgw.target")
ceph.waitForHealthy(pepperEnv, tgt, flags)
if (runHighState) {
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index c067861..a6e5358 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -267,6 +267,54 @@
}
}
+// ceph cluster class ordering for radosgw
+def check_36461(String cluster_name){
+ if (!salt.testTarget(venvPepper, 'I@ceph:radosgw')) {
+ return
+ }
+ def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
+ def checkFile = "${clusterModelPath}/ceph/rgw.yml"
+ def saltTarget = "I@salt:master"
+ try {
+ salt.cmdRun(venvPepper, saltTarget, "test -f ${checkFile}")
+ }
+ catch (Exception e) {
+ common.warningMsg("Unable to check ordering of RadosGW imports, file ${checkFile} not found, skipping")
+ return
+ }
+ def fileContent = salt.cmdRun(venvPepper, saltTarget, "cat ${checkFile}").get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ def yamlData = readYaml text: fileContent
+ def infraClassImport = "cluster.${cluster_name}.infra"
+ def cephClassImport = "cluster.${cluster_name}.ceph"
+ def cephCommonClassImport = "cluster.${cluster_name}.ceph.common"
+ def infraClassFound = false
+ def importErrorDetected = false
+ def importErrorMessage = """Ceph classes in '${checkFile}' are used in wrong order! Please reorder it:
+'${infraClassImport}' should be placed before '${cephClassImport}' and '${cephCommonClassImport}'.
+For additional information please see https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-15/mu-15-addressed/mu-15-dtrain/mu-15-dtrain-manual.html"""
+ for (yamlClass in yamlData.classes) {
+ switch(yamlClass){
+ case infraClassImport:
+ infraClassFound = true;
+ break;
+ case cephClassImport:
+ if (!infraClassFound) {
+ importErrorDetected = true
+ };
+ break;
+ case cephCommonClassImport:
+ if (!infraClassFound) {
+ importErrorDetected = true
+ };
+ break;
+ }
+ }
+ if (importErrorDetected) {
+ common.errorMsg(importErrorMessage)
+ error(importErrorMessage)
+ }
+}
+
def wa32182(String cluster_name) {
if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
@@ -681,6 +729,7 @@
check_34406(cluster_name)
check_35705(cluster_name)
check_35884(cluster_name)
+ check_36461(cluster_name)
common.infoMsg('Perform: Validate reclass medata before processing')
validateReclassModel(minions, 'before')
@@ -714,7 +763,7 @@
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.contrail' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.contrail/system.linux.system.repo.mcp.apt_mirantis.contrail/g'")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
- "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.updates/system.linux.system.repo.mcp.apt_mirantis.update/g'")
+ "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates\$' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.updates\$/system.linux.system.repo.mcp.apt_mirantis.update/g'")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
@@ -894,7 +943,7 @@
}
}
- stage('Update Drivetrain') {
+ stage('Update Drivetrain: Part 1') {
salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.repo', true, true, batchSize, false, 60, 2)
salt.enforceState(venvPepper, '*', 'linux.system.package', true, true, batchSize, false, 60, 2)
@@ -928,15 +977,6 @@
if (nginxAtMaster.toString().toLowerCase() == 'true') {
salt.enforceState(venvPepper, 'I@salt:master', 'nginx', true, true, null, false, 60, 2)
}
-
- // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
- def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
- def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
- salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
-
- salt.enforceStateWithTest(venvPepper, 'I@jenkins:client:security and not I@salt:master', 'jenkins.client.security', "", true, true, null, true, 60, 2)
- salt.enforceStateWithTest(venvPepper, 'I@jenkins:client and I@docker:client:images and not I@salt:master', 'docker.client.images', "", true, true, null, true, 60, 2)
- salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
}
}
catch (Throwable e) {
@@ -945,39 +985,13 @@
throw e
}
}
- // docker.client state may trigger change of jenkins master or jenkins slave services,
- // so we need wait for slave to reconnect and continue pipeline
- sleep(180)
- def cidNodes = []
- node('python') {
- try {
- stage('Update Drivetrain: Phase 2') {
- python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
- checkCICDDocker()
-
- // Apply changes for HaProxy on CI/CD nodes
- salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
- salt.upgradePackageAndRestartSaltMinion(venvPepper, 'I@jenkins:client and not I@salt:master', 'python-jenkins')
- salt.enforceState(venvPepper, 'I@jenkins:client and not I@salt:master', 'jenkins.client', true, true, null, false, 60, 2)
-
- // update Nginx proxy settings for Jenkins/Gerrit if needed
- if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
- salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
- }
- }
- if (packageUpgradeMode) {
- cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
- }
- }
- catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
- currentBuild.result = "FAILURE"
- throw e
- }
- }
-
stage('Upgrade OS') {
if (packageUpgradeMode) {
+ def cidNodes = []
+ node('python') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
+ }
def debian = new com.mirantis.mk.Debian()
def statusFile = '/tmp/rebooted_during_upgrade'
for(cidNode in cidNodes) {
@@ -1002,4 +1016,43 @@
common.infoMsg('Upgrade OS skipped...')
}
}
+
+ node('python') {
+ stage('Update Drivetrain: Part 2') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
+ def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
+ def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
+ salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
+
+ salt.enforceStateWithTest(venvPepper, 'I@jenkins:client:security and not I@salt:master', 'jenkins.client.security', "", true, true, null, true, 60, 2)
+ salt.enforceStateWithTest(venvPepper, 'I@jenkins:client and I@docker:client:images and not I@salt:master', 'docker.client.images', "", true, true, null, true, 60, 2)
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
+ }
+ }
+ // docker.client state may trigger change of jenkins master or jenkins slave services,
+ // so we need wait for slave to reconnect and continue pipeline
+ sleep(180)
+ node('python') {
+ stage('Update Drivetrain: Part 3') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ checkCICDDocker()
+
+ // update Nginx proxy settings for Jenkins/Gerrit if needed
+ if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
+ salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
+ }
+ // Apply changes for HaProxy on CI/CD nodes
+ salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
+ salt.upgradePackageAndRestartSaltMinion(venvPepper, 'I@jenkins:client and not I@salt:master', 'python-jenkins')
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and not I@salt:master' state.sls jenkins.client --async")
+
+ common.warningMsg("Jenkins update started in background in order to handle plugin post-install issues.")
+ common.warningMsg("Please wait until it finished. Jenkins could be restarted during this procedure.")
+ common.warningMsg("You can monitor job progress by running 'salt-run jobs.active' on master node")
+ common.warningMsg("For ensuring that upgrade is done and there are no errors you can run the following command on salt master node")
+ common.warningMsg("salt-run jobs.lookup_jid %salt_job_id%")
+ common.warningMsg("Salt job ID could be found in the log above.")
+ }
+ }
}