Reorder jenkins upgrade steps
New plugins will break jenkins running job leading to big trace in
console. To avoid this we should run plugin update at last step using
salt async call.
PROD-36170
Change-Id: If7b696b1467f5e61a9cc9cb80e366321149e20cc
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index c067861..a6f609e 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -894,7 +894,7 @@
}
}
- stage('Update Drivetrain') {
+ stage('Update Drivetrain: Part 1') {
salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.repo', true, true, batchSize, false, 60, 2)
salt.enforceState(venvPepper, '*', 'linux.system.package', true, true, batchSize, false, 60, 2)
@@ -928,15 +928,6 @@
if (nginxAtMaster.toString().toLowerCase() == 'true') {
salt.enforceState(venvPepper, 'I@salt:master', 'nginx', true, true, null, false, 60, 2)
}
-
- // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
- def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
- def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
- salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
-
- salt.enforceStateWithTest(venvPepper, 'I@jenkins:client:security and not I@salt:master', 'jenkins.client.security', "", true, true, null, true, 60, 2)
- salt.enforceStateWithTest(venvPepper, 'I@jenkins:client and I@docker:client:images and not I@salt:master', 'docker.client.images', "", true, true, null, true, 60, 2)
- salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
}
}
catch (Throwable e) {
@@ -945,39 +936,13 @@
throw e
}
}
- // docker.client state may trigger change of jenkins master or jenkins slave services,
- // so we need wait for slave to reconnect and continue pipeline
- sleep(180)
- def cidNodes = []
- node('python') {
- try {
- stage('Update Drivetrain: Phase 2') {
- python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
- checkCICDDocker()
-
- // Apply changes for HaProxy on CI/CD nodes
- salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
- salt.upgradePackageAndRestartSaltMinion(venvPepper, 'I@jenkins:client and not I@salt:master', 'python-jenkins')
- salt.enforceState(venvPepper, 'I@jenkins:client and not I@salt:master', 'jenkins.client', true, true, null, false, 60, 2)
-
- // update Nginx proxy settings for Jenkins/Gerrit if needed
- if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
- salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
- }
- }
- if (packageUpgradeMode) {
- cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
- }
- }
- catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
- currentBuild.result = "FAILURE"
- throw e
- }
- }
-
stage('Upgrade OS') {
if (packageUpgradeMode) {
+ def cidNodes = []
+ node('python') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
+ }
def debian = new com.mirantis.mk.Debian()
def statusFile = '/tmp/rebooted_during_upgrade'
for(cidNode in cidNodes) {
@@ -1002,4 +967,43 @@
common.infoMsg('Upgrade OS skipped...')
}
}
+
+ node('python') {
+ stage('Update Drivetrain: Part 2') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
+ def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
+ def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
+ salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
+
+ salt.enforceStateWithTest(venvPepper, 'I@jenkins:client:security and not I@salt:master', 'jenkins.client.security', "", true, true, null, true, 60, 2)
+ salt.enforceStateWithTest(venvPepper, 'I@jenkins:client and I@docker:client:images and not I@salt:master', 'docker.client.images', "", true, true, null, true, 60, 2)
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
+ }
+ }
+ // docker.client state may trigger change of jenkins master or jenkins slave services,
+ // so we need wait for slave to reconnect and continue pipeline
+ sleep(180)
+ node('python') {
+ stage('Update Drivetrain: Part 3') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ checkCICDDocker()
+
+ // update Nginx proxy settings for Jenkins/Gerrit if needed
+ if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
+ salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
+ }
+ // Apply changes for HaProxy on CI/CD nodes
+ salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
+ salt.upgradePackageAndRestartSaltMinion(venvPepper, 'I@jenkins:client and not I@salt:master', 'python-jenkins')
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and not I@salt:master' state.sls jenkins.client --async")
+
+ common.warningMsg("Jenkins update started in background in order to handle plugin post-install issues.")
+ common.warningMsg("Please wait until it finished. Jenkins could be restarted during this procedure.")
+ common.warningMsg("You can monitor job progress by running 'salt-run jobs.active' on master node")
+ common.warningMsg("For ensuring that upgrade is done and there are no errors you can run the following command on salt master node")
+ common.warningMsg("salt-run jobs.lookup_jid %salt_job_id%")
+ common.warningMsg("Salt job ID could be found in the log above.")
+ }
+ }
}