Merge "Reorder jenkins upgrade steps" into release/proposed/2019.2.0
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index c067861..a6f609e 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -894,7 +894,7 @@
}
}
- stage('Update Drivetrain') {
+ stage('Update Drivetrain: Part 1') {
salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.repo', true, true, batchSize, false, 60, 2)
salt.enforceState(venvPepper, '*', 'linux.system.package', true, true, batchSize, false, 60, 2)
@@ -928,15 +928,6 @@
if (nginxAtMaster.toString().toLowerCase() == 'true') {
salt.enforceState(venvPepper, 'I@salt:master', 'nginx', true, true, null, false, 60, 2)
}
-
- // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
- def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
- def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
- salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
-
- salt.enforceStateWithTest(venvPepper, 'I@jenkins:client:security and not I@salt:master', 'jenkins.client.security', "", true, true, null, true, 60, 2)
- salt.enforceStateWithTest(venvPepper, 'I@jenkins:client and I@docker:client:images and not I@salt:master', 'docker.client.images', "", true, true, null, true, 60, 2)
- salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
}
}
catch (Throwable e) {
@@ -945,39 +936,13 @@
throw e
}
}
- // docker.client state may trigger change of jenkins master or jenkins slave services,
- // so we need wait for slave to reconnect and continue pipeline
- sleep(180)
- def cidNodes = []
- node('python') {
- try {
- stage('Update Drivetrain: Phase 2') {
- python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
- checkCICDDocker()
-
- // Apply changes for HaProxy on CI/CD nodes
- salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
- salt.upgradePackageAndRestartSaltMinion(venvPepper, 'I@jenkins:client and not I@salt:master', 'python-jenkins')
- salt.enforceState(venvPepper, 'I@jenkins:client and not I@salt:master', 'jenkins.client', true, true, null, false, 60, 2)
-
- // update Nginx proxy settings for Jenkins/Gerrit if needed
- if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
- salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
- }
- }
- if (packageUpgradeMode) {
- cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
- }
- }
- catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
- currentBuild.result = "FAILURE"
- throw e
- }
- }
-
stage('Upgrade OS') {
if (packageUpgradeMode) {
+ def cidNodes = []
+ node('python') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
+ }
def debian = new com.mirantis.mk.Debian()
def statusFile = '/tmp/rebooted_during_upgrade'
for(cidNode in cidNodes) {
@@ -1002,4 +967,43 @@
common.infoMsg('Upgrade OS skipped...')
}
}
+
+ node('python') {
+ stage('Update Drivetrain: Part 2') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
+ def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
+ def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
+ salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
+
+ salt.enforceStateWithTest(venvPepper, 'I@jenkins:client:security and not I@salt:master', 'jenkins.client.security', "", true, true, null, true, 60, 2)
+ salt.enforceStateWithTest(venvPepper, 'I@jenkins:client and I@docker:client:images and not I@salt:master', 'docker.client.images', "", true, true, null, true, 60, 2)
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
+ }
+ }
+ // docker.client state may trigger change of jenkins master or jenkins slave services,
+ // so we need wait for slave to reconnect and continue pipeline
+ sleep(180)
+ node('python') {
+ stage('Update Drivetrain: Part 3') {
+ python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
+ checkCICDDocker()
+
+ // update Nginx proxy settings for Jenkins/Gerrit if needed
+ if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
+ salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
+ }
+ // Apply changes for HaProxy on CI/CD nodes
+ salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
+ salt.upgradePackageAndRestartSaltMinion(venvPepper, 'I@jenkins:client and not I@salt:master', 'python-jenkins')
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and not I@salt:master' state.sls jenkins.client --async")
+
+ common.warningMsg("Jenkins update started in background in order to handle plugin post-install issues.")
+ common.warningMsg("Please wait until it finished. Jenkins could be restarted during this procedure.")
+ common.warningMsg("You can monitor job progress by running 'salt-run jobs.active' on master node")
+ common.warningMsg("For ensuring that upgrade is done and there are no errors you can run the following command on salt master node")
+ common.warningMsg("salt-run jobs.lookup_jid %salt_job_id%")
+ common.warningMsg("Salt job ID could be found in the log above.")
+ }
+ }
}