Update pipeline for package updates
Related-Prod: PROD-29366 (PROD:29366)
Change-Id: I6c97d2b38f48c062f33d9ae7d9146e252c5f7101
diff --git a/update-package.groovy b/update-package.groovy
index 9d36f38..df7655b 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -6,23 +6,12 @@
* SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* TARGET_PACKAGES Space delimited list of packages to be updates [package1=version package2=version], empty string means all updating all packages to the latest version.
- * TARGET_SUBSET_TEST Number of nodes to list package updates, empty string means all targetted nodes.
- * TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
- * TARGET_BATCH_LIVE Batch size for the complete live package update on all nodes, empty string means apply to all targetted nodes.
*
**/
+
pepperEnv = "pepperEnv"
salt = new com.mirantis.mk.Salt()
-def common = new com.mirantis.mk.Common()
-def python = new com.mirantis.mk.Python()
-def targetTestSubset
-def targetLiveSubset
-def targetLiveAll
-def minions
-def result
-def packages
-def command
-def commandKwargs
+common = new com.mirantis.mk.Common()
def installSaltStack(target, pkgs, masterUpdate = false){
salt.cmdRun(pepperEnv, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
@@ -31,131 +20,76 @@
// in case of update Salt Master packages - check all minions are good
minions_reachable = '*'
}
- salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': target, 'target_reachable': minions_reachable])
+ salt.checkTargetMinionsReady(['saltId': pepperEnv, 'target': target, 'target_reachable': minions_reachable])
}
timeout(time: 12, unit: 'HOURS') {
node() {
try {
-
+ def python = new com.mirantis.mk.Python()
+ def command = 'pkg.upgrade'
+ def commandKwargs = null
+ def packages = null
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
- stage('List target servers') {
- minions = salt.getMinions(pepperEnv, TARGET_SERVERS)
-
+ def targetLiveAll = ''
+ stage('Get target servers') {
+ def minions = salt.getMinions(pepperEnv, TARGET_SERVERS)
if (minions.isEmpty()) {
throw new Exception("No minion was targeted")
}
-
- if (TARGET_SUBSET_TEST != "") {
- targetTestSubset = minions.subList(0, Integer.valueOf(TARGET_SUBSET_TEST)).join(' or ')
- } else {
- targetTestSubset = minions.join(' or ')
- }
- targetLiveSubset = minions.subList(0, Integer.valueOf(TARGET_SUBSET_LIVE)).join(' or ')
-
targetLiveAll = minions.join(' or ')
common.infoMsg("Found nodes: ${targetLiveAll}")
- common.infoMsg("Selected test nodes: ${targetTestSubset}")
- common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
}
stage("List package upgrades") {
- common.infoMsg("Listing all the packages that have a new update available on test nodes: ${targetTestSubset}")
- salt.runSaltProcessStep(pepperEnv, targetTestSubset, 'pkg.list_upgrades', [], null, true)
- if(TARGET_PACKAGES != "" && TARGET_PACKAGES != "*"){
- common.infoMsg("Note that only the ${TARGET_PACKAGES} would be installed from the above list of available updates on the ${targetTestSubset}")
- }
- }
-
- stage('Confirm live package upgrades on sample') {
- if(TARGET_PACKAGES==""){
- timeout(time: 2, unit: 'HOURS') {
- def userInput = input(
- id: 'userInput', message: 'Insert package names for update', parameters: [
- [$class: 'TextParameterDefinition', defaultValue: '', description: 'Package names (or *)', name: 'packages']
- ])
- if(userInput!= "" && userInput!= "*"){
- TARGET_PACKAGES = userInput
- }
- }
- }else{
- timeout(time: 2, unit: 'HOURS') {
- input message: "Approve live package upgrades on ${targetLiveSubset} nodes?"
- }
- }
- }
-
- if (TARGET_PACKAGES != "") {
- command = "pkg.install"
- packages = TARGET_PACKAGES.tokenize(' ')
- commandKwargs = ['only_upgrade': 'true']
- }else {
- command = "pkg.upgrade"
- packages = null
- }
-
- stage('Apply package upgrades on sample') {
- if(packages == null || packages.contains("salt-master") || packages.contains("salt-common") || packages.contains("salt-minion") || packages.contains("salt-api")){
- def saltTargets = (targetLiveSubset.split(' or ').collect{it as String})
- for(int i = 0; i < saltTargets.size(); i++ ){
- common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
- common.retry(10, 5) {
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
- }
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
- }
- }
- }
- }
- out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, packages, commandKwargs)
- salt.printSaltCommandResult(out)
- for(value in out.get("return")[0].values()){
- if (value.containsKey('result') && value.result == false) {
- throw new Exception("The package upgrade on sample node has failed. Please check the Salt run result above for more information.")
- }
+ common.infoMsg("Listing all the packages that have a new update available on nodes: ${targetLiveAll}")
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'pkg.list_upgrades', [], null, true)
+ if (TARGET_PACKAGES != '' && TARGET_PACKAGES != '*') {
+ common.warningMsg("Note that only the \"${TARGET_PACKAGES}\" would be installed from the above list of available updates on the ${targetLiveAll}")
+ command = "pkg.install"
+ packages = TARGET_PACKAGES.tokenize(' ')
+ commandKwargs = ['only_upgrade': 'true']
}
}
stage('Confirm package upgrades on all nodes') {
timeout(time: 2, unit: 'HOURS') {
- input message: "Approve live package upgrades on ${targetLiveAll} nodes?"
+ input message: "Approve package upgrades on ${targetLiveAll} nodes?"
}
}
stage('Apply package upgrades on all nodes') {
-
- if(packages == null || packages.contains("salt-master") || packages.contains("salt-common") || packages.contains("salt-minion") || packages.contains("salt-api")){
- def saltTargets = (targetLiveAll.split(' or ').collect{it as String})
- for(int i = 0; i < saltTargets.size(); i++ ){
- common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
+ if (packages == null || packages.contains("salt-master") || packages.contains("salt-common") || packages.contains("salt-minion") || packages.contains("salt-api")) {
+ common.warningMsg('Detected update for some Salt package (master or minion). Updating it first.')
+ def saltTargets = (targetLiveAll.split(' or ').collect { it as String })
+ for (int i = 0; i < saltTargets.size(); i++ ) {
common.retry(10, 5) {
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
+ if (salt.getMinions(pepperEnv, "I@salt:master and ${saltTargets[i]}")) {
installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
- }
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
+ } else if (salt.getMinions(pepperEnv, "I@salt:minion and not I@salt:master and ${saltTargets[i]}")) {
installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
+ } else {
+ error("Minion ${saltTargets[i]} is not reachable!")
}
}
}
}
-
+ common.infoMsg('Starting package upgrades...')
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, packages, commandKwargs)
salt.printSaltCommandResult(out)
for(value in out.get("return")[0].values()){
if (value.containsKey('result') && value.result == false) {
- throw new Exception("The package upgrade on sample node has failed. Please check the Salt run result above for more information.")
+ throw new Exception("The package upgrade on nodes has failed. Please check the Salt run result above for more information.")
}
}
- common.warningMsg("Pipeline has finished successfully, but please, check if any packages have been kept back.")
}
+ common.warningMsg("Pipeline has finished successfully, but please, check if any packages have been kept back.")
+
} catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
currentBuild.result = "FAILURE"
currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
throw e