Run salt packages upgrade in async mode
Related-Prod: PROD-26381 (PROD:26381)
Change-Id: I6b7e52f8f345b8613fe6efef634c4ca74248a75c
(cherry picked from commit 0d0c65f51cf3e442b6f45a1cdb33b495a6970c63)
diff --git a/cloud-update.groovy b/cloud-update.groovy
index 3fd7723..99c661c 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -60,6 +60,17 @@
wait = "${MINIONS_TEST_TIMEOUT}".toInteger()
}
+def updateSaltPackage(pepperEnv, target, pkgs, masterUpdate = false) {
+ def salt = new com.mirantis.mk.Salt()
+ salt.cmdRun(pepperEnv, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
+ def minions_reachable = target
+ if (masterUpdate) {
+ // in case of update Salt Master packages - check all minions are good
+ minions_reachable = '*'
+ }
+ salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': target, 'target_reachable': minions_reachable])
+}
+
def updatePkgs(pepperEnv, target, targetType="", targetPackages="") {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
@@ -155,12 +166,10 @@
// salt master pkg
if (targetType == 'cfg') {
common.warningMsg('salt-master pkg upgrade, rerun the pipeline if disconnected')
- salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-master'], null, true, 5)
- salt.minionsReachable(pepperEnv, 'I@salt:master', '*', null, wait)
+ updateSaltPackage(pepperEnv, target, '["salt-master"]', true)
}
// salt minion pkg
- salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-minion'], null, true, 5)
- salt.minionsReachable(pepperEnv, 'I@salt:master', target, null, wait)
+ updateSaltPackage(pepperEnv, target, '["salt-minion"]')
common.infoMsg('Performing pkg upgrades ... ')
common.retry(3){
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, true, packages, commandKwargs)
diff --git a/update-package.groovy b/update-package.groovy
index 10f3a85..9d36f38 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -23,8 +23,15 @@
def packages
def command
def commandKwargs
-def installSaltStack(target, pkgs){
- salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ["force_yes=True", "pkgs='$pkgs'"], null, true, 30)
+
+def installSaltStack(target, pkgs, masterUpdate = false){
+ salt.cmdRun(pepperEnv, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
+ def minions_reachable = target
+ if (masterUpdate) {
+ // in case of update Salt Master packages - check all minions are good
+ minions_reachable = '*'
+ }
+ salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': target, 'target_reachable': minions_reachable])
}
timeout(time: 12, unit: 'HOURS') {
@@ -97,7 +104,7 @@
common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
common.retry(10, 5) {
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
+ installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
}
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
@@ -128,7 +135,7 @@
common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
common.retry(10, 5) {
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
+ installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
}
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 49f92c3..77c0a26 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -33,11 +33,9 @@
}
def updateSaltStack(target, pkgs) {
- // wait 2 mins when salt-* packages are updated which leads to salt-* services restart
- common.retry(2, 120) {
- salt.runSaltProcessStep(venvPepper, target, 'pkg.install', ["force_yes=True", "pkgs='$pkgs'"], null, true, 5)
- }
-
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
+ // can't use same function from pipeline lib, as at the moment of running upgrade pipeline Jenkins
+ // still using pipeline lib from current old mcp-version
common.retry(20, 60) {
salt.minionsReachable(venvPepper, 'I@salt:master', '*')
def running = salt.runSaltProcessStep(venvPepper, target, 'saltutil.running', [], null, true, 5)