Merge the tip of origin/release/proposed/2019.2.0 into origin/release/2019.2.0
8161851 fix partition removal for ceph nautilus
be9d88f Remove tmp:tmp mount for tempest tests
acc7d28 Retry dist-upgrades
c05da4c Add correct salt minions update
2ce3c94 Revert "Run salt.minion and full refresh consistently"
Change-Id: Id373a601d1796edee1c1486f1ae6fbd86175815d
diff --git a/src/com/mirantis/mcp/Validate.groovy b/src/com/mirantis/mcp/Validate.groovy
index 9c08472..f9a6628 100644
--- a/src/com/mirantis/mcp/Validate.groovy
+++ b/src/com/mirantis/mcp/Validate.groovy
@@ -55,7 +55,6 @@
default_mounts = ["/etc/ssl/certs/": "/etc/ssl/certs/",
"/srv/salt/pki/${cluster_name}/": "/etc/certs",
"/root/test/": "/root/tempest/",
- "/tmp/": "/tmp/",
"/etc/hosts": "/etc/hosts"]
params.mounts = default_mounts + params.mounts
if ( salt.cmdRun(params.master, params.target, "docker ps -f name=^${params.name}\$ -q", false, null, false)['return'][0].values()[0] ) {
diff --git a/src/com/mirantis/mk/Ceph.groovy b/src/com/mirantis/mk/Ceph.groovy
index c959810..8660233 100644
--- a/src/com/mirantis/mk/Ceph.groovy
+++ b/src/com/mirantis/mk/Ceph.groovy
@@ -93,7 +93,7 @@
}
}
if (lvm_enabled && type != 'lockbox') {
- salt.cmdRun(master, target, "ceph-volume lvm zap ${partition_uuid} --destroy")
+ salt.cmdRun(master, target, "ceph-volume lvm zap /dev/disk/by-partuuid/${partition_uuid} --destroy")
} else if (dev != '') {
salt.cmdRun(master, target, "parted ${dev} rm ${part_id}")
} else {
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index 6e74379..47cd772 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -294,8 +294,20 @@
common.retry(3, 5) {
salt.cmdRun(env, target, 'salt-call pkg.refresh_db failhard=true', true, batch)
}
+
+ /* first try to upgrade salt components since they demand asynchronous upgrade */
+ upgradeSaltPackages(env, target)
def cmd = "export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ${mode}"
- salt.cmdRun(env, target, cmd, true, batch)
+
+ /*
+ * This is a long running batch operation that may return empty response
+ * which is a pretty typical salt behavior. This does not represent an error
+ * but might hide the error if it's ignored. If there is no persistent error
+ * with the procedure itself, the consequent run will succeed.
+ */
+ common.retry(2, 120) {
+ salt.cmdRun(env, target, cmd, true, batch)
+ }
rebootRequired = salt.runSaltProcessStep(env, target, 'file.file_exists', ['/var/run/reboot-required'], batch, true, 5)['return'][0].values()[0].toBoolean()
if (rebootRequired) {
@@ -310,3 +322,36 @@
common.errorMsg("Invalid upgrade mode specified: ${mode}. Has to be 'upgrade' or 'dist-upgrade'")
}
}
+
+/**
+* Upgrade salt packages on target asynchronously, wait minions' availability.
+*
+* @param env Salt Connection object or env Salt command map
+* @param target Salt target to upgrade packages on.
+* @param timeout Sleep timeout when doing retries.
+* @param attempts Number of attemps to wait for.
+*/
+def upgradeSaltPackages(env, target, timeout=60, attempts=20) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
+ def saltUpgradeCmd =
+ 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q ' +
+ '-o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" ' +
+ 'install --only-upgrade salt-master salt-common salt-api salt-minion'
+
+ common.infoMsg("Upgrading SaltStack on ${target}")
+ salt.cmdRun(env, target, saltUpgradeCmd, false, null, true, [], [], true)
+ /* wait for 2 mins before checking the availability of minions to give
+ apt some time to finish updating so the dpkg releases its locks */
+ sleep(120)
+ /* taken from upgrade-mcp-release */
+ common.retry(attempts, timeout) {
+ salt.minionsReachable(env, 'I@salt:master', target)
+ def running = salt.runSaltProcessStep(env, target, 'saltutil.running', [], null, true, 5)
+ for (value in running.get("return")[0].values()) {
+ if (value != []) {
+ throw new Exception("Not all salt-minions are ready for execution")
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index c8ff08f..2dacace 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -30,11 +30,11 @@
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['linux.system']])
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.master'], failOnError: false, read_timeout: 120, retries: 2])
- salt.fullRefresh(master, "* ${extra_tgt}", null, true)
+ salt.fullRefresh(master, "* ${extra_tgt}", batch)
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.minion'], failOnError: false, read_timeout: 60, retries: 2])
salt.enforceState([saltId: master, target: "I@salt:master ${extra_tgt}", state: ['salt.minion']])
- salt.fullRefresh(master, "* ${extra_tgt}", null, true)
+ salt.fullRefresh(master, "* ${extra_tgt}", batch)
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.network.proxy'], batch: batch, failOnError: false, read_timeout: 180, retries: 2])
// Make sure all repositories are in place before proceeding with package installation from other states
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.system.repo'], batch: batch, failOnError: false, read_timeout: 180, retries: 2])
@@ -52,13 +52,12 @@
sleep(5)
salt.enforceState([saltId: master, target: "I@linux:system ${extra_tgt}", state: ['linux', 'openssh', 'ntp', 'rsyslog'], batch: batch])
- def saltMinionNodes = salt.getMinions(master, "* ${extra_tgt}")
- for (minion in saltMinionNodes) {
- salt.enforceState([saltId: master, target: minion, state: ['salt.minion'], failOnError: false, batch: null, read_timeout: 180, retries: 2])
- }
+
+ salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['salt.minion'], failOnError: false, batch: batch, read_timeout: 180, retries: 2])
+
sleep(5)
- salt.fullRefresh(master, "* ${extra_tgt}", null, true)
+ salt.fullRefresh(master, "* ${extra_tgt}", batch)
salt.runSaltProcessStep(master, "* ${extra_tgt}", 'mine.update', [], batch, true)
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: ['linux.network.host'], batch: batch])
// WA for PROD-33911
@@ -1048,10 +1047,7 @@
// Install collectd, heka and sensu services on the nodes, this will also
// generate the metadata that goes into the grains and eventually into Salt Mine
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'collectd'])
- def saltMinionsNodes = salt.getMinions(master, "* ${extra_tgt}")
- for (minion in saltMinionsNodes) {
- salt.enforceState([saltId: master, target: minion, state: 'salt.minion', retries: 2])
- }
+ salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'salt.minion', retries: 2])
salt.enforceState([saltId: master, target: "* ${extra_tgt}", state: 'heka'])
// Gather the Grafana metadata as grains
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 30957e5..48309ff 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -332,9 +332,10 @@
* @param output do you want to print output
* @param saltArgs additional salt args eq. ["runas=aptly"]
* @param replacing list with maps for deletion in info message (passwords, logins, etc)
+ * @param async run commands with async client (default false)
* @return output of salt command
*/
-def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = [], replacing = []) {
+def cmdRun(saltId, target, cmd, checkResponse = true, batch=null, output = true, saltArgs = [], replacing = [], async = false) {
def common = new com.mirantis.mk.Common()
def originalCmd = cmd
common.infoSensitivityMsg("Running command ${cmd} on ${target}", true, replacing)
@@ -345,7 +346,8 @@
// add cmd name to salt args list
saltArgs << cmd
- def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.run', batch, saltArgs.reverse())
+ def client = async ? 'local_async' : 'local'
+ def out = runSaltCommand(saltId, client, ['expression': target, 'type': 'compound'], 'cmd.run', batch, saltArgs.reverse())
if (checkResponse) {
// iterate over all affected nodes and check success return code
if (out["return"]){
@@ -355,14 +357,14 @@
def nodeKey = node.keySet()[j]
if (node[nodeKey] instanceof String) {
if (!node[nodeKey].contains("Salt command execution success")) {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
}
} else if (node[nodeKey] instanceof Boolean) {
if (!node[nodeKey]) {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns: ${node[nodeKey]}")
}
} else {
- throw new Exception("Execution of cmd ${originalCmd} failed. Server returns unexpected data type: ${node[nodeKey]}")
+ throw new Exception("Execution of cmd ${originalCmd} failed. ${nodeKey} returns unexpected data type: ${node[nodeKey]}")
}
}
}