Merge "Split restoreGaleraDb function and add restoreType param support" into release/proposed/2019.2.0
diff --git a/src/com/mirantis/mk/Galera.groovy b/src/com/mirantis/mk/Galera.groovy
index acf5e9e..384caaa 100644
--- a/src/com/mirantis/mk/Galera.groovy
+++ b/src/com/mirantis/mk/Galera.groovy
@@ -97,6 +97,25 @@
common.errorMsg("No Galera slave was reachable.")
return 130
}
+ def checkTargets = salt.getMinions(env, "I@xtrabackup:client or I@xtrabackup:server")
+ for (checkTarget in checkTargets) {
+ def nodeStatus = salt.minionsReachable(env, 'I@salt:master', checkTarget, null, 10, 5)
+ if (nodeStatus != null) {
+ def iostatRes = salt.getIostatValues(['saltId': env, 'target': checkTarget, 'parameterName': "%util", 'output': true])
+ if (iostatRes == [:]) {
+ common.errorMsg("Recevived empty response from iostat call on ${checkTarget}. Maybe 'sysstat' package is not installed?")
+ return 140
+ }
+ for (int i = 0; i < iostatRes.size(); i++) {
+ def diskKey = iostatRes.keySet()[i]
+ if (!(iostatRes[diskKey].toString().isBigDecimal() && (iostatRes[diskKey].toBigDecimal() < 50 ))) {
+ common.errorMsg("Disk ${diskKey} has to high i/o utilization. Maximum value is 50 and current value is ${iostatRes[diskKey]}.")
+ return 141
+ }
+ }
+ }
+ }
+ common.infoMsg("Disk i/o utilization was checked and everything seems to be in order.")
if (checkTimeSync && !salt.checkClusterTimeSync(env, "I@galera:master or I@galera:slave")) {
common.errorMsg("Time in cluster is desynchronized or it couldn't be detemined. You should fix this issue manually before proceeding.")
return 131
@@ -170,7 +189,7 @@
for (key in parameters.keySet()) {
param = parameters.get(key)
if (key == 'wsrep_local_recv_queue_avg' || key == 'wsrep_local_send_queue_avg') {
- if (param.get('actualValue') > param.get('expectedThreshold').get('error')) {
+ if (param.get('actualValue') == null || (param.get('actualValue') > param.get('expectedThreshold').get('error'))) {
param << [match: 'error']
} else if (param.get('actualValue') > param.get('expectedThreshold').get('warn')) {
param << [match: 'warn']
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 1dca3e3..b04a59a 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -531,6 +531,42 @@
}
/**
+ * Restart and wait for salt-minions on target nodes.
+ * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
+ * @param target unique identification of a minion or group of salt minions
+ * @param wait timeout for the salt command if minions do not return (default 5)
+ * @param maxRetries finite number of iterations to check status of a command (default 10)
+ * @return output of salt command
+ */
+def restartSaltMinion(saltId, target, wait = 5, maxRetries = 10) {
+ def common = new com.mirantis.mk.Common()
+ common.infoMsg("Restarting salt-minion on ${target} and waiting for they are reachable.")
+ runSaltProcessStep(saltId, target, 'cmd.shell', ['salt-call service.restart salt-minion'], null, true, 60)
+ checkTargetMinionsReady(['saltId': saltId, 'target_reachable': target, timeout: wait, retries: maxRetries])
+ common.infoMsg("All ${target} minions are alive...")
+}
+
+/**
+ * Upgrade package and restart salt minion.
+ * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
+ * @param target unique identification of a minion or group of salt minions
+ * @param the name of pkg_name to upgrade
+ * @param wait timeout for the salt command if minions do not return (default 5)
+ * @param maxRetries finite number of iterations to check status of a command (default 10)
+ * @return output of salt command
+ */
+def upgradePackageAndRestartSaltMinion(saltId, target, pkg_name, wait = 5, maxRetries = 10) {
+ def common = new com.mirantis.mk.Common()
+ def latest_version = getReturnValues(runSaltProcessStep(saltId, target, 'pkg.latest_version', [pkg_name, 'show_installed=True'])).split('\n')[0]
+ def current_version = getReturnValues(runSaltProcessStep(saltId, target, 'pkg.version', [pkg_name])).split('\n')[0]
+ if (current_version && latest_version != current_version) {
+ common.infoMsg("Upgrading current ${pkg_name}: ${current_version} to ${latest_version}")
+ runSaltProcessStep(saltId, target, 'pkg.install', [pkg_name], 'only_upgrade=True')
+ restartSaltMinion(saltId, target, wait, maxRetries)
+ }
+}
+
+/**
* Run command on salt minion (salt cmd.run wrapper)
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
* @param target Get pillar target