Add checkClusterTimeSync method
Related: PROD-26609 (PROD:26609)
Change-Id: I990bf96920b2c0ddf5250a26a90509cac4964395
(cherry picked from commit 232ad90b7e8b46d1d193b23111862782213d4bc6)
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index d2f6d8a..284adec 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -533,9 +533,11 @@
* of Salt mysql.status function. The result is then parsed, validated and outputed to the user.
*
* @param env Salt Connection object or pepperEnv
+ * @param slave Boolean value to enable slave checking (if master in unreachable)
+ * @param checkTimeSync Boolean value to enable time sync check
* @return resultCode int values used to determine exit status in the calling function
*/
-def verifyGaleraStatus(env, slave=false) {
+def verifyGaleraStatus(env, slave=false, checkTimeSync=false) {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
def out = ""
@@ -573,6 +575,10 @@
common.errorMsg("No Galera slave was reachable.")
return 130
}
+ if (checkTimeSync && !salt.checkClusterTimeSync(env, "I@galera:master or I@galera:slave")) {
+ common.errorMsg("Time in cluster is desynchronized or it couldn't be detemined. You should fix this issue manually before proceeding.")
+ return 131
+ }
try {
out = salt.cmdRun(env, "I@salt:master", "salt -C '${testNode}' mysql.status")
} catch (Exception e) {
@@ -736,7 +742,7 @@
* @param env Salt Connection object or pepperEnv
* @return output of salt commands
*/
-def restoreGaleraDb(env) {
+def restoreGaleraDb(env, type) {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
try {
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 09e1199..4e76538 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -1153,3 +1153,46 @@
}
return outputObj
}
+
+/**
+* Check time settings on defined nodes, compares them
+* and evaluates the results
+*
+* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
+* @param target Targeted nodes to be checked
+* @param diff Maximum time difference (in seconds) to be accepted during time sync check
+* @return bool Return true if time difference is <= diff and returns false if time difference is > diff
+*/
+
+def checkClusterTimeSync(saltId, target) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
+
+ times = []
+ try {
+ diff = salt.getReturnValues(salt.getPillar(saltId, 'I@salt:master', 'linux:system:time_diff'))
+ if (diff != null && diff != "" && diff.isInteger()) {
+ diff = diff.toInteger()
+ } else {
+ diff = 5
+ }
+ out = salt.runSaltProcessStep(saltId, target, 'status.time', '%s')
+ outParsed = out['return'][0]
+ def outKeySet = outParsed.keySet()
+ for (key in outKeySet) {
+ def time = outParsed[key].readLines().get(0)
+ common.infoMsg(time)
+ if (time.isInteger()) {
+ times.add(time.toInteger())
+ }
+ }
+ if ((times.max() - times.min()) <= diff) {
+ return true
+ } else {
+ return false
+ }
+ } catch(Exception e) {
+ common.errorMsg("Could not check cluster time sync.")
+ return false
+ }
+}