Update scripts for restoring failed node of Mysql in Galera cluster
Change-Id: Ib1f90bde1f75fc36cbd7902b306e06d3cf5002d1
Related-Prod: PROD-32439
diff --git a/src/com/mirantis/mk/Galera.groovy b/src/com/mirantis/mk/Galera.groovy
index 7319163..3a10a1c 100644
--- a/src/com/mirantis/mk/Galera.groovy
+++ b/src/com/mirantis/mk/Galera.groovy
@@ -277,15 +277,18 @@
if (seqno.isNumber()) {
seqno = seqno.toInteger()
} else {
- seqno = -2
- }
- highestSeqno = lastNode.get('seqno')
- if (seqno > highestSeqno) {
- lastNode << [ip: "${member.host}", seqno: seqno]
+ // in case if /var/lib/mysql/grastate.dat has no any seqno - set it to 0
+ // thus node will be recovered if no other failed found
+ seqno = 0
}
} catch (Exception e) {
common.warningMsg("Could not determine 'seqno' value for node ${member.host} ")
common.warningMsg(e.getMessage())
+ seqno = 0
+ }
+ highestSeqno = lastNode.get('seqno')
+ if (seqno > highestSeqno) {
+ lastNode << [ip: "${member.host}", seqno: seqno]
}
}
}
@@ -297,6 +300,25 @@
}
/**
+ * Wrapper around Mysql systemd service
+ * @param env Salt Connection object or pepperEnv
+ * @param targetNode Node to apply changes
+ * @param checkStatus Whether to check status of Mysql
+ * @param checkState State of service to check
+*/
+def manageServiceMysql(env, targetNode, action, checkStatus=true, checkState='running') {
+ def salt = new com.mirantis.mk.Salt()
+ salt.runSaltProcessStep(env, lastNodeTarget, "service.${action}", ['mysql'])
+ if (checkStatus) {
+ try {
+ salt.commandStatus(env, lastNodeTarget, 'service mysql status', checkState)
+ } catch (Exception er) {
+ input message: "Database is not running please fix it first and only then click on PROCEED."
+ }
+ }
+}
+
+/**
* Restores Galera cluster
* @param env Salt Connection object or pepperEnv
* @param runRestoreDb Boolean to determine if the restoration of DB should be run as well
@@ -305,11 +327,8 @@
def restoreGaleraCluster(env, runRestoreDb=true) {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
- salt.runSaltProcessStep(env, 'I@galera:slave', 'service.stop', ['mysql'])
- salt.runSaltProcessStep(env, 'I@galera:master', 'service.stop', ['mysql'])
lastNodeTarget = getGaleraLastShutdownNode(env)
- salt.cmdRun(env, "( I@galera:master or I@galera:slave ) and not ${lastNodeTarget}", "rm -f /var/lib/mysql/ib_logfile*")
- salt.cmdRun(env, "( I@galera:master or I@galera:slave ) and not ${lastNodeTarget}", "rm -f /var/lib/mysql/grastate.dat")
+ manageServiceMysql(env, lastNodeTarget, 'stop', false)
if (runRestoreDb) {
salt.cmdRun(env, lastNodeTarget, "mkdir -p /root/mysql/mysql.bak")
salt.cmdRun(env, lastNodeTarget, "rm -rf /root/mysql/mysql.bak/*")
@@ -325,30 +344,11 @@
restoreGaleraDb(env, lastNodeTarget)
}
- // start mysql service on the last node
- salt.runSaltProcessStep(env, lastNodeTarget, 'service.start', ['mysql'])
+ manageServiceMysql(env, lastNodeTarget, 'start')
- // wait until mysql service on the last node is up
- try {
- salt.commandStatus(env, lastNodeTarget, 'service mysql status', 'running')
- } catch (Exception er) {
- input message: "Database is not running please fix it first and only then click on PROCEED."
- }
-
- // start mysql services on the rest of the nodes
- salt.runSaltProcessStep(env, "I@galera:master and not ${lastNodeTarget}", 'service.start', ['mysql'])
- salt.runSaltProcessStep(env, "I@galera:slave and not ${lastNodeTarget}", 'service.start', ['mysql'])
-
- // wait until mysql service on the rest of the nodes is up
- try {
- salt.commandStatus(env, "( I@galera:master or I@galera:slave ) and not ${lastNodeTarget}", 'service mysql status', 'running')
- } catch (Exception er) {
- input message: "Database is not running please fix it first and only then click on PROCEED."
- }
-
- // apply any changes in configuration and return value to gcom parameter
+ // apply any changes in configuration and return value to gcom parameter and then restart mysql to catch
salt.enforceState(['saltId': env, 'target': lastNodeTarget, 'state': 'galera'])
-
+ manageServiceMysql(env, lastNodeTarget, 'restart')
}
/**