src/com/mirantis/mk/Galera.groovy

package com.mirantis.mk

/**
 *
 * Galera functions
 *
 */


/**
 * Returns parameters from mysql.status output on given target node
 *
 * @param env           Salt Connection object or pepperEnv
 * @param target        Targeted node
 * @param parameters    Parameters to be retruned (String or list of Strings). If no parameters are provided or is set to '[]', it returns all of them.
 * @return result       List of parameters with its values
 */

def getWsrepParameters(env, target, parameters=[], print=false) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    result = [:]
    out = salt.runSaltProcessStep(env, "${target}", "mysql.status", [], null, false)
    outlist = out['return'][0]
    resultYaml = outlist.get(outlist.keySet()[0]).sort()
    if (print) {
        common.prettyPrint(resultYaml)
    }
    if (parameters instanceof String) {
        parameters = [parameters]
    }
    if (parameters == [] || parameters == ['']) {
        result = resultYaml
    } else {
        for (String param in parameters) {
            value = resultYaml[param]
            if (value instanceof String && value.isBigDecimal()) {
                value = value.toBigDecimal()
            }
            result[param] = value
        }
    }
    return result
}

/**
 * Verifies Galera database
 *
 * This function checks for Galera master, tests connection and if reachable, it obtains the result
 *      of Salt mysql.status function. The result is then parsed, validated and outputed to the user.
 *
 * @param env           Salt Connection object or pepperEnv
 * @param checkTimeSync Boolean value to enable time sync check
 * @return resultCode   int values used to determine exit status in the calling function
 */
def verifyGaleraStatus(env, checkTimeSync=false) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def mysqlStatusReport = [
        'clusterMembersOnPower': [],
        'clusterMembersNotAvailable': [],
        'clusterMembersInClusterAlive': [],
        'clusterMembersNotAlive': [],
        'error': 0
    ]

    try {
        def clusterMembers = salt.getMinions(env, "I@galera:master or I@galera:slave")
        for (minion in clusterMembers) {
            try {
                salt.minionsReachable(env, "I@salt:master", minion)
                mysqlStatusReport['clusterMembersOnPower'] << minion
            } catch (Exception e) {
                common.warningMsg("Slave '${minion}' is not reachable.")
                mysqlStatusReport['clusterMembersNotAvailable'] << minion
            }
        }
    } catch (Exception e) {
        common.errorMsg('Cannot obtain Galera minions list.')
        common.errorMsg(e.getMessage())
        mysqlStatusReport['error'] = 128
        return mysqlStatusReport
    }

    if (!mysqlStatusReport['clusterMembersOnPower']) {
        common.errorMsg("No Galera member was reachable.")
        mysqlStatusReport['error'] = 130
        return mysqlStatusReport
    }

    def checkTargets = salt.getMinions(env, "I@xtrabackup:client or I@xtrabackup:server")
    for (checkTarget in checkTargets) {
        def nodeStatus = salt.minionsReachable(env, 'I@salt:master', checkTarget, null, 10, 5)
        if (nodeStatus != null) {
            def iostatRes = salt.getIostatValues(['saltId': env, 'target': checkTarget, 'parameterName': "%util", 'output': true])
            if (iostatRes == [:]) {
                common.errorMsg("Recevived empty response from iostat call on ${checkTarget}. Maybe 'sysstat' package is not installed?")
                mysqlStatusReport['error'] = 140
                return mysqlStatusReport
            }
            for (int i = 0; i < iostatRes.size(); i++) {
                def diskKey = iostatRes.keySet()[i]
                if (!(iostatRes[diskKey].toString().isBigDecimal() && (iostatRes[diskKey].toBigDecimal() < 50 ))) {
                    common.errorMsg("Disk ${diskKey} has to high i/o utilization. Maximum value is 50 and current value is ${iostatRes[diskKey]}.")
                    mysqlStatusReport['error'] = 141
                    return mysqlStatusReport
                }
            }
        }
    }
    common.infoMsg("Disk i/o utilization was checked and everything seems to be in order.")
    if (checkTimeSync && !salt.checkClusterTimeSync(env, "I@galera:master or I@galera:slave")) {
        common.errorMsg("Time in cluster is desynchronized or it couldn't be detemined. You should fix this issue manually before proceeding.")
        mysqlStatusReport['error'] = 131
        return mysqlStatusReport
    }

    for(member in mysqlStatusReport['clusterMembersOnPower']) {
        def clusterStatus = getWsrepParameters(env, member, 'wsrep_cluster_status')
        if (clusterStatus['wsrep_cluster_status']) {
            mysqlStatusReport['clusterMembersInClusterAlive'] << member
        } else {
            mysqlStatusReport['clusterMembersNotAlive'] << member
        }
    }
    if (!mysqlStatusReport['clusterMembersInClusterAlive']) {
        common.errorMsg("Could not determine mysql status, because all nodes are not connected to cluster.")
        mysqlStatusReport['error'] = 256
        return mysqlStatusReport
    }
    def testNode = mysqlStatusReport['clusterMembersInClusterAlive'].sort().first()

    try {
        mysqlStatusReport['statusRaw'] = salt.runSaltProcessStep(env, testNode, "mysql.status", [], null, false)
    } catch (Exception e) {
        common.errorMsg('Could not determine mysql status.')
        common.errorMsg(e.getMessage())
        mysqlStatusReport['error'] = 256
        return mysqlStatusReport
    }

    def status = "unknown"
    def galeraMasterNode = salt.getReturnValues(salt.getPillar(env, testNode, "galera:master:enabled")) ? true : false

    if (mysqlStatusReport['statusRaw']) {
        try {
            status = validateAndPrintGaleraStatusReport(env, mysqlStatusReport['statusRaw'], testNode, galeraMasterNode)
        } catch (Exception e) {
            common.errorMsg('Could not parse the mysql status output. Check it manually.')
            common.errorMsg(e.getMessage())
        }
    } else {
        common.errorMsg("Mysql status response unrecognized or is empty. Response: ${mysqlStatusReport['statusRaw']}")
    }
    if (mysqlStatusReport['clusterMembersNotAvailable']) {
        common.errorMsg("Next nodes are unavailable: ${mysqlStatusReport['clusterMembersNotAvailable'].join(',')}")
    }
    if (mysqlStatusReport['clusterMembersNotAlive']) {
        common.errorMsg("Next nodes are not connected to cluster: ${mysqlStatusReport['clusterMembersNotAlive'].join(',')}")
    }

    if (status == "OK") {
        common.infoMsg("No errors found - MySQL status is ${status}.")
        return mysqlStatusReport
    } else if (status == "unknown") {
        common.warningMsg('MySQL status cannot be detemined')
        mysqlStatusReport['error'] = 1
        return mysqlStatusReport
    } else {
        common.errorMsg("Errors found.")
        mysqlStatusReport['error'] = 2
        return mysqlStatusReport
    }
}

/** Validates and prints result of verifyGaleraStatus function
@param env      Salt Connection object or pepperEnv
@param out      Output of the mysql.status Salt function
@return status  "OK", "ERROR" or "uknown" depending on result of validation
*/

def validateAndPrintGaleraStatusReport(env, out, minion, nodeRoleMaster=false) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def role = 'slave'
    if (nodeRoleMaster) {
        role = 'master'
    }
    sizeOut = salt.getReturnValues(salt.getPillar(env, minion, "galera:${role}:members"))
    expected_cluster_size = sizeOut.size()
    outlist = out['return'][0]
    resultYaml = outlist.get(outlist.keySet()[0]).sort()
    common.prettyPrint(resultYaml)
    parameters = [
        wsrep_cluster_status: [title: 'Cluster status', expectedValues: ['Primary'], description: ''],
        wsrep_cluster_size: [title: 'Current cluster size', expectedValues: [expected_cluster_size], description: ''],
        wsrep_ready: [title: 'Node status', expectedValues: ['ON', true], description: ''],
        wsrep_local_state_comment: [title: 'Node status comment', expectedValues: ['Joining', 'Waiting on SST', 'Joined', 'Synced', 'Donor'], description: ''],
        wsrep_connected: [title: 'Node connectivity', expectedValues: ['ON', true], description: ''],
        wsrep_local_recv_queue_avg: [title: 'Average size of local reveived queue', expectedThreshold: [warn: 0.5, error: 1.0], description: '(Value above 0 means that the node cannot apply write-sets as fast as it receives them, which can lead to replication throttling)'],
        wsrep_local_send_queue_avg: [title: 'Average size of local send queue', expectedThreshold: [warn: 0.5, error: 1.0], description: '(Value above 0 indicate replication throttling or network throughput issues, such as a bottleneck on the network link.)']
        ]
    for (key in parameters.keySet()) {
        value = resultYaml[key]
        if (value instanceof String && value.isBigDecimal()) {
            value = value.toBigDecimal()
        }
        parameters.get(key) << [actualValue: value]
    }
    for (key in parameters.keySet()) {
        param = parameters.get(key)
        if (key == 'wsrep_local_recv_queue_avg' || key == 'wsrep_local_send_queue_avg') {
            if (param.get('actualValue') == null || (param.get('actualValue') > param.get('expectedThreshold').get('error'))) {
                param << [match: 'error']
            } else if (param.get('actualValue') > param.get('expectedThreshold').get('warn')) {
                param << [match: 'warn']
            } else {
                param << [match: 'ok']
            }
        } else {
            for (expValue in param.get('expectedValues')) {
                if (expValue == param.get('actualValue')) {
                    param << [match: 'ok']
                    break
                } else {
                    param << [match: 'error']
                }
            }
        }
    }
    cluster_info_report = []
    cluster_warning_report = []
    cluster_error_report = []
    for (key in parameters.keySet()) {
        param = parameters.get(key)
        if (param.containsKey('expectedThreshold')) {
            expValues = "below ${param.get('expectedThreshold').get('warn')}"
        } else {
            if (param.get('expectedValues').size() > 1) {
                expValues = param.get('expectedValues').join(' or ')
            } else {
                expValues = param.get('expectedValues')[0]
            }
        }
        reportString = "${param.title}: ${param.actualValue} (Expected: ${expValues}) ${param.description}"
        if (param.get('match').equals('ok')) {
            cluster_info_report.add("[OK     ] ${reportString}")
        } else if (param.get('match').equals('warn')) {
            cluster_warning_report.add("[WARNING] ${reportString}")
        } else {
            cluster_error_report.add("[  ERROR] ${reportString})")
        }
    }
    common.infoMsg("CLUSTER STATUS REPORT: ${cluster_info_report.size()} expected values, ${cluster_warning_report.size()} warnings and ${cluster_error_report.size()} error found:")
    if (cluster_info_report.size() > 0) {
        common.infoMsg(cluster_info_report.join('\n'))
    }
    if (cluster_warning_report.size() > 0) {
        common.warningMsg(cluster_warning_report.join('\n'))
    }
    if (cluster_error_report.size() > 0) {
        common.errorMsg(cluster_error_report.join('\n'))
        return "ERROR"
    } else {
        return "OK"
    }
}

/** Returns last shutdown node of Galera cluster
@param env      Salt Connection object or pepperEnv
@param nodes    List of nodes to check only (defaults to []). If not provided, it will check all nodes.
                Use this parameter if the cluster splits to several components and you only want to check one fo them.
@return status  ip address or hostname of last shutdown node
*/

def getGaleraLastShutdownNode(env, nodes = []) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    members = []
    lastNode = [ip: '', seqno: -2]
    try {
        if (nodes) {
            nodes = salt.getIPAddressesForNodenames(env, nodes)
            for (node in nodes) {
                members = [host: "${node.get(node.keySet()[0])}"] + members
            }
        } else {
            members = salt.getReturnValues(salt.getPillar(env, "I@galera:master", "galera:master:members"))
        }
    } catch (Exception e) {
        common.errorMsg('Could not retrieve members list')
        common.errorMsg(e.getMessage())
        return 'I@galera:master'
    }
    if (members) {
        for (member in members) {
            try {
                salt.minionsReachable(env, 'I@salt:master', "S@${member.host}")
                out = salt.getReturnValues(salt.cmdRun(env, "S@${member.host}", 'cat /var/lib/mysql/grastate.dat | grep "seqno" | cut -d ":" -f2', true, null, false))
                seqno = out.tokenize('\n')[0].trim()
                if (seqno.isNumber()) {
                    seqno = seqno.toInteger()
                } else {
                    // in case if /var/lib/mysql/grastate.dat has no any seqno - set it to 0
                    // thus node will be recovered if no other failed found
                    seqno = 0
                }
            } catch (Exception e) {
                common.warningMsg("Could not determine 'seqno' value for node ${member.host} ")
                common.warningMsg(e.getMessage())
                seqno = 0
            }
            highestSeqno = lastNode.get('seqno')
            if (seqno > highestSeqno) {
                lastNode << [ip: "${member.host}", seqno: seqno]
            }
        }
    }
    if (lastNode.get('ip') != '') {
        return "S@${lastNode.ip}"
    } else {
        return "I@galera:master"
    }
}

/**
 * Wrapper around Mysql systemd service
 * @param env           Salt Connection object or pepperEnv
 * @param targetNode    Node to apply changes
 * @param checkStatus   Whether to check status of Mysql
 * @param checkState    State of service to check
*/
def manageServiceMysql(env, targetNode, action, checkStatus=true, checkState='running') {
    def salt = new com.mirantis.mk.Salt()
    salt.runSaltProcessStep(env, targetNode, "service.${action}", ['mysql'])
    if (checkStatus) {
        try {
            salt.commandStatus(env, targetNode, 'service mysql status', checkState)
        } catch (Exception er) {
            input message: "Database is not running please fix it first and only then click on PROCEED."
        }
    }
}

/**
 * Restores Galera cluster
 * @param env           Salt Connection object or pepperEnv
 * @param galeraStatus  Map, Status of Galera cluster output  from verifyGaleraStatus func
 * @param restoreDb     Run restore DB procedure
 * @return output of salt commands
 */
def restoreGaleraCluster(env, galeraStatus, restoreDb=true) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def nodesToRecover = []
    def total = false // whole cluster
    if (galeraStatus['clusterMembersNotAlive']) {
        nodesToRecover = galeraStatus['clusterMembersNotAlive']
        if (galeraStatus['clusterMembersInClusterAlive'].size() == 0) {
            total = true
        }
    } else {
        nodesToRecover = galeraStatus['clusterMembersInClusterAlive']
        total = true
    }

    def lastNodeTarget = ''
    if (total) {
        manageServiceMysql(env, 'I@galera:slave', 'stop', true, 'inactive')
        manageServiceMysql(env, 'I@galera:master', 'stop', true, 'inactive')
        lastNodeTarget = getGaleraLastShutdownNode(env) // in case if master was already down before
        salt.cmdRun(env, "( I@galera:master or I@galera:slave ) and not ${lastNodeTarget}", "rm -f /var/lib/mysql/ib_logfile*")
        salt.cmdRun(env, "( I@galera:master or I@galera:slave ) and not ${lastNodeTarget}", "rm -f /var/lib/mysql/grastate.dat")
    } else {
        lastNodeTarget = nodesToRecover.join(' or ')
        manageServiceMysql(env, lastNodeTarget, 'stop', true, 'inactive')
    }

    if (restoreDb) {
        def timestamp = common.getDatetime()
        def bakDir = "/root/mysql/mysql.bak.${timestamp}".toString()
        salt.cmdRun(env, lastNodeTarget, "mkdir -p ${bakDir}")
        salt.cmdRun(env, lastNodeTarget, "mv /var/lib/mysql/* ${bakDir} || echo 'Nothing to backup from directory /var/lib/mysql/'")
    }
    if (total) {
        // make sure that gcom parameter is empty
        salt.cmdRun(env, lastNodeTarget, "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
    } else if (!restoreDb) {
        // node rejoin
        salt.cmdRun(env, lastNodeTarget, "rm -f /var/lib/mysql/ib_logfile*")
        salt.cmdRun(env, lastNodeTarget, "rm -f /var/lib/mysql/grastate.dat")
    }

    if (restoreDb) {
        restoreGaleraDb(env, lastNodeTarget)
    }

    manageServiceMysql(env, lastNodeTarget, 'start')

    if (total) {
        manageServiceMysql(env, "( I@galera:master or I@galera:slave ) and not ${lastNodeTarget}", 'start')
        salt.runSaltProcessStep(env, lastNodeTarget, 'state.sls_id', ['galera_config', 'galera'])
    }
}

/**
 * Restores Galera database
 * @param env           Salt Connection object or pepperEnv
 * @param targetNode    Node to be targeted
 */
def restoreGaleraDb(env, targetNode) {
    def salt = new com.mirantis.mk.Salt()
    def backup_dir = salt.getReturnValues(salt.getPillar(env, targetNode, 'xtrabackup:client:backup_dir'))
    if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
    salt.runSaltProcessStep(env, targetNode, 'file.remove', ["${backup_dir}/dbrestored"])
    salt.enforceState(['saltId': env, 'target': targetNode, 'state': 'xtrabackup.client'])
    salt.enforceState(['saltId': env, 'target': targetNode, 'state': 'xtrabackup.client.restore'])
}

def restoreGaleraDb(env) {
    def common = new com.mirantis.mk.Common()
    common.warningMsg("This method was renamed to 'restoreGaleraCluster'. Please change your pipeline to use this call instead! If you think that you really wanted to call 'restoreGaleraDb' you may be missing 'targetNode' parameter in you call.")
    return restoreGaleraCluster(env)
}

/**
 * Start first node in mysql cluster. Cluster members stay removed in mysql config, additional service restart will be needed once all nodes are up.
 * https://docs.mirantis.com/mcp/q4-18/mcp-operations-guide/tshooting/
 * tshoot-mcp-openstack/tshoot-galera/restore-galera-cluster/
 * restore-galera-manually.html#restore-galera-manually
 *
 * @param env             Salt Connection object or pepperEnv
 * @param target  last stopped Galera node
 * @return                output of salt commands
 */
def startFirstNode(env, target) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()

    // make sure that gcom parameter is empty
    salt.cmdRun(env, target, "sed -i '/wsrep_cluster_address/ s/^#*/#/' /etc/mysql/my.cnf")
    salt.cmdRun(env, target, "sed -i '/wsrep_cluster_address/a wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")

    // start mysql service on the last node
    salt.runSaltProcessStep(env, target, 'service.start', ['mysql'])

    // wait until mysql service on the last node is up

    common.retry(30, 10) {
        value = getWsrepParameters(env, target, 'wsrep_evs_state')
        if (value['wsrep_evs_state'] == 'OPERATIONAL') {
            common.infoMsg('WSREP state: OPERATIONAL')
        } else {
            throw new Exception("Mysql service is not running please fix it.")
        }
    }
}