#!/bin/bash
# Copyright 2013 Severalnines AB
#
# MODIFY THE BELOW TO SUIT YOU ENV:
LOCKFILE="/tmp/s9s_galera.lock"
jobid=0
function init
{    
    FILES=`ls /etc/cmon.cnf 2>&1`
    FILES2=`ls /etc/cmon.d/*.cnf 2>&1`    
    FILES="$FILES $FILES2"
    configfile=""
    for f in $FILES 
    do
	X=`grep -l cluster_id=${CLUSTER_ID} $f 2>&1 `	
	if [ $? -eq 0 ]; then 
	    source $f
            configfile=$f
	fi
    done

    if [ -z "$configfile" ]; then
	echo "No matching configuration file found having cluster_id=${CLUSTER_ID}"
	exit 1
    fi
    
    source $configfile   

    CMON_DB_HOST=$mysql_hostname
    CMON_DB_PORT=$mysql_port
    CMON_USER=cmon
    CMON_DB_DB=cmon
    CMON_PASSWORD=$mysql_password
    MYSQL_BIN=$mysql_basedir/bin/mysql
    MYSQL_BIN2=$mysql_bindir/mysql 
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    if ! test -f $MYSQL_BIN; then
	echo "Could not find mysql client binary"
	log_job_message "Could not find mysql client binary" 1
	log_job 'FAILED' 'backup failed' 1
	exit 1
    fi

    CONNECT_TIMEOUT=10
    CLUSTER_TYPE=$type
    MYSQL_OPTS="--connect-timeout=$CONNECT_TIMEOUT"
    OSUSER=$USER
        
    if [ "$OSUSER" != "root" ]; then
	echo "must be executed as 'root' or with 'sudo'"
	exit 1
    fi
}
 
function log_job_message
{    
    MSG=$1
    EXIT_CODE=$2
    if [ -z "$EXIT_CODE" ]; then
	EXIT_CODE=1
    fi
    QUERY="INSERT INTO cmon_job_message(cid, jobid,message,exit_code,report_ts) VALUES($CLUSTER_ID,$jobid,\"$MSG\",$EXIT_CODE,now())"
    if [ $jobid -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    else
	if [ "$EXIT_CODE" -eq 0 ]; then
	    echo "Ok: $MSG"
	else
	    echo "Error: $MSG"
	fi
    fi
}

function log_job
{    
    STATUS=$1
    STATUS_TXT=$2
    EXIT_CODE=$3
    QUERY="UPDATE cmon_job SET status='$STATUS', status_txt='$STATUS_TXT', exit_code=$EXIT_CODE, report_ts=NOW()  WHERE cid=$CLUSTER_ID AND jobid=$jobid"
    if [ $jobid -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    fi
}




function remote_cmd_getreply()
{
   desthost=$1
   xcommand=$2
   x=`ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand"`
   if [ $? -ne 0 ]; then
       echo "Command failed: ssh -q $SSH_OPTS $SSH_USER@$desthost \"$SUDO $xcommand\""
       log_job_message "FAILED: ${desthost}@${xcommand}" 1
       exit 1
   fi
   x=`echo $x | grep -v "password"`
   echo $x
}

function remote_copy()
{
   srcfile=$1
   desthost=$2
   destfile=$3
   printf "%-4s: Copying '%s' " "$desthost" "$srcfile"
   scp $SSH_OPTS2 $srcfile $SSH_USER@$desthost:$destfile >> $HOME/s9s_deploy.log  2>/dev/null
   if [ $? -eq 0 ]; then
      printf "\033[32m[ok]\033[0m\n"
      log_job_message "copied file $srcfile --> $desthost:$destfile" 0
      return 0
   else
      log_job_message "Failed copying file $srcfile --> $desthost:$destfile" 1
      log_job_message "Failed: scp $SSH_OPTS2 $srcfile $SSH_USER@$desthost:$destfile " 1
      log_job 'FAILED' 'job failed' 1
      printf "\033[31m[failed]\033[0m\n"
      exit 1
   fi
}

function remote_cmdx()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s'" "$desthost" "$xcommand"
   ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand "
   if [ $? -eq 0 ]; then
      printf "\033[32m[ok]\033[0m\n"
      log_job_message "$desthost: Executed $xcommand" 0
      return 0
   else
      log_job_message "command failed: $xcommand" 1
      log_job 'FAILED' 'backup failed' 1
      printf "\033[31m[failed]\033[0m\n"
      exit 1
   fi
}

function remote_cmd_nofail()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s'" "$desthost" "$xcommand"
   ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand "  >> $HOME/s9s_deploy.log 2>&1
#   echo "   ssh -q $SSH_OPTS $SSH_USER@$desthost "
   ret=$?
   log_job_message "$desthost: Executed $xcommand" 0
   printf "\033[32m[ok]\033[0m\n"
   return $ret
}

function remote_cmd()
{
   desthost=$1
   xcommand=$2
   MAX_RETRIES=1
   printf "%-4s: Executing '%s' " "$desthost" "$xcommand"
   retry=0
   while [ $retry -lt $MAX_RETRIES ]; 
   do
      x=`ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand " 2>&1  >> $HOME/s9s_deploy.log`
      if [ $? -eq 0 ]; then
        printf "\033[32m[ok]\033[0m\n"
	log_job_message "$desthost: Executed $xcommand" 0
        return 0
      fi
      retry=`expr $retry + 1`
      printf "\033[31m[failed: retrying ${retry}/${MAX_RETRIES}]\033[0m\n"
      ssh -q $SSH_OPTS $SSH_USER@$desthost " sync " 2>&1  >> $HOME/s9s_deploy.log
      sleep 1
   done
   log_job_message "FAILED: ${desthost}@${xcommand}" 1
   printf "\033[31m[failed]\033[0m\n"
   echo $x
   echo 'The following command failed:'
   echo "ssh -q $SSH_OPTS $SSH_USER@$desthost \" $xcommand \""
   echo 'Try running the command on the line above again, contact http://support.severalnines.com/ticket/new, attach the output from deploy.sh and the error from running the command to the Support issue.'
   exit 1
}


function remote_cmd3()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s' " "$desthost" "$xcommand"
   ssh $SSH_OPTS $SSH_OPTS_EXTRA $SSH_USER@$desthost "$SUDO $xcommand " &
   if [ $? -eq 0 ]; then
      printf "\033[32m[ok]\033[0m\n"
      log_job_message "$desthost: Executed $xcommand" 0
      return 0
   else
      printf "\033[31m[failed]\033[0m\n"
      log_job_message "FAILED: ${desthost}@${xcommand}" 1
      exit 1
   fi
}
function install_agent
{
    args=`getopt p:i:P:a:h: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -h)
		HOSTNAME="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -a)
		AGENT_BINARY="$2"; shift;
		shift;;
	    -P)
		PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_galera "
	exit 1
    fi
    init 

    if [ -z "$CMON_PASSWORD" ]; then
	echo "s9s_galera "
	exit 1
    fi
    
    if [ ! -f $AGENT_BINARY ]; then
	echo "s9s_galera "
	exit 1
    fi

    if [ -z "$HOSTNAME" ]; then
	echo "s9s_galera "
	exit 1
    fi
    echo "Not implemented"
    exit 1
    
}

function load_opts 
{
    local CLUSTER_ID=$1
    echo "load opts $CLUSTER_ID"
    OS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS' AND cid=$CLUSTER_ID" 2>/dev/null`
    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    GALERA_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='GALERA_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    GALERA_VERSION=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='GALERA_VERSION' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_BASEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_BASEDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_SCRIPTDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SCRIPTDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_IDENTITY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_IDENTITY' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_USER=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_USER' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_OPTSX=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_OPTS' AND cid=$CLUSTER_ID"`
    SUDO=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SUDO' AND cid=$CLUSTER_ID" 2>/dev/null`    
    OS_USER_HOME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS_USER_HOME' AND cid=$CLUSTER_ID" 2>/dev/null`
    TMPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='STAGING_DIR' AND cid=$CLUSTER_ID"`
    HTTP_PROXY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='HTTP_PROXY' AND cid=$CLUSTER_ID"`
    S9S_TMPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='STAGING_DIR' AND cid=$CLUSTER_ID"`
    VENDOR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='VENDOR' AND cid=$CLUSTER_ID" 2>/dev/null`
    CLUSTER_RECOVERY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='ENABLE_CLUSTER_AUTORECOVERY' AND cid=$CLUSTER_ID" 2>/dev/null`
    DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon.cluster_config where variable='datadir' and cid=$CLUSTER_ID order by id asc limit 1" 2>/dev/null`
    MYCNF=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select data  from cmon.cluster_configuration where cid=$CLUSTER_ID limit 1" 2>/dev/null`


    echo $MYCNF |  sed 's#\\n#\n\r#g' > /tmp/my.cnf

    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi
    
    SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10"
    if [ "$SSH_IDENTITY" = "" ]; then
	SSH_IDENTITY="-oIdentityFile=${OS_USER_HOME}/.ssh/id_rsa"
    else
	SSH_IDENTITY="-oIdentityFile=$SSH_IDENTITY"
    fi

    if [ "$GALERA_VERSION" = "" ]; then
	GALERA_VERSION='2.x'
    fi

    if [ -z "$GALERA_VERSION" ]; then
	GALERA_VERSION='2.x'
    fi
        

    SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    SSH_OPTS2="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    if [ "$SSH_USER" != "root" ]; then
	SSH_OPTS_EXTRA="-ftt"
	if [ -z "$SSH_OPTSX" ] || [ "SSH_OPTSX" = "NULL" ]; then
	    SSH_OPTSX="-tt"
	fi
	SSH_OPTS=" -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
	if [ "$SUDO" = "" ] || [ "$SUDO" = "NULL" ];  then
           SUDO="sudo"
        fi
    fi
    if [ "$SSH_PORT" = "" ] || [ "$SSH_PORT" = "NULL" ];  then
        SSH_PORT="22"
    fi

    if [ "$S9S_TMPDIR" = "" ] || [ "$TMPDIR" = "NULL" ];  then
	S9S_TMPDIR="/tmp/"
	TMPDIR="/tmp/"
    fi
    SSH_OPTS="$SSH_OPTSX -p$SSH_PORT $SSH_OPTS"    
    SSH_OPTS2="-P$SSH_PORT $SSH_OPTS2"    
}

function install_mysql
{
    args=`getopt p:i:P:a:h:g:d: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -h)
		TARGET_HOSTNAME="$2"; shift;
		shift;;
            -d)
		DONOR="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -a)
		MYSQL_BINARY="$2"; shift;
		shift;;
            -g)
		GALERA_BINARY="$2"; shift;
		shift;;
	    -P)
		PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_galera "
	exit 1
    fi
    init

    if [ -z "$CMON_PASSWORD" ]; then
	echo "s9s_galera "
	exit 1
    fi

    if [ -z "$DONOR" ]; then
	echo "donor -d not set "
	exit 1
    fi
    
    if [ ! -f $MYSQL_BINARY ]; then
	echo "MYSQL BINARY NOT FOUND"
	exit 1
    fi

    if [ ! -f $GALERA_BINARY ]; then
	echo "GALERA LIBRARY NOT FOUND"
	exit 1
    fi

    if [ -z "$HOSTNAME" ]; then
	echo "s9s_galera "
	exit 1
    fi
    
    
    echo "NOT IMPLEMENTED"
    exit
    load_opts
    exit
}


function stop_cluster
{
    KILLSIG=15
    args=`getopt p:u:P:h:t:N:j:i:f $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
            -f)
		KILLSIG="9"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    
   
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --stop-cluster -i <clusterid>  missing"
      exit 1
    fi
    init



    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --stop-cluster -p  missing"
      exit 1
    fi

    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "Invalid cluster type - only galera is supported"
	    exit 1;	           	     
    esac
    load_opts $CLUSTER_ID 
    
    QUERY="select group_concat(concat(hostname, ',', nodeid) SEPARATOR ' ') from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID}"
    SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`
    if [ -z "$OS" ]; then
	if [ `hash /usr/sbin/update-rc.d 2>/dev/null`  ]; then
	    OS="redhat"
	else
	    OS="debian"
	fi	
    fi
    if [ "$SERVER_LIST" != "NULL" ]; then
	for S in $SERVER_LIST
	do
	    HNAME=`echo $S | awk -F ',' '{print $1};'`
            NODEID=`echo $S | awk -F ',' '{print $2};'`
	    stop_node_int $HNAME $NODEID
	done
    fi
    set_clusterstate "STOPPED"
}



function install_notifycc
{
    KILLSIG=15    
    args=`getopt p:u:P:h:t:N:j:i:f $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
            -h)
		XHOSTNAME="$2"; shift;
		shift;;
            -f)
		KILLSIG="9"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    
   
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --install-notifycc -i <clusterid>  missing"
      exit 1
    fi
    init

   
    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --install-notifycc -p  missing"
      exit 1
    fi

    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "Invalid cluster type - only galera is supported"
	    exit 1;	           	     
    esac
    load_opts $CLUSTER_ID 
    if [ -z "$XHOSTNAME" ]; then
	QUERY="select group_concat(hostname SEPARATOR ' ') from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID}"
	SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`
	if [ -z "$OS" ]; then
	    if [ `hash /usr/sbin/update-rc.d 2>/dev/null`  ]; then
		OS="redhat"
	    else
		OS="debian"
	    fi	
	fi
    else
	SERVER_LIST="$XHOSTNAME"
    fi


cat << 'EOF' > $S9S_TMPDIR/wsrep_notify_cc
#!/bin/bash

CMON_DB_HOST=CMON_MYSQL_HOSTNAME
CMON_DB_PORT=CMOM_MYSQL_PORT
CMON_USER=cmon
CMON_DB_DB=cmon
CMON_PASSWORD=CMON_MYSQL_PASSWORD
MYSQL_BIN=MYSQL_BINDIR
CONNECT_TIMEOUT=10
MYSQL_HOSTNAME=MANAGED_MYSQL_HOSTNAME
CLUSTER_ID=CMON_CLUSTERID


while [ $# -gt 0 ] 
do 
    case $1 in 
    --status) 
        STATUS=$2 
        shift 
        ;; 
    --uuid) 
        CLUSTER_UUID=$2 
        shift 
        ;; 
    --primary) 
        [ "$2" = "yes" ] && PRIMARY="1" || PRIMARY="0" 
        COM=configuration_change 
        shift 
        ;; 
    --index) 
        INDEX=$2 
        shift 
        ;; 
    --members) 
        MEMBERS=$2 
        shift 
        ;; 
    esac 
    shift 
done 



QUERY="select id from cmon.hosts where (hostname='$MYSQL_HOSTNAME' OR ip='$MYSQL_HOSTNAME') AND cid=${CLUSTER_ID}"
HID=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`

case $STATUS in
	Synced)
	   STATUS_VAL=4
           ;;
	Initialized)
	   STATUS_VAL=0
           ;;
	Donor)
	   STATUS_VAL=2
           ;;
	Joined2)
	   STATUS_VAL=3
           ;;
	Joiner)
	   STATUS_VAL=5
           ;;
	Joined)
	   STATUS_VAL=6
           ;;
        Undefined)
           STATUS='Initialized'
	   STATUS_VAL=0
           ;;
        *)
           STATUS_VAL=0
           ;;
esac

if [ -n "$MEMBERS" ]; then
  CLU_SZ=$(echo $MEMBERS | sed 's/, /_ /g' | tr ',' '\n' | wc -l)
else
  CLU_SZ="0"
fi

if [ -n "$MYSQL_HOSTNAME" ]; then
    QUERY="BEGIN;  REPLACE INTO galera_status SET value=$STATUS_VAL, value_txt='$STATUS', report_ts=NOW(), cid=$CLUSTER_ID , hostid=$HID , var='WSREP_LOCAL_STATE'"
    
    if [ -n "$CLU_SZ" ]; then
       QUERY="$QUERY; REPLACE INTO galera_status SET value=$CLU_SZ, value_txt='$CLU_SZ', report_ts=NOW(), cid=$CLUSTER_ID , hostid=$HID , var='WSREP_CLUSTER_SIZE'"
    fi
       QUERY="$QUERY; REPLACE INTO galera_status SET value=0, value_txt='$STATUS', report_ts=NOW() , cid=$CLUSTER_ID , hostid=$HID , var='WSREP_LOCAL_STATE_COMMENT';COMMIT"
    $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/dev/null
fi
EOF

  CMON_PASSWORD_ESC=$(printf '%q' $CMON_PASSWORD)
  if [ "$SERVER_LIST" != "NULL" ]; then
      for S in $SERVER_LIST
      do
	  HNAME=$S
	  remote_cmd_nofail $HNAME "mkdir -p $S9S_TMPDIR"
	  remote_cmd $HNAME "chown -R $SSH_USER:$SSH_USER $S9S_TMPDIR"
	  remote_copy $S9S_TMPDIR/wsrep_notify_cc ${HNAME} ${S9S_TMPDIR}/wsrep_notify_cc
	  remote_cmd $HNAME "sed -i.bak 's#.*CMON_DB_HOST=.*#CMON_DB_HOST=$CMON_DB_HOST#g' ${S9S_TMPDIR}/wsrep_notify_cc"
	  remote_cmd $HNAME "sed -i.bak 's#.*CMON_DB_PORT=.*#CMON_DB_PORT=$CMON_DB_PORT#g' ${S9S_TMPDIR}/wsrep_notify_cc"
	  remote_cmd $HNAME "sed -i.bak \"s#.*CMON_PASSWORD=.*#CMON_PASSWORD=\'$CMON_PASSWORD_ESC\'#g\" ${S9S_TMPDIR}/wsrep_notify_cc"
	  remote_cmd $HNAME "sed -i.bak 's#.*CLUSTER_ID=.*#CLUSTER_ID=$CLUSTER_ID#g' ${S9S_TMPDIR}/wsrep_notify_cc"
	  remote_cmd $HNAME "sed -i.bak 's#.*MYSQL_BIN=.*#MYSQL_BIN=$MYSQL_BIN#g' ${S9S_TMPDIR}/wsrep_notify_cc"
	  remote_cmd $HNAME "sed -i.bak 's#.*MYSQL_HOSTNAME=.*#MYSQL_HOSTNAME=$HNAME#g' ${S9S_TMPDIR}/wsrep_notify_cc"
	  remote_cmd $HNAME "mv -f ${S9S_TMPDIR}/wsrep_notify_cc /usr/bin"
	  remote_cmd $HNAME "chown mysql:mysql /usr/bin//wsrep_notify_cc"
	  remote_cmd $HNAME "chmod u+x /usr/bin/wsrep_notify_cc"
	  remote_cmd $HNAME "chmod 755 /usr/bin/wsrep_notify_cc"
	  remote_cmd $HNAME "sed -i.bak 's#.*wsrep_notify_cmd=.*#wsrep_notify_cmd=/usr/bin/wsrep_notify_cc#g' $CONFIGDIR/my.cnf"
      done
  fi

}



function stop_node
{
    KILLSIG=15
    args=`getopt p:u:P:h:t:N:j:i:f $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -h)
		XHOSTNAME="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
            -f)
		KILLSIG="9"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    
   
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --stop-node -i <clusterid>  missing"
      exit 1
    fi
    init
    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --create -p  missing"
      exit 1
    fi

    if [ -z "$XHOSTNAME" ]; then
      echo "s9s_galera --stop-node -h <hostname> missing"
      exit 1
    fi

    
    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "Invalid cluster type - only galera is supported"
	    exit 1;	           	     
    esac
    load_opts $CLUSTER_ID 
    
    QUERY="select group_concat(concat(hostname, ',', nodeid) SEPARATOR ' ') from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID} AND hostname='$XHOSTNAME'"
    SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`
    if [ -z "$OS" ]; then
	if [ `hash /usr/sbin/update-rc.d 2>/dev/null`  ]; then
	    OS="redhat"
	else
	    OS="debian"
	fi	
    fi
    if [ "$SERVER_LIST" != "NULL" ]; then
	for S in $SERVER_LIST
	do
	    HNAME=`echo $S | awk -F ',' '{print $1};'`
            NODEID=`echo $S | awk -F ',' '{print $2};'`
	    stop_node_int $HNAME $NODEID
	done
    else
	echo "Node $XHOSTNAME is not part of the Cluster"
	exit 1
    fi
    exit 0
}

function stop_node_int
{    
    local xHNAME=$1
    local xNODEID=$2
    WAIT_FOR_STOP=1200
    WAIT_TIME=0
    remote_cmd_nofail $xHNAME "killall -${KILLSIG} mysqld mysqld_safe"
    sleep 2
    STATE=$(check_node_up $xHNAME)
    if [ "$STATE" = "UP" ]; then
	echo "Waiting for mysqld to shutdown"
	while [ $WAIT_TIME -lt $WAIT_FOR_STOP ]; 
	do
	    WAIT_TIME=`expr $WAIT_TIME + 1`
	    echo -n "."
	    sleep 1
	    STATE=$(check_node_up $xHNAME)
	    if [ "$STATE" = "DOWN" ]; then
		break
	    fi
	    remote_cmd_nofail $xHNAME "killall -${KILLSIG} mysqld mysqld_safe"
	    remote_cmd_nofail $xHNAME "killall -9 nc socat xtrabackup innobackupex"
	done
    fi

    echo "Waiting for node to fully stop and setting node state"
    if [ "$STATE" = "DOWN" ]; then
	i=0
	while [ $i -lt 10 ]; 
	do
            set_nodestate $xNODEID 17
	    sleep 1
	    i=`expr $i + 1`
	done
    else
	echo "Failed to stop node $xHNAME"
	exit 1
    fi
    echo "DOWN"
    
}


function start_node_donor
{    
    local xHNAME=$1
    local xNODEID=$2
    STATE=$(check_node_up $xHNAME)
    if [ "$STATE" = "UP" ]; then
	echo "Node $xHNAME is already started"
	exit 1
    fi
    ###remote_cmd3 $xHNAME "nohup nice /etc/init.d/mysql start --wsrep-cluster-address=gcomm:// > /dev/null 2>&1"   
##    remote_cmd $XHNAME "sed -i.bak 's/^wsrep_cluster_address=.*/\#wsrep_cluster_address=gcomm:\/\//g' $CONFIGDIR/my.cnf" 
 ##   remote_cmd $XHNAME "sed -i.bak 's/^\#wsrep_cluster_address=.*/wsrep_cluster_address=gcomm:\/\//g' $CONFIGDIR/my.cnf"
    remote_cmd3 $xHNAME "nohup $MYSQL_BASEDIR/bin/mysqld_safe --wsrep-cluster-address=gcomm://  >/dev/null 2>&1"    
    sleep 2
    STATE=$(check_node_up $xHNAME)
    if [ "$STATE" = "UP" ]; then	
        set_nodestate $xNODEID 15
	sleep 1
        set_nodestate $xNODEID 15
	sleep 1
        set_nodestate $xNODEID 15
    else
	echo "Failed to start node $xHNAME"
	exit 1
    fi    
}


function install_garbd
{
    args=`getopt p:i:P:t:d:j:h:f:V: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -h)
		XHOSTNAME="$2"; shift;
		shift;;
            -f)
		XFILENAME="$2"; shift;
		shift;;
            -V)
		XVENDOR="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
	    -P)
		XGALERA_PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --install-garbd -i <cluster id> missing"
      exit 1
    fi
    init     

    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --install-garbd -p  missing"
      exit 1
    fi

    if [ -z "$XHOSTNAME" ]; then
      echo "s9s_galera --install-garbd -h <hostname to install garbd on is missing>"
      exit 1
    fi

    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "./s9s_galera -- only galera is supported"
	    exit 1;	           	     
    esac
    load_opts $CLUSTER_ID
    if [ -z "$VENDOR" ]; then
	VENDOR=$XVENDOR
    fi

    if [ -z "$VENDOR" ]; then
	log_job_message  "No vendor was specified" 1
	echo "./s9s-galera --install-garbd -V [percona|mariadb|codership]"
	log_job 'FAILED' 'install garbd failed' 1
	exit 1
    fi
    if [ -n "$XGALERA_PORT" ]; then
	GALERA_PORT="$XGALERA_PORT"
    fi    
    if [ -z "$GALERA_PORT" ]; then
	GALERA_PORT=4567
    fi


    sleep 2
    log_job 'RUNNING' 'installing garbd' 0

    QUERY="SELECT group_concat(concat(hostname, ':', $GALERA_PORT)) FROM $CMON_DB_DB.mysql_server WHERE cid=${CLUSTER_ID}"
    DONOR_HOSTNAME=`$MYSQL_BIN -B -N $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$CMON_DB_HOST --port=$CMON_DB_PORT --database=$CMON_DB_DB  -e "$QUERY"`
    if [ -z "$DONOR_HOSTNAME" ]; then
	echo $QUERY
	log_job_message "Could not get a cluster address" 1
	log_job 'FAILED' 'install garbd failed' 1
	exit 1
    fi

    ANY_HOSTNAME=`$MYSQL_BIN -B -N $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$CMON_DB_HOST --port=$CMON_DB_PORT --database=$CMON_DB_DB  -e "SELECT hostname FROM $CMON_DB_DB.mysql_server WHERE cid=${CLUSTER_ID} AND status=0 LIMIT 1"`
    if [ -z "$ANY_HOSTNAME" ]; then
	log_job_message "Could not get a cluster address" 1
	log_job 'FAILED' 'install garbd failed' 1
	exit 1
    fi
    
    QUERY="SHOW GLOBAL VARIABLES LIKE 'wsrep_cluster_name'"
    CLUSTER_NAME=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$ANY_HOSTNAME --port=$MONITORED_MYSQL_PORT -A -Bse "$QUERY"`
    if [ $? -ne 0 ]; then
        log_job_message "Failed to get wsrep_cluster_name" 1
	log_job 'FAILED' 'install garbd failed' 1
        exit 1
    fi
    CLUSTER_NAME=`echo $CLUSTER_NAME | awk   '{print $2;}'`

    echo "Hostname: $XHOSTNAME"
    echo "Donor: $DONOR_HOSTNAME"  
    echo "Galera port: $GALERA_PORT"
    echo "Cluster name: $CLUSTER_NAME"
    mkdir -p $S9S_TMPDIR
    chown $SSH_USER:$SSH_USER $S9S_TMPDIR
    remote_cmd_nofail $XHOSTNAME "mkdir -p $S9S_TMPDIR"
    remote_cmd $XHOSTNAME "chown $SSH_USER:$SSH_USER $S9S_TMPDIR"
    case $VENDOR in
	percona)
	    if [ "$OS" = "redhat" ]; then
		remote_cmd $XHOSTNAME "rpm -Uhv --force http://www.percona.com/downloads/percona-release/percona-release-0.0-1.x86_64.rpm"
		if [ "$GALERA_VERSION" = "3.x" ]; then
		    remote_cmd $XHOSTNAME "yum -y install Percona-XtraDB-Cluster-galera-3"
		fi
		if [ "$GALERA_VERSION" = "2.x" ]; then
		    remote_cmd $XHOSTNAME "yum -y install Percona-XtraDB-Cluster-galera-2"
		fi
	    else
		remote_cmd $XHOSTNAME "apt-get update"
		remote_cmd_nofail $XHOSTNAME "apt-get -y install  libssl0.9.8  wget rsync psmisc"
		remote_cmd_nofail $XHOSTNAME "sed -i.bak '/repo.percona.com/d'  /etc/apt/sources.list"
		$HTTP_PROXY wget --no-check-certificate --tries=5  -O ${S9S_TMPDIR}/RPM-GPG-KEY-percona http://www.percona.com/redir/downloads/RPM-GPG-KEY-percona
		if [ $? -ne 0 ]; then
		    log_job_message "Failed: $HTTP_PROXY wget --no-check-certificate --tries=5  -O ${S9S_TMPDIR}/RPM-GPG-KEY-percona http://www.percona.com/redir/downloads/RPM-GPG-KEY-percona" 1
		    log_job 'FAILED' 'install garbd failed' 1
		    exit 1
		fi
		remote_cmd $XHOSTNAME "chown -R $SSH_USER:$SSH_USER $S9S_TMPDIR"
		remote_copy $S9S_TMPDIR/RPM-GPG-KEY-percona ${XHOSTNAME} ${S9S_TMPDIR}/RPM-GPG-KEY-percona
		remote_cmd $XHOSTNAME "gpg --import ${S9S_TMPDIR}/RPM-GPG-KEY-percona"
		remote_cmd $XHOSTNAME "/bin/sh -c 'gpg -a --export CD2EFD2A | apt-key add -'"
		remote_cmd $XHOSTNAME "/bin/sh -c 'echo \"deb http://repo.percona.com/apt `lsb_release -c -s` main\" >> /etc/apt/sources.list'"
		remote_cmd $XHOSTNAME "/bin/sh -c 'echo \"deb-src http://repo.percona.com/apt `lsb_release -c -s` main\" >> /etc/apt/sources.list'"
		remote_cmd $XHOSTNAME "apt-get -q -y update"

		if [ "$GALERA_VERSION" = "3.x" ]; then
		    remote_cmd $XHOSTNAME "$HTTP_PROXY LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' -y -q install percona-xtradb-cluster-garbd-3.x"
		fi
		if [ "$GALERA_VERSION" = "2.x" ]; then
		    remote_cmd $XHOSTNAME "$HTTP_PROXY LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' -y -q install  percona-xtradb-cluster-galera-2.x"
		fi

		remote_cmd $XHOSTNAME "sync"
	    fi
	    ;;
	codership)	    

	    if [ -z "$XFILENAME" ]; then
		
		if [ "$OS" = "redhat" ]; then
		    QUERY="select concat(path, filename) from cmon_uploads a , cmon_sw_package b where a.packageid=b.packageid and a.cid=b.cid and a.cid=${CLUSTER_ID} and b.selected=1 and (filename like 'galera%.rpm')"
		else
		    QUERY="select concat(path, filename) from cmon_uploads a , cmon_sw_package b where a.packageid=b.packageid and a.cid=b.cid and a.cid=${CLUSTER_ID} and b.selected=1 and (filename like 'galera%.deb')"
		fi
		
		XFILENAME=`$MYSQL_BIN -B -N $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$CMON_DB_HOST --port=$CMON_DB_PORT --database=$CMON_DB_DB  -e "$QUERY"`
		if [ ! -f $XFILENAME ]; then
		    log_job_message "Could not find $XFILENAME - You must have a selected deployment package, or specify with -f <package name>" 1
		    log_job 'FAILED' 'install garbd failed' 1
		    exit 1
		fi
	    fi
	    
	    if [ -z "$XFILENAME" ]; then
		log_job_message "s9s_galera --install-garbd -f <galera package missing>" 1
		log_job 'FAILED' 'install garbd failed' 1
		exit 1
	    fi
	    if [ ! -f "$XFILENAME" ]; then
		log_job_message "s9s_galera --install-garbd -f <galera package missing>" 1
		log_job 'FAILED' 'install garbd failed' 1
		exit 1
	    fi	    
	    remote_cmd $XHOSTNAME "chown -R $SSH_USER:$SSH_USER $S9S_TMPDIR"
	    remote_copy $XFILENAME $XHOSTNAME $TMPDIR	    
	    XFILE=`basename $XFILENAME`
	    if [ "$OS" = "redhat" ]; then
		remote_cmd $XHOSTNAME "rpm -Uhv --force $TMPDIR/$XFILE"
	    else
		remote_cmd $XHOSTNAME "dpkg -P galera"
		remote_cmd $XHOSTNAME "apt-get install -y libssl0.9.8"
		remote_cmd $XHOSTNAME "dpkg -i $TMPDIR/$XFILE"
	    fi
	    ;;
	mariadb)
	    if [ "$OS" = "redhat" ]; then
		cat << 'EOF' > $S9S_TMPDIR/MariaDB.centos5
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/5.5/centos5-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1                 
EOF
		cat << 'EOF' > $S9S_TMPDIR/MariaDB.centos6
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/5.5/centos6-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1                 
EOF
		cat << 'EOF' > $S9S_TMPDIR/MariaDB.rhel5
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/5.5/rhel5-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1                 
EOF
		cat << 'EOF' > $S9S_TMPDIR/MariaDB.rhel6
[mariadb]
name = MariaDB
baseurl = http://yum.mariadb.org/5.5/rhel6-amd64
gpgkey=https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
gpgcheck=1                 
EOF
		if [ -f /etc/system-release ]; then
		    RHEL=6
		    OS2='rhel'		    
		else
		    RHEL=$(cat /etc/redhat-release | egrep "[0-9]" -o | head -1)
		    OS2=$(awk '{print $1}' /etc/redhat-release | tr '[:upper:]' '[:lower:]')
		fi
		remote_cmd $XHOSTNAME "chown -R $SSH_USER:$SSH_USER $S9S_TMPDIR"
		remote_copy ${S9S_TMPDIR}/MariaDB.${OS2}${RHEL} $XHOSTNAME ${S9S_TMPDIR}/MariaDB.${OS2}${RHEL}.tmp
		remote_cmd $XHOSTNAME "mv -f ${S9S_TMPDIR}/MariaDB.${OS2}${RHEL}.tmp /etc/yum.repos.d/MariaDB.repo"
                remote_cmd $XHOSTNAME "yum -y install galera"
	    else
		wget --no-check-certificate --tries=5  -O ${S9S_TMPDIR}/RPM-GPG-KEY-percona http://www.percona.com/redir/downloads/RPM-GPG-KEY-percona	
		remote_copy ${S9S_TMPDIR}/RPM-GPG-KEY-percona ${XHOSTNAME} ${S9S_TMPDIR}/RPM-GPG-KEY-percona
		remote_cmd $XHOSTNAME "gpg --import ${S9S_TMPDIR}/RPM-GPG-KEY-percona"
		remote_cmd $XHOSTNAME "/bin/sh -c 'gpg -a --export CD2EFD2A | apt-key add -'"
		remote_cmd $XHOSTNAME "apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xcbcb082a1bb943db"
		remote_cmd $XHOSTNAME "/bin/sh -c 'echo \"deb http://ftp.osuosl.org/pub/mariadb/repo/5.5/`lsb_release -i -s | tr '[A-Z]' '[a-z]'` `lsb_release -c -s` main\" > /etc/apt/sources.list.d/MariaDB.list'"
		remote_cmd $XHOSTNAME "/bin/sh -c 'echo \"deb-src http://ftp.osuosl.org/pub/mariadb/repo/5.5/`lsb_release -i -s | tr '[A-Z]' '[a-z]'` `lsb_release -c -s` main\" >> /etc/apt/sources.list.d/MariaDB.list'"
		remote_cmd $XHOSTNAME "/bin/sh -c 'echo \"Package: *\nPin: origin osuosl.org\nPin-Priority: 1000\n\nPackage: *\nPin: origin ftp.osuosl.org\nPin-Priority: 1000\n\nPackage: libmysqlclient18\nPin: origin ftp.osuosl.org\nPin-Priority: 1000\" > /etc/apt/preferences.d/00MariaDB.pref'"
		remote_cmd $XHOSTNAME "apt-get -q -y update"
		remote_cmd $XHOSTNAME "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::='--force-confnew' --force-yes -y -q install  galera "
	    fi
	    ;;
	*)
	    echo "Not a supported vendor"
	    exit 1;	           	     
    esac
    QUERY="INSERT IGNORE INTO cmon_configuration(cid,param,value,report_ts) VALUES($CLUSTER_ID,'VENDOR', '$VENDOR', now())"
    `$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`
    if [  $? -ne 0 ]; then
	log_job_message  "Query failed. $QUERY"  1	
    fi
    echo "Starting garbd"
    start_garbd $XHOSTNAME $DONOR_HOSTNAME $GALERA_PORT $CLUSTER_NAME
    sleep 2
    QUERY="REPLACE INTO ext_proc (cid, hostname,bin, opts,cmd, proc_name, port, status) VALUES($CLUSTER_ID, '$XHOSTNAME','/usr/bin/garbd', '-a gcomm://${DONOR_HOSTNAME} -g -o gmcast.listen_addr=tcp://0.0.0.0:${GALERA_PORT}  -d', '/usr/bin/garbd  -a gcomm://${DONOR_HOSTNAME} -g $CLUSTER_NAME -o gmcast.listen_addr=tcp://0.0.0.0:${GALERA_PORT} -d','garbd', $GALERA_PORT , 0)"
    $MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$CMON_DB_HOST --port=$CMON_DB_PORT --database=$CMON_DB_DB -N -Bse "$QUERY"  2>&1 >${S9S_TMPDIR}/err.log    
    if [  $? -ne 0 ]; then
	log_job_message "Query failed: $QUERY" 1
	log_job_message "Garbd started but ClusterControl will not be able to manage the garbd" 1
	log_job 'FAILED' 'install garbd failed' 1
	cat ${S9S_TMPDIR}/err.log
	exit 1
    fi
    log_job 'FINISHED' 'install garbd' 0
    log_job_message "Successfully installed garbd on $XHOSTNAME" 0
}



function remove_garbd
{
    args=`getopt p:i:P:t:d:j:h: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -h)
		XHOSTNAME="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
	    -P)
		GALERA_PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --remove-garbd -i <cluster id> missing"
      exit 1
    fi
    init       

    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --remove-garbd -p  missing"
      exit 1
    fi

    if [ -z "$XHOSTNAME" ]; then
      echo "s9s_galera --remove-garbd -h <hostname to install garbd on is missing>"
      exit 1
    fi

    if [ -z "$GALERA_PORT" ]; then
	GALERA_PORT="4567"
    fi

##    CMON_DB_DB="cmon_${CLUSTER_ID}"
    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "./s9s_galera -- only galera is supported"
	    exit 1;	           	     
    esac
    load_opts $CLUSTER_ID
    
    QUERY="DELETE FROM ext_proc WHERE cid=$CLUSTER_ID AND hostname='$XHOSTNAME' AND proc_name='garbd'"
    
    $MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$CMON_DB_HOST --port=$CMON_DB_PORT --database=$CMON_DB_DB -N -Bse "$QUERY"  2>&1 >${S9S_TMPDIR}/err.log    
    if [  $? -ne 0 ]; then
	echo "Query failed."
	echo "Could not delete garbd from ext_proc"
	cat ${S9S_TMPDIR}/err.log
	exit 1
    fi
    remote_cmd_nofail $XHOSTNAME "killall -9 garbd"   
    log_job 'FINISHED' 'remove garbd' 0
    log_job_message "Successfully removed garbd on $XHOSTNAME" 0
}



function start_garbd
{    
    local xHNAME=$1
    local xDONOR=$2
    local xGALERA_PORT=$3
    local xCLUSTER_NAME=$4
    if [ "$STATE" = "UP" ]; then
	echo "Node $xHNAME is already started"
	exit 1
    fi
    remote_cmd3 $xHNAME "nohup nice nohup nice /usr/bin/garbd  -a gcomm://${xDONOR}:${xGALERA_PORT} -g $xCLUSTER_NAME -o gmcast.listen_addr=tcp://0.0.0.0:${xGALERA_PORT} -d > /dev/null 2>&1"    
}


function start_node_int
{    
    local xHNAME=$1
    local xNODEID=$2
    local xGALERA_PORT=$3

    STATE=$(check_node_up $xHNAME)
    if [ "$STATE" = "DOWN" ]; then
       remote_cmd3 $xHNAME "nohup $MYSQL_BASEDIR/bin/mysqld_safe --wsrep-cluster-address=gcomm://$DONOR_HOSTNAME:$xGALERA_PORT   >/dev/null 2>&1"    
       ###remote_cmd3 $xHNAME "nohup nice /etc/init.d/mysql start --wsrep-cluster-address=gcomm://$DONOR_HOSTNAME:$xGALERA_PORT > /dev/null 2>&1"
       sleep 2
       STATE=$(check_node_up $xHNAME)
       if [ "$STATE" = "UP" ]; then
           set_nodestate $xNODEID 15
       else
  	   echo "Failed to start node $xHNAME"
	   exit 1
       fi
    else
         echo "Node $xHNAME is not DOWN - refusing to start"
    fi
}



function check_node_up
{
    local xHNAME=$1
    PID=`ssh $SSH_OPTS $SSH_USER@$xHNAME "pgrep -f mysqld" 2>/dev/null`
#    PID=$(remote_cmd_getreply $xHNAME "pgrep -f mysqld")    
    if [ -n "$PID" ]; then
        echo "UP"
        return
    fi
    echo "DOWN"
}

## end create_cluster

function check_host_exists
{
    echo "x"
}


function get_galera_nodestate
{
    local xHNAME=$1
    local xVAR="$2"
    QUERY="SHOW GLOBAL STATUS LIKE '$xVAR'"
    STATE=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$xHNAME --port=$MONITORED_MYSQL_PORT -N -Bse "$QUERY"`
    if [ $? -ne 0 ]; then
	STATE=$(check_node_up $xHNAME)
	if [ "$STATE" = "DOWN" ]; then
	    echo "DOWN"
	else
	    echo "UNKNOWN - Failed to get status, but process chk indicates it is up"
	fi
    else
	STATE=`echo $STATE | awk '{print $2;}'`
	echo "$STATE"
    fi
}


function get_galera_clusterstate
{
    local xHNAME=$1
    QUERY="SHOW GLOBAL STATUS LIKE 'wsrep_cluster_status'"
    STATE=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$xHNAME --port=$MONITORED_MYSQL_PORT -N -Bse "$QUERY"  2>/dev/null`
    
    if [ $? -ne 0 ]; then
	STATE=$(check_node_up $xHNAME)
	if [ "$STATE" = "DOWN" ]; then
	    echo "DOWN"
	else
	    echo "UNKNOWN"
	fi
    else
	STATE=`echo $STATE | awk '{print $2;}'`
	echo "$STATE"
    fi
}


function set_nodestate
{
    NODEID=$1
    STATE=$2
    QUERY="UPDATE mysql_server SET status=$STATE WHERE cid=$CLUSTER_ID AND nodeid=$NODEID"
    `$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY" > /tmp/err`
    if [ $? -ne 0 ]; then
        echo "Warning! setting state failed - could not reach CMON DB (which can be ok)."
 	cat /tmp/err
    fi
}

function set_clusterstate
{
    STATE=$1
    QUERY="UPDATE cluster_state SET report_ts=NOW(), status='$STATE' WHERE id=$CLUSTER_ID"
    `$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`
    if [ $? -ne 0 ]; then
        echo "setting cluster state failed"
    fi
}

function reinit_cluster
{

    args=`getopt p:i:P:t:d:j:fr $*`
    set -- $args
    FORCE=0
    RESET=0
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -d)
		DONOR_HOSTNAME="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
            -f)
		FORCE="1"; shift;
		shift;;
            -r)
		RESET="1"; shift;
		shift;;
	    -P)
		GALERA_PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --reinit-cluster -i <cluster id> missing"
      exit 1
    fi
    init
    

    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --reinit-cluster -p  missing"
      exit 1
    fi
    if [ -z "$DONOR_HOSTNAME" ]; then
      echo "s9s_galera --reinit-cluster -d <donor_addres -donor node is missing, specify one server address part of the cluster> missing"
      exit 1
    fi

    if [ -z "$GALERA_PORT" ]; then
	GALERA_PORT="4567"
    fi

##    CMON_DB_DB="cmon_${CLUSTER_ID}"
    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "./s9s_galera -- only galera is supported"
	    exit 1;	           	     
    esac
    echo "Stopping all nodes to make sure the Cluster is down."
    stop_cluster  $2 $3 $4 $5 $6 $7 $8 $9
    set_clusterstate "STOPPED"
    echo "Starting cluster, intiializing from $DONOR_HOSTNAME"
    load_opts $CLUSTER_ID
    

    CLUSTER_RECOVERY=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`

    QUERY="select group_concat(concat(hostname, ',', nodeid) SEPARATOR ' ') from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID}"
    SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`
    if [ -z "$OS" ]; then
	if [ `hash /usr/sbin/update-rc.d 2>/dev/null`  ]; then
	    OS="redhat"
	else
	    OS="debian"
	fi	
    fi


    QUERY="select nodeid from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID} and hostname='$DONOR_HOSTNAME'"
    DONOR_ID=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`

    if [ "$DONOR_ID" = "NULL" ]; then
	echo "$DONOR_HOSTNAME not found in cluster - check server address"
	exit 1
    fi
    set_clusterstate "STARTING"
    start_node_donor $DONOR_HOSTNAME $DONOR_ID
    MAX_RETRY=60
    RETRY=0
    while [ 1 ]; 
    do
	X=$(get_galera_nodestate $DONOR_HOSTNAME 'wsrep_local_state_comment')
	if [ "$X" = "Synced" ]; then
	    break
	fi
	RETRY=`expr $RETRY + 1`
	sleep 1
	if [ $RETRY -eq $MAX_RETRY ]; then
	    echo "Donor not Synced in $MAX_RETRY retries"
	    echo "Stop cluster, start again with another -d <donor_address>"
	    exit 1
	fi
    done
    echo ""
    echo "$DONOR_HOSTNAME is Synced"
    if [ $RESET -eq 1 ]; then
      echo "Preparing non-donor nodes"
      QUERY="SHOW GLOBAL VARIABLES LIKE 'datadir'"
      DATADIR=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$DONOR_HOSTNAME --port=$MONITORED_MYSQL_PORT -A -Bse "$QUERY"`
      if [ $? -ne 0 ]; then
         echo "Failed to get datadir"
         exit 1
      fi
      DATADIR=`echo $DATADIR | awk   '{print $2;}'`
      for S in $SERVER_LIST
      do
          HNAME=`echo $S | awk -F ',' '{print $1};'`
	  if [ "$HNAME" != "$DONOR_HOSTNAME" ]; then
             echo "-r (RESET) was specified, removing $DATADIR/grastate.dat"
	     remote_cmd $HNAME "rm -rf $DATADIR/grastate.dat"
	     remote_cmd $HNAME "rm -rf /tmp/xtrabackup_pid"
	  fi
      done
    fi
    echo "Start another node (one at a time):"
    for S in $SERVER_LIST
    do
	HNAME=`echo $S | awk -F ',' '{print $1};'`
	if [ "$HNAME" != "$DONOR_HOSTNAME" ]; then
	    echo "s9s_galera --start-node -i $CLUSTER_ID -d $DONOR_HOSTNAME -h $HNAME  "
	fi
    done
    
}


function make_primary
{

    args=`getopt p:i:P:t:h:j:f $*`
    set -- $args
    FORCE=0
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -h)
		XHOSTNAME="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
	    -P)
		GALERA_PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --make-primary -i <cluster id> missing"
      exit 1
    fi
    init    

    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --make-primary -p  missing"
      exit 1
    fi

    if [ -z "$XHOSTNAME" ]; then
      echo "s9s_galera --make-primary -h <hostname> missing"
      exit 1
    fi

    if [ -z "$GALERA_PORT" ]; then
	GALERA_PORT="4567"
    fi

##    CMON_DB_DB="cmon_${CLUSTER_ID}"
    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "./s9s_galera -- only galera is supported"
	    exit 1;	           	     
    esac

    load_opts $CLUSTER_ID
    
    QUERY="SHOW GLOBAL VARIABLES LIKE 'wsrep_provider_options'"
    PROVIDER_OPTS=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$XHOSTNAME --port=$MONITORED_MYSQL_PORT -A -Bse "$QUERY"` 
    if [ $? -ne 0 ]; then
	echo "Failed to get provider options"
	exit 1
    fi
    
    PROVIDER_OPTS=`echo $PROVIDER_OPTS | sed 's#wsrep_provider_options##g'`
    PROVIDER_OPTS=`echo $PROVIDER_OPTS | sed 's#\t##g'`
    PROVIDER_OPTS=`echo $PROVIDER_OPTS | sed 's#\\\t##g'`
    QUERY="set global wsrep_provider_options='${PROVIDER_OPTS};pc.bootstrap=1'"    
    STATE=$(get_galera_nodestate $XHOSTNAME "wsrep_local_state_comment")
    echo "State: $STATE"
    if [ "$STATE" = "Initialized" ]; then
	CSTATE=$(get_galera_clusterstate $XHOSTNAME)
	if [ "$CSTATE" = "non-Primary" ]; then	   
	    QUERY="set global wsrep_provider_options='${PROVIDER_OPTS};pc.bootstrap=1'"
	    $MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --host=$XHOSTNAME --port=$MONITORED_MYSQL_PORT -e "$QUERY"
	    if [ $? -ne 0 ]; then
		echo "$QUERY failed"
		exit 1
	    fi
	else
	    echo "Server $XHOSTNAME is not Non-primary"
	    exit 1
	fi	
    else
	echo "Server $XHOSTNAME is not Initialized"
	exit 1
    fi

    exit 0
    

}

function start_cluster
{
    args=`getopt p:i:P:t:d:j: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -d)
		DONOR_HOSTNAME="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
	    -P)
		GALERA_PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --start-cluster -i <cluster id> missing"
      exit 1
    fi
    init

    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --start-cluster -p  missing"
      exit 1
    fi

    if [ -z "$DONOR_HOSTNAME" ]; then
      echo "s9s_galera --start-cluster -d <donor_address - donor node is missing, specify one server address part of the cluster> missing"
      exit 1
    fi

    if [ -z "$GALERA_PORT" ]; then
	GALERA_PORT="4567"
    fi

##    CMON_DB_DB="cmon_${CLUSTER_ID}"
    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "./s9s_galera -- only galera is supported"
	    exit 1;	           	     
    esac
    load_opts $CLUSTER_ID
    
    QUERY="select group_concat(concat(hostname, ',', nodeid) SEPARATOR ' ') from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID}"
    SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/tmp/err`
    if [ $? -ne 0 ]; then
        echo "Could not reach CMON DB. It may be down."
        cat /tmp/err
        echo ""
        exit 1
    fi
    if [ -z "$OS" ]; then
	if [ `hash /usr/sbin/update-rc.d 2>/dev/null`  ]; then
	    OS="redhat"
	else
	    OS="debian"
	fi	
    fi


    QUERY="select nodeid from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID} and hostname='$DONOR_HOSTNAME'"
    DONOR_ID=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`

    if [ "$DONOR_ID" = "NULL" ]; then
	echo "$DONOR_HOSTNAME not found in cluster - check server address"
	exit 1
    fi

    start_node_donor $DONOR_HOSTNAME $DONOR_ID
    set_clusterstate "RECOVERING"
    
    if [ "$SERVER_LIST" != "NULL" ]; then
	for S in $SERVER_LIST
	do
	    HNAME=`echo $S | awk -F ',' '{print $1};'`
            NODEID=`echo $S | awk -F ',' '{print $2};'`
	    if [ "$HNAME" != "$DONOR_HOSTNAME" ]; then 
		start_node_int $HNAME $NODEID $GALERA_PORT
	    fi
	done
    fi
}



function status_cluster
{
    args=`getopt p:i:P:t:d:j:h: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -h)
		HOSTLIST="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
	    -P)
		GALERA_PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --status -i <cluster id> missing"
      exit 1
    fi
    init    

    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --status -p  missing"
      exit 1
    fi

##    CMON_DB_DB="cmon_${CLUSTER_ID}"
    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "./s9s_galera -- only galera is supported"
	    exit 1;	           	     
    esac

    if [ -n "$HOSTLIST" ]; then
        SERVER_LIST=`echo $HOSTLIST | sed "s/,/ /g" | sed "s/^//g" | sed "s/$//g"`
    else
	load_opts $CLUSTER_ID
	QUERY="select group_concat(hostname SEPARATOR ' ') from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID}"
        SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/tmp/err`
        if [ $? -ne 0 ]; then
            echo "Could not reach CMON DB. It may be down."
            cat /tmp/err
	    echo ""
	    echo "If you are unable to reach CMON DB due to e.g site failure then do"
	    echo "./s9s-galera --status -i$CLUSTER_ID -h<comma-separated list of galera server addresses>" 
	    exit 1
	fi
    fi
    if [ -z "$OS" ]; then
	if [ `hash /usr/sbin/update-rc.d 2>/dev/null`  ]; then
	    OS="redhat"
	else
	    OS="debian"
	fi	
    fi

    
    if [ "$SERVER_LIST" != "NULL" ]; then
	for S in $SERVER_LIST
	do
	    HNAME=`echo $S | awk -F ',' '{print $1};'`
	    X=$(get_galera_nodestate $HNAME "wsrep_local_state_comment") 
	    Y=$(get_galera_nodestate $HNAME "wsrep_cluster_size") 
	    Z=$(get_galera_nodestate $HNAME "wsrep_last_committed")
	    echo "$HNAME    status: $X, cluster_size: $Y, last committed: $Z "	    	    
	done
    fi
}




function start_node
{
    BLOCK=0
    args=`getopt p:i:P:t:d:j:h:w $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -d)
		DONOR_HOSTNAME="$2"; shift;
		shift;;
            -h)
		XHOSTNAME="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
	    -P)
		GALERA_PORT="$2"; shift;
		shift;;
	    -w)
		BLOCK=1; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_galera --start-node -i <cluster id> missing"
      exit 1
    fi
    init
    

    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_galera --start-node -p  missing"
      exit 1
    fi

    if [ -z "$DONOR_HOSTNAME" ]; then
      echo "s9s_galera --start-node -d <donor node is missing, specify one server address part of the cluster> missing"
      exit 1
    fi

    if [ -z "$XHOSTNAME" ]; then
      echo "s9s_galera --start-node -h <hostname to start is missing>"
      exit 1
    fi

    if [ -z "$GALERA_PORT" ]; then
	GALERA_PORT="4567"
    fi

##    CMON_DB_DB="cmon_${CLUSTER_ID}"
    
    case $CLUSTER_TYPE in
	galera)
	    ;;
	*)
	    echo "./s9s_galera -- only galera is supported"
	    exit 1;	           	     
    esac
    load_opts $CLUSTER_ID
    
    QUERY="select group_concat(concat(hostname, ',', nodeid) SEPARATOR ' ') from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID} and hostname='$XHOSTNAME'"
    SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`
	
    if [ $? -ne 0 ]; then
        echo "Could not reach CMON DB. It may be down."
        cat /tmp/err
        echo ""
        exit 1
    fi
    if [ -z "$OS" ]; then
	if [ `hash /usr/sbin/update-rc.d 2>/dev/null`  ]; then
	    OS="redhat"
	else
	    OS="debian"
	fi	
    fi


    QUERY="select nodeid from ${CMON_DB_DB}.mysql_server where cid=${CLUSTER_ID} and hostname='$DONOR_HOSTNAME'"
    DONOR_ID=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"  2>/dev/null`

    if [ "$DONOR_ID" = "NULL" ]; then
	echo "$DONOR_HOSTNAME not found in cluster - check server address"
	exit 1
    fi
    
    X=$(get_galera_nodestate $DONOR_HOSTNAME "wsrep_local_state_comment")
    if [ "$X" != "Synced" ]; then
	echo "Donor $DONOR_HOSTNAME is not Synced, pick another -d <donor_address>"
	exit 1
    fi
    
    if [ "$SERVER_LIST" != "NULL" ]; then
	for S in $SERVER_LIST
	do
	    HNAME=`echo $S | awk -F ',' '{print $1};'`
            NODEID=`echo $S | awk -F ',' '{print $2};'`
	    if [ "$HNAME" != "$DONOR_HOSTNAME" ]; then 
		start_node_int $HNAME $NODEID $GALERA_PORT
	    else
		echo "$DONOR_HOSTNAME and $HNAME cannot be the same"
		exit 1
	    fi
	done
    else
	echo "Node $XHOSTNAME is not part of the Cluster"
	exit 1
    fi

    if [ $BLOCK -eq 1 ]; then
        while [ 1 ];
        do
          X=$(get_galera_nodestate $HNAME 'wsrep_local_state_comment')
          if [ "$X" = "Synced" ]; then
             break
          fi
          if [ "$X" = "DOWN" ]; then
             echo "Recovery failed"
             exit 1
          fi
          sleep 1
       done
    fi
    exit 0
}


function check_mysql_client()
{
   if [ ! -f $MYSQL_BIN ]; then
        # Try normal locations:
        MYSQL_BIN="/usr/bin/mysql"
        if [ ! -f $MYSQL_BIN ]; then
             MYSQL_BIN="/usr/local/mysql/bin/mysql"
             if [ ! -f $MYSQL_BIN ]; then
		 echo "The MySQL client binary could not be found"         
		 if [ "$mysql_basedir" = "" ]; then
		     echo "mysql_basedir in /etc/cmon.cnf is not set. Add it to /etc/cmon.cnf"
		     exit 1		     
		 fi
             fi
	fi
   fi
}

if [ ! -e $LOCKFILE ]; then
    trap "rm -f $LOCKFILE; exit" INT TERM EXIT
    touch $LOCKFILE
    ARG=$1
    shift
    check_mysql_client		 
    
    case $ARG in
	--stop-cluster)
	    stop_cluster $*
	    ;;
	--start-cluster)
	    start_cluster $*
	    ;;
	--reinit-cluster)
	    reinit_cluster $*
	    ;;
	--make-primary)
	    make_primary $*
	    ;;
	--stop-node)
	    stop_node $*
	    ;;
	--start-node)
	    start_node $*
	    ;;
	--install-garbd)
	    install_garbd $*
	    ;;
	--install-notifycc)
	    install_notifycc $*
	    ;;
	--remove-garbd)
	    remove_garbd $*
	    ;;
	--status)
	    status_cluster $*
	    ;;
	*)
	    echo "Usage: "
	    echo "s9s_galera --[start-cluster|stop-cluster|reinit-cluster|make-primary|start-node|stop-node|status|install-garbd|remove-garbd|install-notifycc] <options follows>"
	    exit 1
	    ;;
    esac
    rm $LOCKFILE
    trap - INT TERM EXIT
else
    echo "$LOCKFILE exists, perhaps someone is using s9s_galera? Else remove the lock file if it is spurious"
fi

