#!/bin/bash
# Copyright 2012 Severalnines AB
#
# MODIFY THE BELOW TO SUIT YOU ENV:

jobid=0    
BACKUP_LOGFILE=/tmp/s9s_backup.log
LOCKFILE="/tmp/s9s_backup.lock"
XTRABACKUP_BIN=/usr/bin/innobackupex
ULIMIT=32768
### NO MODS BELOW UNLESS YOU KNOW WHAT YOU DO:
OSUSER=$USER   
CMON_USER=cmon
CMON_DB_DB=cmon
CMON_DB_PORT=3306
LOCAL_MYQL_PORT=3306
function init
{
    FILES=`ls /etc/cmon.cnf 2>&1`
    FILES2=`ls /etc/cmon.d/cmon*cnf 2>&1`    
    FILES="$FILES $FILES2"
    configfile=""
    for f in $FILES 
    do
	X=`grep -l cluster_id=${CLUSTER_ID} $f 2>&1 `	
	if [ $? -eq 0 ]; then 
	    source $f
            configfile=$f
	fi
    done

    if [ -z "$configfile" ]; then
	echo "No matching configuration file found having cluster_id=${CLUSTER_ID}"
	exit 1
    fi
    
    source $configfile
#CLUSTER_ID=$cluster_id
    CMON_PASSWORD=$mysql_password
    CMON_DB_HOST=$mysql_hostname
    CMON_DB_DB=cmon

    MYSQL_BIN=$mysql_basedir/bin//mysql
    MYSQL_BIN2=$mysql_bindir/mysql 
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    if ! test -f $MYSQL_BIN; then
	echo "Could not find mysql client binary"
	log_job_message "Could not find mysql client binary" 1
	log_job 'FAILED' 'backup failed' 1
	exit 1
    fi
}
function test_connection
{
    HNAME=$1    
    local MYSQL_PORT=$2
    QUERY="select 1"
    $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --host=$HNAME --port=$MYSQL_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    if [ $? -ne 0 ]; then
	echo "Could not connect the mysql server @ $HNAME:${MYSQL_PORT} with user=$CMON_USER and password='hidden'. GRANT ALL PRIVILEGES ON *.* TO 'cmon'@'127.0.0.1' IDENTIFIED BY <cmon_password> " 
	log_job_message "Could not connect the mysql server @ $HNAME:${MYSQL_PORT} with user=$CMON_USER and password='hidden'. GRANT ALL PRIVILEGES ON *.* TO 'cmon'@'127.0.0.1' IDENTIFIED BY <cmon_password> " 1
	log_job 'FAILED' 'backup failed' 1
	exit 1
    fi
}
function load_opts 
{
    local CLUSTER_ID=$1
    echo "load opts $CLUSTER_ID"
    OS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS' AND cid=$CLUSTER_ID" 2>/dev/null`
    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    HTTP_PROXY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='HTTP_PROXY' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    GALERA_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='GALERA_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_BASEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_BASEDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_SCRIPTDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SCRIPTDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_IDENTITY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_IDENTITY' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_USER=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_USER' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    SUDO=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SUDO' AND cid=$CLUSTER_ID" 2>/dev/null`
    S9S_TMPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='STAGING_DIR' AND cid=$CLUSTER_ID"`
    OS_USER_HOME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS_USER_HOME' AND cid=$CLUSTER_ID" 2>/dev/null`
    DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon.cluster_config where variable='datadir' and cid=$CLUSTER_ID order by id asc limit 1" 2>/dev/null`
    
    SSH_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10"
    if [ "$SSH_IDENTITY" = "" ]; then
	SSH_IDENTITY="-oIdentityFile=${OS_USER_HOME}/.ssh/id_rsa"
    else
	SSH_IDENTITY="-oIdentityFile=$SSH_IDENTITY"
    fi

    SSH_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    SCP_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    if [ "$SSH_USER" != "root" ]; then
	SSH_OPTS="-q -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
	SCP_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
	if [ "$SUDO" = "" ] || [ "$SUDO" = "NULL" ];  then
           SUDO=sudo
        fi
    fi
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi
    if [ "$SSH_PORT" = "" ] || [ "$SSH_PORT" = "NULL" ];  then
        SSH_PORT="22"
    fi
    SSH_OPTS="-p$SSH_PORT $SSH_OPTS -n"    
    SSH_OPTS2="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY -p$SSH_PORT"    
    if [ "$SSH_USER" != "root" ]; then
	SSH_OPTS="$SSH_OPTS -t"    
    fi
    SCP_OPTS="-P$SSH_PORT $SCP_OPTS"    
    if [ "$S9S_TMPDIR" = "" ] || [ "$TMPDIR" = "NULL" ];  then
	    S9S_TMPDIR="/tmp/"
	    TMPDIR="/tmp/"
    fi
}


function write_email
{    
    true
#  SUBJECT=$1
#  MSG=$2
#  cat $BACKUP_LOGFILE | sed  -e "s/'/\\\'/g" -e 's/"/\\"/g' > /tmp/s9s_backup_log_escaped
#  MSG="$MSG \nBackup log follows:\n `cat /tmp/s9s_backup_log_escaped`" 
#  QUERY="INSERT INTO simple_email(cid,subject,message) VALUES($CLUSTER_ID, \"$SUBJECT\", \"$MSG\")"
#  echo "$QUERY" > ${S9S_TMPDIR}/query.sql
#  $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT < ${S9S_TMPDIR}/query.sql 2>&1 >/tmp/err.log
}

function log_job_message
{    
    MSG=$1
    EXIT_CODE=$2
    QUERY="INSERT INTO cmon_job_message(cid, jobid,message,exit_code,report_ts) VALUES($CLUSTER_ID,$jobid,'$MSG',$EXIT_CODE,now())"
    if [ $jobid -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    else
	if [ $EXIT_CODE -eq 0 ]; then
	    echo "$MSG"
	else
	    echo "Failed: $MSG"
	fi
    fi
}

function log_job
{    
    STATUS=$1
    STATUS_TXT=$2
    EXIT_CODE=$3
    QUERY="UPDATE cmon_job SET status='$STATUS', status_txt='$STATUS_TXT', exit_code=$EXIT_CODE, report_ts=NOW()  WHERE cid=$CLUSTER_ID AND jobid=$jobid"
    if [ $jobid -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    fi
}

function write_logfile
{
   BACKUPID=$1
   cat $BACKUP_LOGFILE | sed  -e "s/'/\\\'/g" -e 's/"/\\"/g' > /tmp/s9s_backup_log_escaped
   LOG_CONTENT=`cat /tmp/s9s_backup_log_escaped`
   SIZE=`stat --printf='%s' /tmp/s9s_backup_log_escaped`
   HNAME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" | awk '{print $2;}'`
   QUERY="REPLACE INTO cmon_host_log(cid, hostname, filename, result_len, result, report_ts,description, tag) VALUES ($CLUSTER_ID,'$HNAME', '$BACKUP_LOGFILE', $SIZE, \"$LOG_CONTENT\", NOW(), 's9s_backup log', 's9s_backup')"

   echo "$QUERY" > ${S9S_TMPDIR}/query.sql

   $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT < ${S9S_TMPDIR}/query.sql 2>&1 >/tmp/err.log
   
   QUERY="UPDATE mysql_backup SET logfile=\"$LOG_CONTENT\" WHERE cid=$CLUSTER_ID and backupid=$BACKUPID "
   echo "$QUERY" > ${S9S_TMPDIR}/query.sql
   $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT < ${S9S_TMPDIR}/query.sql  2>&1 >/tmp/err.log
   rm -rf /tmp/s9s_backup_log_escaped
}


function start_backup
{        
    args=`getopt u:P:p:h:i:b:t:j:B:N:CT $*`
    set -- $args
    for i
    do
	    case "$i" in
            -u)
		        CMON_USER="$2"; shift;
		        shift;;
            -P)
		        CMON_DB_PORT="$2"; shift;
		        shift;;
            -h)
		        CMON_DB_HOST="$2"; shift;
		        shift;;
            -p)
		        CMON_PASSWORD="$2"; shift;
		        shift;;
	        -B)
		        MYSQL_BINDIR="$2"; shift;
		        shift;;
            -i)
		        CLUSTER_ID="$2"; shift;
		        shift;;
            -t)
		        TYPE="$2"; shift;
		        shift;;
            -j)
		        jobid="$2"; shift;
		        shift;;
            -C)
		        DISABLE_COMPRESSION="1"; 
		        COMPRESSION=0
		        shift;;
            -T)
		        USE_PIGZ="1";
		        shift;;
            -b)
		        BACKUPDIR="$2"; shift;
		        shift;;
            --)
		        shift; break;;
	    esac
    done    
    
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_backup  --backup -i <clusterid> missing"
      exit 1
    fi


    if [ -z "$DISABLE_COMPRESSION" ]; then
	COMPRESSION=1
    fi

    if [ -z "$USE_PIGZ" ]; then
	USE_PIGZ="0"
	COMPRESSION_TOOL="gzip"
    else
	COMPRESSION_TOOL="pigz"
    fi

    if [ -z "$MYSQL_BINDIR" ]; then
	init
    else
	MYSQL_BIN=$MYSQL_BINDIR
	if [ ! -f $MYSQL_BIN ]; then
	    echo "s9s_backup - mysql client not found, change -B $MYSQL_BINDIR"
	    exit 1
	fi
    fi

    if [ -z "$TYPE" ]; then
      echo "s9s_backup  --backup -t <full|incr> missing"
      exit 1
    fi

    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    if ! test -f $MYSQL_BIN; then
	    echo "Could not find mysql client binary"
	    log_job_message "Could not find mysql client binary" 1
	    log_job 'FAILED' 'backup failed' 1
	    exit 1
    fi
    test_connection $CMON_DB_HOST $CMON_DB_PORT
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi
    test_connection 127.0.0.1 $MONITORED_MYSQL_PORT

    if [ "$S9S_TMPDIR" = "" ] || [ "$TMPDIR" = "NULL" ];  then
	    S9S_TMPDIR="/tmp/"
	    TMPDIR="/tmp/"
    fi

    if [ -z "$BACKUPDIR" ]; then
	BACKUPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='BACKUPDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    fi
    if [ -z "$BACKUPDIR" ]; then
	    echo "s9s_backup  --backup -b <backupdir> missing"
	    log_job_message "BACKUPDIR is not set." 1
	    log_job 'FAILED' 'backup failed' 1
	    exit 1
    fi
    if [ -e $LOCKFILE ]; then
        PID=`cat $LOCKFILE`
        PID=`ps -ef |grep $PID |grep s9s_backup |grep -v $$`
        if [ $? -eq 0 ]; then
            log_job_message "An instance of s9s_backup is already running." 1; 
	        log_job 'FAILED' 'backup failed' 1
            echo "An instance of s9s_backup is already running.";  
            exit 1
       fi
       rm $LOCKFILE
    fi
    if [ ! -e $LOCKFILE ]; then
	    trap "rm -f $LOCKFILE; killall -9 s9s_backup_wd; rm -rf /tmp/xtrabackup_pid exit" INT TERM EXIT
	    echo $$ >  $LOCKFILE
	    if which pv &> /dev/null; then
            #THROTTLE="--throttle=30"
	        HAS_PV=1
	    else
 # echo "pv not found"
 #echo "Warning: Throttling (--throttle=) will not be possible"
 #echo "Install with: "
 #echo "  apt: apt-get install pv "
 #echo "  yum: yum install pv "
	        THROTTLE=""
	    fi
  # There is something wrong with throttle.. innobackup never seems to terminate
	    THROTTLE=""
  ##TYPE=<incremental|full>
  ##MYCNF - Path to my.cnf
  ##Where to store the backup
	    FAIL=0
	    if [ "$jobid" = "" ]; then 
	        jobid=0
	    fi
	    I=0
	    while [ $I -lt 5 ]; 
	    do
	        log_job 'RUNNING' 'Job is running' 0
	        I=`expr $I + 1`
	        sleep 1
	    done	
        
        
	    if ! test -f $XTRABACKUP_BIN ; then
	        install $2 $3 $4 $5 $6 $7 $8 $9
	    fi
	    if [ hash apt-get 2>/dev/null ]; then
	        OS="redhat"
	    else
	        OS="debian"
	    fi
        
	    if [ "$COMPRESSION_TOOL" = "pigz" ]; then
	        if [ "$OS" = "debian" ]; then
		        hash pigz 2>/dev/null || {  $HTTP_PROXY apt-get -y install pigz ; }
	    else
		        hash pigz 2>/dev/null || {  $HTTP_PROXY yum -y install pigz ; }
	        fi
	    fi
        
	    QUERY="select count(column_name) from information_schema.columns where table_schema='$CMON_DB_DB' and table_name='mysql_backup' and column_name='md5sum'"
	    CNT=`$MYSQL_BIN  -A -B -N --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>/dev/null`
	    if [ $CNT -eq 0 ]; then 
	        QUERY="ALTER TABLE $CMON_DB_DB.mysql_backup ADD COLUMN md5sum VARCHAR(255) DEFAULT ''"
	        $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
	        if [  $? -ne 0 ]; then
		        echo "Query failed: $QUERY"
		        echo ""
		        cat /tmp/err.log
		        log_job_message "Query 1 failed" 1
		        log_job 'FAILED' 'backup failed' 1
		        exit 1
	        fi
	fi
        
	    QUERY="select count(column_name) from information_schema.columns where table_schema='$CMON_DB_DB' and table_name='mysql_backup' and column_name='compressed'"
	    CNT=`$MYSQL_BIN  -A -B -N --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>/dev/null`
	    if [ $CNT -eq 0 ]; then 
	        QUERY="ALTER TABLE $CMON_DB_DB.mysql_backup ADD COLUMN compressed TINYINT DEFAULT '1'"
	        $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
	        if [  $? -ne 0 ]; then
		        echo "Query failed: $QUERY"
		        echo ""
		        cat /tmp/err.log
		        log_job_message "Query 1 failed" 1
		        log_job 'FAILED' 'backup failed' 1
		        exit 1
	        fi
	    fi

        
	    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
	    HNAME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" 2>/dev/null | awk '{print $2;}'`
        
	    if [ -z "$TYPE" ]; then
	        TYPE="full"
	    fi
        
	    MYCNF="${CONFIGDIR}/my.cnf"
        
	    
	    if [ -z "$BACKUPDIR" ]; then
	        echo "s9s_backup --backup -i <clusterid> -t <full|incremental> -b <backup dir>"
	        log_job_message "Wrong arguments - backupdir not specified" 1
	        log_job 'FAILED' 'backup failed' 1
	    exit 1
	    fi
	    
	    if [ ! -f $MYCNF ]; then
	        log_job_message "Could not find $MYCNF" 1
	        log_job 'FAILED' 'backup failed' 1
	        exit 1
	    fi
	    
	    curr_time=`date +%Y-%m-%d-%H-%M-%S`;
	    case $TYPE in
	        full)
		        if [ $COMPRESSION -eq 1 ]; then
		            FILENAME="backup-full-${curr_time}.tar.gz"
		        else
		            FILENAME=""
		        fi
		        PARENTID=0
		        log_job_message "Starting a FULL backup" 0
		        ;;      
	        incr|incremental)
		        TYPE="incremental"
		        PARENTID=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select backupid from mysql_backup where backup_type='full' and hostname='$HNAME' and cid=$CLUSTER_ID and lsn>0 order by report_ts desc limit 1" 2>/dev/null`
		        LSN=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select lsn from mysql_backup where cid=$CLUSTER_ID and hostname='$HNAME' and status='completed' and lsn>0 order by report_ts desc LIMIT 1" 2>/dev/null`
		        if [ -z "$LSN" ]; then
		            log_job_message "No valid LSN, refusing incremental backup. Take a full backup" 1
		            log_job 'FAILED' 'backup failed' 1
		            echo "No valid LSN, refusing incremental backup. Take a full backup"
		            exit 1
		        fi
		        if [ "$LSN" = "NULL" ]; then
		            log_job_message "No valid LSN, refusing incremental backup. Take a full backup" 1
		            log_job 'FAILED' 'backup failed' 1
		            echo "No valid LSN, refusing incremental backup. Take a full backup"
		            exit 1
		        fi
		        if [ $PARENTID -eq 0 ]; then
		            log_job_message "No valid LSN, refusing incremental backup. Take a full backup" 1
		            log_job 'FAILED' 'backup failed' 1
                    
		            echo "No valid parentid, refusing incremental backup. Take a full backup"
		            exit 1
		        fi
		        
		        COMPRESSION=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select compressed from mysql_backup where cid=$CLUSTER_ID and hostname='$HNAME' and status='completed' and backup_type='full' and lsn>0  order by report_ts desc LIMIT 1" 2>/dev/null`
		        if [ -z "$COMPRESSION" ]; then
		            COMPRESSION=1
		        fi
		        if [ "$COMPRESSION" = "NULL" ]; then
		            COMPRESSION=1
		        fi
                
		        if [ $COMPRESSION -eq 1 ]; then
		            FILENAME="backup-incr-${curr_time}.tar.gz"
		        else
		            FILENAME=""
		        fi
		        log_job_message "Starting an INCREMENTAL backup" 0
		        ;;
	        *)
		    echo "s9s_backup --backup -t <full|incremental> -i <clusterid>"
		    exit 1
	    esac
	    
	    if [ ! -e $MYCNF ]; then
	        echo "$MYCNF could not be found"
	        exit 1
	    fi
	    
        
	    DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT -e "SHOW GLOBAL VARIABLES LIKE 'datadir'" 2>/dev/null | awk '{print $2;}'`
	    
	    ID=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select ifnull(max(backupid)+1,1) from mysql_backup" 2>/dev/null`
	    CLUSTER_TYPE=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select type from cluster where id=$CLUSTER_ID" 2>/dev/null`
	    GALERA_INFO=""
	    BACKUPDIR=${BACKUPDIR}/BACKUP-${ID}
	    if [ $COMPRESSION -eq 1 ]; then
	        mkdir -p $BACKUPDIR	
	    fi
	    log_job_message "Backup directory: $BACKUPDIR" 0
	    if [ "$CLUSTER_TYPE" = "galera" ]; then
	        GALERA_INFO="--galera-info"
	    fi
	    
	    if [ $? -ne 0 ]; then
	        log_job_message "Failed to retrieve next backup id" 1
	        log_job 'FAILED' 'backup failed' 1	    
	        echo "Failed to retrieve next backup id"
	        exit 1
	    fi
	    if [ $? -ne 0 ]; then
	        log_job_message "Failed to retrieve next backup id" 1
	        log_job 'FAILED' 'backup failed' 1
	        echo "Failed to retrieve next backup id"
	        exit 1
	    fi
	    QUERY="UPDATE mysql_backup SET status='failed', error=1 WHERE status='running' AND hostname='$HNAME' "
	    $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY" 2>/dev/null
	    
	    QUERY="INSERT INTO mysql_backup(backupid, cid, hostname,directory,filename, size, error, status, report_ts, backup_type, backup_method, storage_host, compressed) VALUES($ID, $CLUSTER_ID, '$HNAME', '$BACKUPDIR','',0,0,'running',now(), '$TYPE', 'xtrabackup', '$HNAME', $COMPRESSION)"
	    $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY" 2>/dev/null
	    if [ $? -ne 0 ]; then
	        log_job_message "Failed to create backup record" 1
	        log_job 'FAILED' 'backup failed' 1
	        echo "Failed to create backup record"
	        exit 1
	    fi
	    log_job 'RUNNING' 'Job is running' 0
        
	    rm -rf $BACKUP_LOGFILE
	    if [ -f /usr/bin/s9s_backup_wd ]; then
            /usr/bin/s9s_backup_wd $BACKUP_LOGFILE &
        else
  	        if [ -f /usr/local/cmon/bin/s9s_backup_wd ]; then
                /usr/local/cmon/bin/s9s_backup_wd $BACKUP_LOGFILE &
	        else
                echo "s9s_backup_wd not found in /usr/bin/s9s_backup_wd  or /usr/local/cmon/bin/s9s_backup_wd " 
	            exit 1
            fi 
        fi
        pid="$!"
	    log_job 'RUNNING' 'Job is running' 0
	    if [ "$TYPE" = "full" ]; then 
	        if [ $COMPRESSION -eq 0 ]; then  
		        $XTRABACKUP_BIN $GALERA_INFO  --defaults-file=$MYCNF --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT --user=$CMON_USER --password=$CMON_PASSWORD --no-timestamp $BACKUPDIR 2>$BACKUP_LOGFILE 
		        ret=$?
	        else
		        $XTRABACKUP_BIN $GALERA_INFO  --defaults-file=$MYCNF --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT --user=$CMON_USER --password=$CMON_PASSWORD $THROTTLE  --stream=tar ./  2>$BACKUP_LOGFILE | $COMPRESSION_TOOL - > $BACKUPDIR/$FILENAME 
		        ret=$?
	        fi
	        LSN=`cat $BACKUP_LOGFILE | grep 'xtrabackup: The latest check point (for incremental):' | awk -F ':' '{print $3;}'`
	        if [ -z "$LSN" ]; then
		        LSN=0
		        ret=1
	        fi
	    else
	        if [ $COMPRESSION -eq 0 ]; then  
		        $XTRABACKUP_BIN $GALERA_INFO  --defaults-file=$MYCNF --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT --user=$CMON_USER --password=$CMON_PASSWORD --incremental --incremental-lsn=$LSN --no-timestamp $BACKUPDIR 2>$BACKUP_LOGFILE 
		        ret=$?
	        else
		        $XTRABACKUP_BIN $GALERA_INFO  --defaults-file=$MYCNF --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT --user=$CMON_USER --password=$CMON_PASSWORD $THROTTLE --incremental --incremental-lsn=$LSN  --stream=xbstream ./ 2>$BACKUP_LOGFILE | $COMPRESSION_TOOL - > $BACKUPDIR/$FILENAME 		
		        ret=$?
	        fi
	        LSN=`cat $BACKUP_LOGFILE | grep 'xtrabackup: The latest check point (for incremental):' | awk -F ':' '{print $3;}'`
	        if [ -z "$LSN" ]; then
		        ret=1
		        LSN=0
	        fi
	    fi
        wait $pid  || let "FAIL=1"
	    log_job_message "wait $pid  --> FAIL=$FAIL  (FAIL=1 means failed)" 0
	    write_logfile $ID
	    if [ $ret -ne 0 ] || [ $FAIL -eq 1 ] ; then
	        STATUS='failed'
	        SIZE=0
	        FILENAME=''
	        ret=1
            write_email "Backup failed" "s9s_backup failed - check log on ${HOSTNAME}:$BACKUP_LOGFILE."
	        log_job_message "Backup failed, innobackupex failed" 1
	        log_job 'FAILED' 'backup failed' 1
	    else
	        STATUS='completed'
            ret=0
	        if [ $COMPRESSION -eq 1 ]; then  
		        SIZE=`stat --printf='%s' $BACKUPDIR/$FILENAME`
		        SUM=`md5sum $BACKUPDIR/$FILENAME | awk '{print $1;}'`
	        else
		        SIZE=`du -sb $BACKUPDIR |awk {'print $1'}`
		        SUM=0
	        fi
	        if [ $? -ne 0 ]; then
		        SIZE=0
		        FILENAME=''
		        ret=1
                write_email "Backup failed" "s9s_backup failed - getting md5sum of backup failed"
		        log_job_message "Backup failed, getting md5sum of backup failed" 1
		        log_job 'FAILED' 'backup failed' 1
	        fi
            if [ $SIZE -eq 0 ]; then
                SIZE=0
                FILENAME=''
                write_email "Backup failed" "s9s_backup failed - getting size of backup failed"
		        log_job_message "Backup failed, getting size of backup failed" 1
		        log_job 'FAILED' 'backup failed' 1
                ret=1
            fi
	    fi
	    if [ $TYPE = "full" ]; then
	        PARENTID=$ID
	    fi
	    
	    QUERY="UPDATE mysql_backup SET filename='$FILENAME', size=$SIZE, error=$ret, status='$STATUS', lsn=$LSN, md5sum='$SUM', report_ts=now(), parentid=$PARENTID WHERE backupid=$ID"
	    $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY" 2>/dev/null
	    if [ $? -ne 0 ]; then
	        log_job_message "Failed to update backup record" 1
	        log_job 'FAILED' 'backup failed' 1
	        echo "Failed to upate backup record"
	        rm -rf /tmp/xtrabackup_pid
	    exit 1
	    fi
        if [ $ret -eq 0 ]; then
            write_email "Backup finished" "s9s_backup finished ok - check Backups for status"
	        log_job_message "Backup finished - check Backups for status" 0
   	        log_job 'FINISHED' 'Command OK' 0
	    else
 	        write_email "Backup failed" "s9s_backup failed - check Backups and the backup log for status. If backup log is empty, then check diskspace."
   	        log_job_message "Backup failed - check Backups for status" 1
  	        log_job 'FAILED' 'Backup failed' 1
        fi
	    rm $LOCKFILE
	    rm -rf /tmp/xtrabackup_pid
	    trap - INT TERM EXIT
	    exit $ret
    else
        log_job_message "Lockfile $LOCKFILE exists" 1
        log_job 'FAILED' 'backup failed' 1
        exit 1
    fi	
}


function purge_backups
{    
    args=`getopt u:P:B:h:p:N:i:r: $*`
    set -- $args
    for i
    do
	case "$i" in
            -u)
		CMON_USER="$2"; shift;
		shift;;
            -P)
		CMON_DB_PORT="$2"; shift;
		shift;;
            -h)
		CMON_DB_HOST="$2"; shift;
		shift;;
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
	    -B)
		MYSQL_BINDIR="$2"; shift;
		shift;;    
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;     
            -r)
		RETENTION="$2"; shift;
		shift;;     
            --)
		shift; break;;
		esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_backup  --backup -i <clusterid> missing"
      exit 1
    fi
    if [ -z "$MYSQL_BINDIR" ]; then
	    init
    else
	    MYSQL_BIN=$MYSQL_BINDIR
	    if [ ! -f $MYSQL_BIN ]; then
	        echo "s9s_backup - mysql client not found, change -B $MYSQL_BINDIR"
	        exit 1
	    fi
    fi    
    if ! test -f $MYSQL_BIN; then
	    if ! test -f $MYSQL_BIN2; then
	        echo "Could not find mysql client binary"
	        echo "Change MYSQL_BIN in beginning of the scipt"
	        exit 1
	    fi
	    MYSQL_BIN=$MYSQL_BIN2
    fi
    test_connection $CMON_DB_HOST $CMON_DB_PORT
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi
    test_connection 127.0.0.1 $MONITORED_MYSQL_PORT
    HNAME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" 2>/dev/null | awk '{print $2;}' `
    if [ -z "$RETENTION" ]; then
	    RETENTION=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='BACKUP_RETENTION' AND cid=$CLUSTER_ID" 2>/dev/null`
	    if [ $? -ne 0 ]; then
            RETENTION=32
	    fi   
    fi
    
    if [ $RETENTION -eq 0 ]; then
	    echo "Retention period is 0, no retenetion. Set in Cluster Setup or do"
	    echo "s9s_backup --purge -r <retention days>"
	    echo "Backups older than <retention days> will be purged"
	    exit 0
    fi
    
    echo "Round 1: purge failed backups - retention $RETENTION"
    
    VICTIMS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select group_concat(concat(backupid, ',', directory,'/', filename) SEPARATOR ' ') from mysql_backup where status='failed' AND hostname='$HNAME' AND backup_method='xtrabackup' AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ $? -ne 0 ]; then
        echo "Failed to get files to purge"
        exit 1
    fi
    
    if [ -z "$VICTIMS" ]; then
	    echo "No failed backups to purge"
	    exit 0
    fi 
    if [  "$VICTIMS" = "NULL" ]; then
	    echo "No failed backups to purge"
    fi 
    
    for f in $VICTIMS 
    do
	    backupid=`echo $f | awk -F ',' '{print $1;}'`
	    backuppath=`echo $f | awk -F ',' '{print $2;}'`
	    if [ -f $backuppath ]; then
            echo "Purging $backuppath"
            rm -rf $backuppath
	    fi
	    `$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID" 2>/dev/null`
    done
  ## TODO: clean up purging (now ugly dupliated) but no time
    echo "Round 2: looking for candidates matching > retention period ($RETENTION)."
    VICTIMS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select group_concat(concat(backupid, ',', directory,'/', filename) SEPARATOR ' ') from mysql_backup where date_sub(now(), INTERVAL $RETENTION day) > report_ts AND backup_method='xtrabackup' AND hostname='$HNAME'  AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ $? -ne 0 ]; then
	    echo "Failed to get files to purge"
	    exit 1
    fi
    
    if [ -z "$VICTIMS" ]; then
	    echo "Nothing to purge"
	    exit 0
    fi 
    
    
    if [  "$VICTIMS" = "NULL" ]; then
	    echo "Nothing to purge"
	    exit 0
    fi 
    
    for f in $VICTIMS 
    do
	    backupid=`echo $f | awk -F ',' '{print $1;}'`
	    backuppath=`echo $f | awk -F ',' '{print $2;}'`
	    if [ -f $backuppath ]; then
            echo "Purging $backuppath"
            rm -rf $backuppath
            if [ $? -eq 0 ];  then
    		    `$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID" 2>/dev/null`
            fi
	    else
    	    `$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID" 2>/dev/null`
	    fi
    done
    
    
    echo "Round 3: removing failed mysqldumps."
    VICTIMS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select group_concat(concat(backupid, ',', directory,'/', filename) SEPARATOR ' ') from mysql_backup where status='failed' AND backup_method='mysqldump' AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ $? -ne 0 ]; then
	    echo "Failed to get files to purge"
	    exit 1
    fi
    
    if [ -z "$VICTIMS" ]; then
	    echo "Nothing to purge"
	    exit 0
    fi 
    
    
    if [  "$VICTIMS" = "NULL" ]; then
	    echo "Nothing to purge"
	    exit 0
    fi 
    
    for f in $VICTIMS 
    do
	    backupid=`echo $f | awk -F ',' '{print $1;}'`
	    `$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID" 2>/dev/null`
    done
}

function restore
{    
    args=`getopt P:u:B:p:h:i:b:t:e:N: $*`
    set -- $args
    for i
    do
	case "$i" in
            -u)
		CMON_USER="$2"; shift;
		shift;;
            -P)
		CMON_DB_PORT="$2"; shift;
		shift;;
            -h)
		CMON_DB_HOST="$2"; shift;
		shift;;
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
	    -B)
		MYSQL_BINDIR="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -b)
		BACKUPID="$2"; shift;
		shift;;
            -t)
		TARGETDIR="$2"; shift;
		shift;;
            -e)
		EXCLUDE_BACKUPID="$2"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    
    
    if [ -z "$CLUSTER_ID" ]; then
	    echo "s9s_backup  --restore -i <clusterid> missing"
	    exit 1
    fi
    if [ -z "$MYSQL_BINDIR" ]; then
	    init
    else
	    MYSQL_BIN=$MYSQL_BINDIR
	    if [ ! -f $MYSQL_BIN ]; then
	        echo "s9s_backup --restore -B <mysql_bindir>  - mysql client not found, change -B $MYSQL_BINDIR"
	        exit 1
	    fi
    fi
    if [ -z "$BACKUPID" ]; then
	    echo "s9s_backup  --restore -b <backup set> missing"
	    exit 1
    fi
    if [ -z "$TARGETDIR" ]; then
	    echo "s9s_backup  --restore -t <targetdir> missing"
	    exit 1
    fi
    
    if ! test -f $MYSQL_BIN; then
        if ! test -f $MYSQL_BIN2; then
	        echo "Could not find mysql client binary"
	        echo "Change MYSQL_BIN in beginning of the scipt"
	        exit 1
        fi
        MYSQL_BIN=$MYSQL_BIN2
    fi
    test_connection $CMON_DB_HOST $CMON_DB_PORT
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi
    test_connection 127.0.0.1 $MONITORED_MYSQL_PORT
    
    
    rm -rf $TARGETDIR/$BACKUPID
    mkdir -p $TARGETDIR/$BACKUPID
    
    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    
    
    MYCNF=${CONFIGDIR}/my.cnf
    if [ ! -f $MYCNF ]; then
        echo "Could not find my.cnf at $MYCNF"
        exit 1
    fi
    
    LIST=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "set group_concat_max_len=1024*1024;select group_concat(concat(parentid ,',', backup_type, ',', concat(directory,'/', filename), ',' , status, ',', size, ',\'' ,  date_format(report_ts, \"%Y-%M-%d_%H:%i:%s\") , '\'', ',', directory, ',', filename, ',', md5sum, ',', compressed) SEPARATOR ' ') from mysql_backup where cid=$CLUSTER_ID and parentid=$BACKUPID and backup_type='full' and lsn>0 and status='completed' order by parentid, backupid" 2>/dev/null`
    if [ $? -ne 0 ]; then
        echo "Failed to read backups"
        exit 1
    fi
    
    if [ -z "$LIST" ]; then
        echo "No backups found"
        exit 0
    fi
    
    if [ "$LIST" = "NULL" ]; then
        echo "No backups found"
        exit 0
    fi
    mkdir -p  $TARGETDIR/tmp/
    
    for f in $LIST
    do
        backupid=`echo $f | awk -F ',' '{print $1;}'`
        backup_type=`echo $f | awk -F ',' '{print $2;}'`
        location=`echo $f | awk -F ',' '{print $3;}'`
        status=`echo $f | awk -F ',' '{print $4;}'`
        size=`echo $f | awk -F ',' '{print $5;}'`
        report_ts=`echo $f | awk -F ',' '{print $6;}'`
        directory=`echo $f | awk -F ',' '{print $7;}'`
        filename=`echo $f | awk -F ',' '{print $8;}'`
        md5=`echo $f | awk -F ',' '{print $9;}'`
        compressed=`echo $f | awk -F ',' '{print $10;}'`
        if [ $compressed -eq 1 ]; then 
	        BASE_DIR="$TARGETDIR/$BACKUPID/"
	        echo "Restoring into ${BASE_DIR}:"
	        SUM=`md5sum $directory/$filename | awk '{print $1;}'`
	        
	        if [ -n "$md5" ]; then
	            if [ "$SUM" != "$md5" ]; then
		            echo "Md5sum does not match! The md5sum of $directory/$filename does not match md5sum stored in cmon db."
		            echo "Refusing to restore."
		            exit 1
	            fi
	        fi
	        tar -xizf $directory/$filename -C $TARGETDIR/$BACKUPID/
	        if [ $? -ne 0 ]; then
                echo "untar $directory/$filename failed"
	            exit 1
	        fi
        else
	        BASE_DIR="$directory"
	        echo "Restoring into ${BASE_DIR}:"
        fi
        $XTRABACKUP_BIN --defaults-file=$MYCNF  --apply-log --redo-only $BASE_DIR
        ret=$?
        if [ $ret -ne 0 ]; then
	        if [ $compressed -eq 1 ]; then 
                echo "restore full $directory/$filename failed"
	        else
                echo "restore full $directory failed"
	        fi
	        exit 1
        fi
        
        
    done
    
    LIST=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "set group_concat_max_len=1024*1024;select group_concat(concat(parentid ,',', backup_type, ',', concat(directory,'/', filename), ',' , status, ',', size, ',\'' ,  date_format(report_ts, \"%Y-%M-%d_%H:%i:%s\") , '\'', ',', directory, ',', filename, ',', md5sum, ',', compressed) SEPARATOR ' ') from mysql_backup where cid=$CLUSTER_ID and parentid=$BACKUPID and backup_type='incremental' and status='completed' and lsn>0 order by parentid, backupid" 2>/dev/null`
    if [ $? -ne 0 ]; then
        echo "Failed to read backups"
        echo "Restore FAILED"
        exit 1
    fi
    
    if [ -z "$LIST" ]; then
        echo "No increments found"
        echo "Restore completed OK"
        exit 0
    fi
    
    
  if [ "$LIST" = "NULL" ]  || [ -z "$LIST" ]; then
      echo "No increments found"   
  else
      sleep 3
      for f in $LIST
      do      
	      backupid=`echo $f | awk -F ',' '{print $1;}'`
	      if [ $backupid -ge $EXCLUDE_BACKUPID ];  then
	          continue
	      fi
	      backup_type=`echo $f | awk -F ',' '{print $2;}'`
	      location=`echo $f | awk -F ',' '{print $3;}'`
	      status=`echo $f | awk -F ',' '{print $4;}'`
	      size=`echo $f | awk -F ',' '{print $5;}'`
	      report_ts=`echo $f | awk -F ',' '{print $6;}'`
	      directory=`echo $f | awk -F ',' '{print $7;}'`
	      filename=`echo $f | awk -F ',' '{print $8;}'`
	      md5=`echo $f | awk -F ',' '{print $9;}'`
	      compressed=`echo $f | awk -F ',' '{print $10;}'`
	      if [ $compressed -eq 1 ]; then 
	          echo "Restoring INCREMENTS $directory/$filename into  $TARGETDIR/$BACKUPID/:"
	          INCR_BASE_DIR="$TARGETDIR/tmp/"
	          SUM=`md5sum $directory/$filename | awk '{print $1;}'`      
	          if [ -n "$md5" ]; then
		          if [ "$SUM" != "$md5" ]; then
		              echo "Md5sum does not match! The md5sum of $directory/$filename does not match md5sum stored in cmon db."
		              echo "Refusing to restore."
		              exit 1
		          fi
	          fi
	          
	          rm -rf ${INCR_BASE_DIR}*
	          zcat $directory/$filename  |xbstream -x -C  $INCR_BASE_DIR
	          if [ $? -ne 0 ]; then
		          echo "xbstream failed"
		          echo "Restore FAILED"
		          exit 1
	          fi
	      else
	          INCR_BASE_DIR=$directory
	      fi
	      $XTRABACKUP_BIN --defaults-file=$MYCNF   --apply-log --redo-only --incremental-dir=$INCR_BASE_DIR $BASE_DIR
	      ret=$?
	      if [ $ret -ne 0 ]; then
              echo "restore incremental $directory/$filename failed"
	          echo "Restore FAILED"
	          exit 1
	      fi
      done
  fi
  rm -rf $TARGETDIR/tmp/
  
  $XTRABACKUP_BIN --defaults-file=$MYCNF   --apply-log $BASE_DIR
  ret=$?
  if [ $ret -ne 0 ]; then
      echo "restore innobackupex --apply-log failed"
      echo "Restore FAILED"
      exit 1
  fi
  
  if [ -f ${BASE_DIR}/xtrabackup_galera_info ]; then
      UUID=`cat ${BASE_DIR}/xtrabackup_galera_info | awk -F: '{print $1;}'`
      SEQNO=`cat ${BASE_DIR}/xtrabackup_galera_info | awk -F: '{print $2;}'`
      cat << 'EOF' > ${BASE_DIR}/grastate.dat
version: 2.1
uuid:    _UUID
seqno:   _SEQNO
cert_index:
EOF
      sed -i.bak "s/_UUID/$UUID/g" ${BASE_DIR}/grastate.dat
      sed -i.bak "s/_SEQNO/$SEQNO/g" ${BASE_DIR}/grastate.dat
  fi
  echo "Restore OK"
  echo "To copy back the restored data into your datadir of mysqld do:"
  echo "* Shutdown the cmon controller to prevent automatic recovery"
  echo "* Shutdown the cmon agent on this host to prevent automatic recovery"
  echo "* Shutdown the mysql server "
  echo "* innobackupex --copy-back ${BASE_DIR}"
  echo "* chwon mysql:mysql -R <mysqld datadir>"  

  exit 0
}


function list_backups
{    
    args=`getopt u:P:h:B:p:i: $*`
    set -- $args
    for i
    do
	case "$i" in
            -u)
		CMON_USER="$2"; shift;
		shift;;
            -P)
		CMON_DB_PORT="$2"; shift;
		shift;;
            -h)
		CMON_DB_HOST="$2"; shift;
		shift;;
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
	    -B)
		MYSQL_BINDIR="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;     
            --)
		shift; break;;
		esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_backup  --list -i <clusterid> missing"
      exit 1
    fi
    if [ -z "$MYSQL_BINDIR" ]; then
	init
    else
	MYSQL_BIN=$MYSQL_BINDIR
	if [ ! -f $MYSQL_BIN ]; then
	    echo "s9s_backup --restore -B <mysql_bindir>  - mysql client not found, change -B $MYSQL_BINDIR"
	    exit 1
	fi
    fi
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    test_connection $CMON_DB_HOST $CMON_DB_PORT
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi
    test_connection 127.0.0.1 $MONITORED_MYSQL_PORT

    HNAME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" 2>/dev/null | awk '{print $2;}'`
    LIST=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "set group_concat_max_len=1024*1024;select group_concat(concat(parentid ,',', backup_type, ',', concat(directory,'/', filename), ',' , status, ',', size, ',\'' ,  date_format(report_ts, \"%Y-%M-%d_%H:%i:%s\") , '\'', ',',lsn, ',', md5sum, ',', compressed) SEPARATOR ' ') from mysql_backup where cid=$CLUSTER_ID and backup_method='xtrabackup' and hostname='$HNAME' and parentid>0 order by parentid, backupid" 2>/dev/null`
    if [ $? -ne 0 ]; then
	echo "Failed to query for backups"
	exit 1
    fi
    
    if [ -z "LIST" ]; then
	echo "No backups found"
	exit 0
    fi

    for f in $LIST
    do
	backupid=`echo $f | awk -F ',' '{print $1;}'`
	backup_type=`echo $f | awk -F ',' '{print $2;}'`
	location=`echo $f | awk -F ',' '{print $3;}'`
	status=`echo $f | awk -F ',' '{print $4;}'`
	size=`echo $f | awk -F ',' '{print $5;}'`
	report_ts=`echo $f | awk -F ',' '{print $6;}'`
	lsn=`echo $f | awk -F ',' '{print $7;}'`
	md5=`echo $f | awk -F ',' '{print $8;}'`
	compressed=`echo $f | awk -F ',' '{print $9;}'`
	VERIFICATION_PARENT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select count(backupid) from mysql_backup where cid=$CLUSTER_ID and backup_method='xtrabackup' and hostname='$HNAME' and backupid=$backupid order by parentid, backupid" 2>/dev/null`
	if [ $VERIFICATION_PARENT -gt 0 ]; then	  
	    if [ "$backup_type" = "full" ];  then
		echo "==========================================================================="
		echo "Full backup taken $report_ts:"
		echo "   server:     $HNAME"
		echo "   backup set: $backupid"
		echo "   status:     $status"
		echo "   location:   $location ($size bytes)"
		echo "   compressed: $compressed"
		echo "   lsn:        $lsn"
		echo "   md5:        $md5"
	    else
		echo "   >> incremental backup taken $report_ts:"
		echo "      server:     $HNAME"
		echo "      parent:     $backupid"
		echo "      status:     $status"
		echo "      location:   $location ($size bytes)"	
		echo "      compressed: $compressed"
		echo "      lsn:        $lsn"
		echo "      md5:        $md5"
	    fi
	else
            echo "==========================================================================="
	  echo "WARNING! Incremental backupid $backupid is incomplete, no FULL backup found"
	fi
    done
}




function install
{    
    CROND=""
    BACKUPDIR=""
    args=`getopt u:P:B:h:p:i:b:N:c $*`
    set -- $args
    for i
    do
	case "$i" in
            -u)
		CMON_USER="$2"; shift;
		shift;;
            -P)
		CMON_DB_PORT="$2"; shift;
		shift;;
            -h)
		CMON_DB_HOST="$2"; shift;
		shift;;
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
	    -B)
		MYSQL_BINDIR="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;     
            -b)
		BACKUPDIR="$2"; shift;
		shift;;     
            -c)
		CROND="1"; shift;
		shift;;     
            --)
		shift; break;;
		esac
    done    

    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_backup  --install -i <clusterid> missing"
      exit 1
    fi

    if [ -z "$MYSQL_BINDIR" ]; then
	init
    else
	MYSQL_BIN=$MYSQL_BINDIR
	if [ ! -f $MYSQL_BIN ]; then
	    echo "s9s_backup - mysql client not found, change -B $MYSQL_BINDIR"
	    exit 1
	fi
    fi

    if [ -z "$BACKUPDIR" ]; then
	BACKUPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='BACKUPDIR' AND cid=$CLUSTER_ID"  2>/dev/null`
    fi

    OS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS' AND cid=$CLUSTER_ID"  2>/dev/null`
    if [ $? -ne 0 ]; then 
	log_job_message "Failed to read OS property from cmon_configuration" 1
	echo "Failed - ensure that user '$CMON_USER' is allowed to SELECT from '$CMON_DB_HOST'"
	echo "Try and issue the following GRANT on $CMON_DB_HOST if you have not doen so:"
	echo "GRANT SELECT, INSERT, UPDATE, DELETE ON cmon.* TO '$CMON_USER'@'<ip of this server>' IDENTIFIED BY '$CMON_PASSWORD';"
	echo "GRANT SUPER ON *.* TO '$CMON_USER'@'<ip of this server>' IDENTIFIED BY '$CMON_PASSWORD';"
	exit 1
    fi
    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID"  2>/dev/null`
    MYCNF=${CONFIGDIR}/my.cnf
    
    if [ -z "$OS" ]; then
	log_job_message "Could not start s9s_backup - OS is not defined in cmon_configuration" 1
	echo "./s9s_backup --install -i <clusterid> -c" 
        exit 1
    fi
    
    if [ -z "$MYCNF" ]; then
	MYCNF=/etc/my.cnf
    fi        
    
    if [ ! -f $MYCNF ]; then
	echo "$MYCNF not found!"
	log_job_message "Could not start s9s_backup - $MYCNF not found" 1
	echo "./s9s_backup --install -i <clusterid -c "
	exit 1
    fi
    load_opts $CLUSTER_ID
    mkdir -p $S9S_TMPDIR
    chown -R ${SSH_USER}:${SSH_USER} ${S9S_TMPDIR}
    chown mysql:mysql  $MYCNF 
    log_job_message "Installing xtrabackup" 0
    case $OS in
	redhat)
            $HTTP_PROXY rpm -Uhv http://www.percona.com/downloads/percona-release/percona-release-0.0-1.x86_64.rpm
	        $HTTP_PROXY yum -y install percona-xtrabackup  
	        $HTTP_PROXY yum -y install perl-Time-HiRes
	        $HTTP_PROXY yum -y install nc
            
	        if [ $? -ne 0 ]; then
                   log_job_message "Installation of xtrabackup failed" 1
		           echo "Installation of xtrabackup failed"
		           exit 1
	        fi
	        ;;
	    debian)
	        $HTTP_PROXY apt-get -q -y update 
	        hash lsb_release 2>/dev/null || {  $HTTP_PROXY apt-get install -y lsb-release ; }
	        LSB=`lsb_release -c -s`
	        if [ -z "$LSB" ]; then
		        echo "'lsb_release -c -s' return empty"
                log_job_message "lsb_release -c -s return empty" 1
		        exit 1
	        fi
	        wget -O${S9S_TMPDIR}/RPM-GPG-KEY-percona http://www.percona.com/downloads/RPM-GPG-KEY-percona
            sync
	        $HTTP_PROXY gpg --import ${S9S_TMPDIR}/RPM-GPG-KEY-percona
	        if [ $? -ne 0 ]; then
		        echo "FAILED: $HTTP_PROXY gpg --import ${S9S_TMPDIR}/RPM-GPG-KEY-percona"
                log_job_message "FAILED: $HTTP_PROXY gpg --import ${S9S_TMPDIR}/RPM-GPG-KEY-percona" 1
		        exit 1
	        fi
	        gpg -a --export CD2EFD2A | apt-key add -
	        if [ $? -ne 0 ]; then
                log_job_message "FAILED: gpg -a --export CD2EFD2A | apt-key add -" 1
		        echo "FAILED: gpg -a --export CD2EFD2A | apt-key add -"
		        exit 1
	        fi
	        sed -i.bak '/percona/d' /etc/apt/sources.list
	        echo "deb http://repo.percona.com/apt $LSB main" |  tee --append /etc/apt/sources.list
	        echo "deb-src http://repo.percona.com/apt $LSB main" | tee --append /etc/apt/sources.list
	        $HTTP_PROXY apt-get -q -y update
	        LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -o Dpkg::Options::='--force-confnew' -y -q install percona-xtrabackup 
	        if [ $? -ne 0 ]; then
		        echo "Installation of xtrabackup failed"
                log_job_message "Installation of xtrabackup failed : $HTTP_PROXY apt-get install percona-xtrabackup nc" 1
		        cp  $CONFIGDIR/my.cnf.dpkg-old $CONFIGDIR/my.cnf
		        exit 1
	    fi
	        LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -o Dpkg::Options::='--force-confnew' -y -q install netcat-openbsd netcat
	        cp  $CONFIGDIR/my.cnf.dpkg-old $CONFIGDIR/my.cnf
	        ;;
	    *)
	        echo "Unrecognized OS, must be 'redhat' or 'debian' class"
	        echo "./s9s_backup --install <my.cnf> <crond optional>"
	        exit 1;	           	     
    esac
    

#    if [ "$PWD" != "/usr/bin/" ]; then
#    echo "You must run this from /usr/bin"
#    echo "cd /usr/bin"
#    echo "./s9s_backup --install <package mgr=yum|apt> <my.cnf> <crond optional>"
#    exit 1
#    fi
#
# Full backup every SUNDAY at 0300
# Incremental backup every other day at 0300
#
    if [ -n "$CROND" ]; then
	if [ -z "$BACKUPDIR" ]; then
	    echo "s9s_backup  --install -b <backupdir> missing"
	    echo "cron will not be installed"
	else
	    cat > /etc/cron.d/s9s_backup <<EOF
0 3 * * 0 root /usr/bin/s9s_backup --backup -i $CLUSTER_ID -t full -b $BACKUPDIR
0 3 * * 1 root /usr/bin/s9s_backup --backup -i $CLUSTER_ID -t incr -b $BACKUPDIR
0 3 * * 2 root /usr/bin/s9s_backup --backup -i $CLUSTER_ID -t incr -b $BACKUPDIR
0 3 * * 3 root /usr/bin/s9s_backup --backup -i $CLUSTER_ID -t incr -b $BACKUPDIR
0 3 * * 4 root /usr/bin/s9s_backup --backup -i $CLUSTER_ID -t incr -b $BACKUPDIR
0 3 * * 5 root /usr/bin/s9s_backup --backup -i $CLUSTER_ID -t incr -b $BACKUPDIR
0 3 * * 6 root /usr/bin/s9s_backup --backup -i $CLUSTER_ID -t incr -b $BACKUPDIR
0 6 * * 0 root /usr/bin/s9s_backup --purge -i $CLUSTER_ID

EOF
	fi
    fi
    log_job_message "Installation of xtrabackup completed" 0
    echo "Install complete"
}


function checkenv
{
    echo "Checking environment:"
    if ! test -f $XTRABACKUP_BIN ; then
	echo "$XTRABACKUP_BIN not found."
	echo "Change XTRABACKUP_BIN in beginning of the scipt"
    fi
    echo "Found: $XTRABACKUP_BIN"
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    echo "Found: $MYSQL_BIN"
    echo "Environment check is ok!"
    exit 0
}

function check_mysql_client()
{
    if [ ! -f $MYSQL_BIN ]; then
        # Try normal locations:
        MYSQL_BIN="/usr/bin/mysql"
        if [ ! -f $MYSQL_BIN ]; then
            MYSQL_BIN="/usr/local/mysql/bin/mysql"
            if [ ! -f $MYSQL_BIN ]; then
		echo "The MySQL client binary could not be found"         
		if [ "$mysql_basedir" = "" ]; then
		    echo "mysql_basedir in /etc/cmon.cnf is not set. Add it to /etc/cmon.cnf"
		    exit 1		     
		fi
            fi
	fi
    fi
}

check_mysql_client
ulimit -n $ULIMIT
ARG=$1
shift
case $ARG in
    --backup)
	start_backup $*
	;;
    --purge)
	purge_backups $*
	;;
    --list)
	list_backups $*
	;;
    --install)
	install $*
	;;
    --restore)
	restore $*
	;;
    --checkenv)
	checkenv $*
	;;
    *)
    echo "Usage:"
    echo "bash ./s9s_backup <--backup|--list|--install|--purge|--restore|--checkenv> <options follows>"
    echo "e.g:"
    echo "./s9s_backup --backup"
    echo "will print out additional arguments needed"
    exit 1
    ;;
esac
