#!/bin/bash
# Copyright 2012 Severalnines AB
#
# MODIFY THE BELOW TO SUIT YOU ENV:
jobid=0
MONITORED_MYSQL_PORT=3306
function init
{    
    FILES=`ls /etc/cmon.cnf 2>&1`
    FILES2=`ls /etc/cmon.d/cmon*cnf 2>&1`    
    FILES="$FILES $FILES2"
    configfile=""
    for f in $FILES 
    do
	X=`grep -l cluster_id=${CLUSTER_ID} $f 2>&1 `	
	if [ $? -eq 0 ]; then 
	    source $f
            configfile=$f
	fi
    done

    if [ -z "$configfile" ]; then
	echo "No matching configuration file found having cluster_id=${CLUSTER_ID}"
	exit 1
    fi
    
    source $configfile
    
#CLUSTER_ID=$cluster_id
    CMON_USER=cmon
    CMON_PASSWORD=$mysql_password
    CMON_DB_HOST=$mysql_hostname
    CMON_DB_DB=cmon
    CMON_DB_PORT=$mysql_port
    BACKUP_LOGFILE=/tmp/s9s_backupc_${CLUSTER_ID}.log
    LOCKFILE="/tmp/s9s_backupc_${CLUSTER_ID}.lock"
    XTRABACKUP_BIN=/usr/bin/innobackupex
    NETCAT_PORT=59999
    ULIMIT=32768
### NO MODS BELOW UNLESS YOU KNOW WHAT YOU DO:
    OSUSER=$USER    
    MYSQL_BIN=$mysql_basedir/bin//mysql
    MYSQL_BIN2=$mysql_bindir/mysql 
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    if ! test -f $MYSQL_BIN; then
	echo "Could not find mysql client binary"
	log_job_message "Could not find mysql client binary" 1
	log_job 'FAILED' 'backup failed' 1
	exit 1
    fi
    check_mysql_client

    
    if [ "$OSUSER" != "root" ]; then
	log_job_message "must be executed as 'root' or with 'sudo'" 1
	exit 1
    fi

}

#INNOBACKUP_OPTS=" --lock-wait-threshold=40 --lock-wait-query-type=all --lock-wait-timeout=180 --kill-long-queries-timeout=20 --kill-long-query-type=all"
function write_email
{
    true
    #SUBJECT=$1
    #MSG=$2
    #cat $BACKUP_LOGFILE | sed  -e "s/'/\\\'/g" -e 's/"/\\"/g' > /tmp/s9s_backup_log_escaped_${CLUSTER_ID}
    #df -h > /tmp/s9s_diskspace_${CLUSTER_ID}
    #MSG="$MSG \nBackup log follows:\n `cat /tmp/s9s_backup_log_escaped_${CLUSTER_ID}`\n\n`cat /tmp/s9s_diskspace_${CLUSTER_ID}`" 
    #QUERY="INSERT INTO simple_email(cid,subject,message) VALUES($CLUSTER_ID, \"$SUBJECT\", \"$MSG\")"
    #echo "$QUERY" > /tmp/query.sql
    #$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  < /tmp/query.sql 2>&1 >/tmp/err.log
}

function log_job_message
{    
    MSG=$1
    EXIT_CODE=$2
    QUERY="INSERT INTO cmon_job_message(cid, jobid,message,exit_code,report_ts) VALUES($CLUSTER_ID,$jobid,'$MSG',$EXIT_CODE,now())"
    if [ $jobid -ne 0 ]; then 
	    $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    else
	    if [ $EXIT_CODE -eq 0 ]; then
	        echo "$MSG"
	    else
	        echo "Failed: $MSG"
	    fi
    fi
}

function log_job
{    
    STATUS=$1
    STATUS_TXT=$2
    EXIT_CODE=$3
    QUERY="UPDATE cmon_job SET status='$STATUS', status_txt='$STATUS_TXT', exit_code=$EXIT_CODE, report_ts=NOW()  WHERE cid=$CLUSTER_ID AND jobid=$jobid"
    if [ $jobid -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    fi
}


function write_defaults_file
{ 
   user=$1
   pass=$2
   ssh $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME "echo [client] > /tmp/s9s_pass"
   ssh $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME "echo user='$user' >> /tmp/s9s_pass"
   ssh $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME "echo password='$pass' >> /tmp/s9s_pass"
}

function write_logfile
{
   BACKUPID=$1
   cat $BACKUP_LOGFILE | sed  -e "s/'/\\\'/g" -e 's/"/\\"/g' > /tmp/s9s_backup_log_escaped_${CLUSTER_ID}
   LOG_CONTENT=`cat /tmp/s9s_backup_log_escaped_${CLUSTER_ID}`
   SIZE=`stat --printf='%s' /tmp/s9s_backup_log_escaped_${CLUSTER_ID}`
   
   QUERY="REPLACE INTO cmon_host_log(cid, hostname, filename, result_len, result, report_ts,description, tag) VALUES ($CLUSTER_ID,'$BACKUP_HOSTNAME', '$BACKUP_LOGFILE', $SIZE, \"$LOG_CONTENT\", NOW(), 's9s_backup log', 's9s_backup')"

   echo "$QUERY" > /tmp/query.sql
   
   $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  < /tmp/query.sql 2>&1 >/tmp/err.log
   
   QUERY="UPDATE mysql_backup SET logfile=\"$LOG_CONTENT\" WHERE cid=$CLUSTER_ID and backupid=$BACKUPID "

   echo "$QUERY" > /tmp/query.sql
   
   $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  < /tmp/query.sql 2>&1 >/tmp/err.log

   rm -rf /tmp/s9s_backup_log_escaped_${CLUSTER_ID}
}


function start_backup
{        
    args=`getopt p:h:i:b:t:j:CT $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -h)
		BACKUP_HOSTNAME="$2"; shift;
		shift;;
            -t)
		TYPE="$2"; shift;
		shift;;
            -j)
		jobid="$2"; shift;
		shift;;
            -b)
		BACKUPDIR="$2"; shift;
		shift;;
            -C)
		DISABLE_COMPRESSION="1"; 
		COMPRESSION=0
		shift;;
            -T)
		USE_PIGZ="1";
		shift;;
            --)
		shift; break;;
		esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_backupc  --backup -i <clusterid> missing"
      exit 1
    fi
    if [ -z "$DISABLE_COMPRESSION" ]; then
	    COMPRESSION=1
    fi
    
    if [ -z "$USE_PIGZ" ]; then
	    USE_PIGZ="0"
	    COMPRESSION_TOOL="gzip"
    else
	    COMPRESSION_TOOL="pigz"
    fi
    
    if [ -z "$BACKUP_HOSTNAME" ]; then
        echo "s9s_backupc  --backup -h <backup hostname> missing"
        exit 1
    fi
    
    if [ -z "$TYPE" ]; then
        echo "s9s_backupc  --backup -t <full|incr> missing"
        exit 1
    fi
    if [ "$jobid" = "" ]; then 
	    jobid=0
    fi
    init 
    I=0
    log_job_message "Loading options" 0
    load_opts $CLUSTER_ID
    MYCNF=${CONFIGDIR}/my.cnf
    if ! test -f $MYSQL_BIN; then
	    if ! test -f $MYSQL_BIN2; then
	        log_job_message "Could not find mysql client" 1
	        echo "Could not find mysql client binary"
	        echo "Change MYSQL_BIN in beginning of the scipt"
	        exit 1
	    fi
	    MYSQL_BIN=$MYSQL_BIN2
    fi
    if ! test -f $MYSQL_BIN; then
	    echo "Could not find mysql client binary"
	    log_job_message "Could not find mysql client binary" 1
	    log_job 'FAILED' 'backup failed' 1
	    exit 1
    fi
    
    write_defaults_file cmon $CMON_PASSWORD
    
    if [ -z "$BACKUPDIR" ]; then
	    BACKUPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='BACKUPDIR' AND cid=$CLUSTER_ID"`
    fi
    
    if [ -z "$BACKUPDIR" ]; then
      echo "s9s_backupc  --install -b <backupdir> missing"
      exit 1
    fi
    if [ -e $LOCKFILE ]; then
        PID=`cat $LOCKFILE`
        PID=`ps -ef |grep $PID |grep s9s_backup |grep -v $$`
        if [ $? -eq 0 ]; then
            log_job_message "An instance of s9s_backup is already running." 1; echo "An instance of s9s_backup is already running.";  
            exit 1
        fi
        rm $LOCKFILE
    fi
    if [ ! -e $LOCKFILE ]; then
	    trap "rm -f $LOCKFILE; killall -9 s9s_backupc_wd nc; rm -rf /tmp/xtrabackup_pid; ssh $SSH_OPTS $SSH_USER@$BACKUP_HOSTNAME rm /tmp/s9s_pass; exit" INT TERM EXIT
	    echo $$ $LOCKFILE	
	    if which pv &> /dev/null; then
            #THROTTLE="--throttle=30"
	        HAS_PV=1
	    else
 # echo "pv not found"
 #echo "Warning: Throttling (--throttle=) will not be possible"
 #echo "Install with: "
 #echo "  apt: apt-get install pv "
 #echo "  yum: yum install pv "
	        THROTTLE=""
	fi
  # There is something wrong with throttle.. innobackup never seems to terminate
	THROTTLE=""
  ##TYPE=<incremental|full>
  ##MYCNF - Path to my.cnf
  ##Where to store the backup

	FAIL=0
    I=0
	while [ $I -lt 5 ]; 
	do
	    log_job 'RUNNING' 'Job is running' 0
	    I=`expr $I + 1`
	    sleep 1
	done	
#        echo $SSH_OPTS
	x=`ssh $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME '$SUDO which innobackupex' 2>/dev/null`
	if [ "$x" = ""  ]; then
	    log_job_message "installing xtrabackup" 0	
	    ssh $SSH_OPTS $SSH_USER@$BACKUP_HOSTNAME "$SUDO /usr/bin/s9s_backup --install -i$CLUSTER_ID -b$BACKUPDIR "	    
	fi
    
	x=`ssh $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME '$SUDO which innobackupex' 2>/dev/null`
	if [ "$x" = "" ]; then
	    log_job_message "innobackupex not found" 1
	    log_job 'FAILED' 'backup failed' 1
	    exit 1
	fi
	log_job 'RUNNING' 'Job is running' 0
	QUERY="select count(id) from mysql_server where cid=$CLUSTER_ID AND hostname='$BACKUP_HOSTNAME'"
	CNT=`$MYSQL_BIN  -A -B -N --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" `
	if [ $CNT -eq 0 ]; then 
	    log_job_message "-h <backup hostname> not found in cluster $CLUSTER_ID" 1
	    log_job 'FAILED' 'backup failed' 1
	    exit 1
	fi

	QUERY="select count(column_name) from information_schema.columns where table_schema='$CMON_DB_DB' and table_name='mysql_backup' and column_name='md5sum'"
	CNT=`$MYSQL_BIN  -A -B -N --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" `
	if [ $CNT -eq 0 ]; then 
	    QUERY="ALTER TABLE $CMON_DB_DB.mysql_backup ADD COLUMN md5sum VARCHAR(255) DEFAULT ''"
	    $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
	    if [  $? -ne 0 ]; then
		echo "Query failed: $QUERY"
		echo ""
		cat /tmp/err.log
		log_job_message "Query 1 failed" 1
		log_job 'FAILED' 'backup failed' 1
		exit 1
	    fi
	fi


	CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID"`
	
	STORAGE_HNAME=$CMON_DB_HOST
	DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=$BACKUP_HOSTNAME --port=$MONITORED_MYSQL_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" | awk '{print $2;}'`

	if [ -z "$TYPE" ]; then
	    TYPE="full"
	fi
	
	
	curr_time=`date +%Y-%m-%d-%H-%M-%S`;
	case $TYPE in
	    full)
		if [ $COMPRESSION -eq 1 ]; then
		    FILENAME="backup-full-${curr_time}.tar.gz"
		else
		    FILENAME="backup-full-${curr_time}.tar"
		fi
		PARENTID=0
		log_job_message "Starting a FULL backup of $BACKUP_HOSTNAME" 0
		;;      
	    incr|incremental)
		TYPE="incremental"
		PARENTID=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select backupid from mysql_backup where backup_type='full' and hostname='$BACKUP_HOSTNAME' and cid=$CLUSTER_ID and lsn>0 order by report_ts desc limit 1"`
		LSN=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select lsn from mysql_backup where cid=$CLUSTER_ID and hostname='$BACKUP_HOSTNAME' and status='completed' and lsn>0 order by report_ts desc LIMIT 1"`
		if [ -z "$LSN" ]; then
		    log_job_message "No valid LSN, refusing incremental backup. Take a full backup" 1
		    log_job 'FAILED' 'backup failed' 1
		    echo "No valid LSN, refusing incremental backup. Take a full backup"
		    exit 1
		fi
		if [ "$LSN" = "NULL" ]; then
		    log_job_message "No valid LSN, refusing incremental backup. Take a full backup" 1
		    log_job 'FAILED' 'backup failed' 1
		    echo "No valid LSN, refusing incremental backup. Take a full backup"
		    exit 1
		fi
		if [ $LSN -eq 0 ]; then
		    log_job_message "No valid LSN, refusing incremental backup. Take a full backup" 1
		    log_job 'FAILED' 'backup failed' 1

		    echo "No valid LSN, refusing incremental backup. Take a full backup"
		    exit 1
		fi
		if [ $PARENTID -eq 0 ]; then
		    log_job_message "No valid LSN, refusing incremental backup. Take a full backup" 1
		    log_job 'FAILED' 'backup failed' 1

		    echo "No valid parentid, refusing incremental backup. Take a full backup"
		    exit 1
		fi
		
		COMPRESSION=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select compressed from mysql_backup where cid=$CLUSTER_ID and hostname='$BACKUP_HOSTNAME' and status='completed' and backup_type='full' and lsn>0 order by report_ts desc LIMIT 1" 2>/dev/null`

		if [ -z "$COMPRESSION" ]; then
		    COMPRESSION=1
		fi
		
		if [ "$COMPRESSION" = "NULL" ]; then
		    COMPRESSION=1
		fi
		
		if [ $COMPRESSION -eq 1 ]; then
		    FILENAME="backup-incr-${curr_time}.tar.gz"
		else
		    FILENAME="backup-incr-${curr_time}.tar"
		fi
		log_job_message "Starting an INCREMENTAL backup of $BACKUP_HOSTNAME" 0
		;;
	    *)
		echo "s9s_backupc --backup -t <full|incremental> -i <clusterid>"
		exit 1
	esac
	
	echo $FILENAME 
	if [ ! -e $MYCNF ]; then
	    echo "$MYCNF could not be found!"
	    exit 1
	fi
	DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --host=127.0.0.1 --port=$CMON_DB_PORT -e "SHOW GLOBAL VARIABLES LIKE 'datadir'" | awk '{print $2;}'`
	ID=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select ifnull(max(backupid)+1,1) from mysql_backup"`
	if [ $? -ne 0 ]; then
	    log_job_message "Failed to retrieve next backup id" 1
	    log_job 'FAILED' 'backup failed' 1	    
	    echo "Failed to retrieve next backup id"
	    exit 1
	fi
	CLUSTER_TYPE=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select type from cluster where id=$CLUSTER_ID"`
	GALERA_INFO=""
	BACKUPDIR=${BACKUPDIR}/${BACKUP_HOSTNAME}/BACKUP-${ID}
	mkdir -p $BACKUPDIR	
	chown -R $SSH_USER:$SSH_USER $BACKUPDIR
	


	if [ "$CLUSTER_TYPE" = "galera" ]; then
	    GALERA_INFO="--galera-info"
	fi
	

	if [ $? -ne 0 ]; then
	    log_job_message "Failed to retrieve next backup id" 1
	    log_job 'FAILED' 'backup failed' 1
	    echo "Failed to retrieve next backup id"
	    exit 1
	fi
	QUERY="UPDATE mysql_backup SET status='failed', error=1 WHERE status='running' AND cc_storage=1"
	$MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"
	
	QUERY="INSERT INTO mysql_backup(backupid, cid, storage_host, hostname,cc_storage,directory,filename, size, error, status, report_ts, backup_type, backup_method,compressed) VALUES($ID, $CLUSTER_ID,'$STORAGE_HNAME', '$BACKUP_HOSTNAME', 1, '$BACKUPDIR','',0,0,'running',now(), '$TYPE', 'xtrabackup', $COMPRESSION)"
	$MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"
	if [ $? -ne 0 ]; then
	    log_job_message "Failed to create backup record" 1
	    log_job 'FAILED' 'backup failed' 1
	    echo "Failed to create backup record"
	    exit 1
	fi
	rm -rf $BACKUP_LOGFILE 
	remote_cmd_nofail $BACKUP_HOSTNAME "rm -rf $BACKUP_LOGFILE"
	remote_cmd_nofail $BACKUP_HOSTNAME "/bin/sh -c 'echo start_backup >>  $BACKUP_LOGFILE'"
	if [ -f /usr/bin/s9s_backupc_wd ]; then
            /usr/bin/s9s_backupc_wd $BACKUP_LOGFILE $BACKUP_HOSTNAME $SSH_USER "$SSH_OPTS2" "$SCP_OPTS" "$SUDO" $NETCAT_PORT $BACKUPDIR/$FILENAME   &
            pid="$!"
	    #echo "   /usr/bin/s9s_backupc_wd $BACKUP_LOGFILE $BACKUP_HOSTNAME $SSH_USER "$SSH_OPTS2" "$SCP_OPTS" "$SUDO" $NETCAT_PORT $BACKUPDIR/$FILENAME   "
	    ##exit 1
	    log_job 'RUNNING' 'Job is running' 0
        else
  	   if [ -f /usr/local/cmon/bin/s9s_backupc_wd ]; then
               /usr/local/cmon/bin/s9s_backupc_wd $BACKUP_LOGFILE $BACKUP_HOSTNAME $SSH_USER "$SSH_OPTS2" "$SCP_OPTS" "$SUDO" $NETCAT_PORT $BACKUPDIR/$FILENAME &
               pid="$!"
	   else
               echo "s9s_backup_wd not found in /usr/bin/s9s_backupc_wd  or /usr/local/cmon/bin/s9s_backupc_wd " 
	       log_message "s9s_backupc_wd was not found in /usr/bin/s9s_backupc_wd  or /usr/local/cmon/bin/s9s_backupc_wd " 1 
	       log_job 'FAILED' 'job failed' 0
	       exit 1
           fi 
        fi
	log_job_message "s9s_backupc_wd started with pid=$pid" 0
	sleep 5
	nc_pid=`pgrep -f 'nc -d'`
	if [ -z "$nc_pid" ]; then
                write_email "Backup failed" "could not start nc -d -l 59999"
                log_job_message "could not start nc -d -l 59999" 1
                log_job 'FAILED' 'backup failed' 1
                rm -rf $LOCKFILE
                trap - INT TERM EXIT
                exit 1
	fi		
	log_job_message "nc started with pid=$nc_pid" 0
	if [ "$TYPE" = "full" ]; then   
	    log_job 'RUNNING' 'Job is running' 0
	    if [ $COMPRESSION -eq 1 ]; then
 		    ssh $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME "$SUDO sh -c 'ulimit -n $ULIMIT && LC_ALL=C $XTRABACKUP_BIN $GALERA_INFO   --defaults-extra-file=/tmp/s9s_pass --host=127.0.0.1  --port=$MONITORED_MYSQL_PORT $INNOBACKUP_OPTS $THROTTLE  --stream=tar ./  2>$BACKUP_LOGFILE | $COMPRESSION_TOOL  - | nc $STORAGE_HNAME $NETCAT_PORT  '" 	   		
		    ret=$?
	    else
		    ssh $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME "$SUDO sh -c 'ulimit -n $ULIMIT && $XTRABACKUP_BIN $GALERA_INFO   --defaults-extra-file=/tmp/s9s_pass --host=127.0.0.1 --port=$MONITORED_MYSQL_PORT  $INNOBACKUP_OPTS $THROTTLE  --stream=tar ./  2>$BACKUP_LOGFILE | nc $STORAGE_HNAME $NETCAT_PORT  '" 
		    ret=$?
	    fi
	    scp -q $SCP_OPTS $SSH_USER@$BACKUP_HOSTNAME:$BACKUP_LOGFILE $BACKUP_LOGFILE
	    LSN=`cat $BACKUP_LOGFILE | grep 'xtrabackup: The latest check point (for incremental):' | awk -F ':' '{print $3;}'`
	    if [ -z "$LSN" ]; then
		LSN=0
		ret=1
	    fi
	else
	    log_job 'RUNNING' 'Job is running' 0
	    if [ $COMPRESSION -eq 1 ]; then
 		    ssh  $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME "$SUDO sh -c 'ulimit -n $ULIMIT && $XTRABACKUP_BIN $GALERA_INFO  --defaults-extra-file=/tmp/s9s_pass --host=127.0.0.1  --port=$MONITORED_MYSQL_PORT  --incremental --incremental-lsn=$LSN $INNOBACKUP_OPTS $THROTTLE  --stream=xbstream ./  2>$BACKUP_LOGFILE | $COMPRESSION_TOOL  - | nc $STORAGE_HNAME $NETCAT_PORT  '" 
		    ret=$?
	    else
 		    ssh  $SSH_OPTS  $SSH_USER@$BACKUP_HOSTNAME "$SUDO sh -c 'ulimit -n $ULIMIT && $XTRABACKUP_BIN $GALERA_INFO  --defaults-extra-file=/tmp/s9s_pass --host=127.0.0.1  --port=$MONITORED_MYSQL_PORT  --incremental --incremental-lsn=$LSN $INNOBACKUP_OPTS $THROTTLE  --stream=xbstream ./  2>$BACKUP_LOGFILE | nc $STORAGE_HNAME $NETCAT_PORT  '" 
		    ret=$?
	    fi
	    
	    scp -q $SCP_OPTS $SSH_USER@$BACKUP_HOSTNAME:$BACKUP_LOGFILE $BACKUP_LOGFILE
	    LSN=`cat $BACKUP_LOGFILE | grep 'xtrabackup: The latest check point (for incremental):' | awk -F ':' '{print $3;}'`	    
	    if [ -z "$LSN" ]; then
		ret=1
		LSN=0
	    fi
	fi

	searchexpr2="innobackupex: completed OK!"
	err=`grep "$searchexpr2" $BACKUP_LOGFILE`
	ret=0
	if [ -z "$err" ]; then
	    log_job_message "Backup failed, innobackupex failed" 1
	    ret=1
	fi
    wait $pid  || let "FAIL=1"
	log_job_message "wait $pid  --> FAIL=$FAIL" 0
	scp -q $SCP_OPTS  $SSH_USER@$BACKUP_HOSTNAME:$BACKUP_LOGFILE  $BACKUP_LOGFILE	
	write_logfile $ID
	remote_cmd_nofail $BACKUP_HOSTNAME "rm -rf /tmp/xtrabackup_pid"
	if [ $ret -ne 0 ] || [ $FAIL -eq 1 ] ; then
	    STATUS='failed'
	    SIZE=0
	    ret=1
	    FILENAME=''
        write_email "Backup failed" "s9s_backupc failed, s9s_backupc_wd failed"
	    log_job_message "Backup failed, s9s_backupc_wd failed" 1
	    log_job 'FAILED' 'backup failed' 1
	else
	    STATUS='completed'
	    ret=0
	    SIZE=`stat --printf='%s' $BACKUPDIR/$FILENAME`
	    SUM=`md5sum $BACKUPDIR/$FILENAME | awk '{print $1;}'`
	    if [ $? -ne 0 ]; then
		SIZE=0
		FILENAME=''
		ret=1
		STATUS='failed'
		write_email "Backup failed" "s9s_backupc failed, getting md5sum of backup failed."
		log_job_message "Backup failed, getting md5sum of backup failed" 1
		log_job 'FAILED' 'backup failed' 1
	    fi
            if [ $SIZE -eq 0 ]; then
                SIZE=0
                FILENAME=''
		STATUS='failed'
		write_email "Backup failed" "s9s_backupc failed,  getting size of backup failed."
		log_job_message "Backup failed, getting size of backup failed" 1
		log_job 'FAILED' 'backup failed' 1
                ret=1
            fi
            if [ $SIZE -lt 1024 ]; then
                SIZE=0
                FILENAME=''
		STATUS='failed'
		write_email "Backup failed" "s9s_backupc failed, size of backup is < 1024B, which indicated the backup failed."
		log_job_message "Backup failed, size of backup is < 1024B, which indicated the backup failed." 1
		log_job 'FAILED' 'backup failed' 1
                ret=1
            fi
	fi
	if [ $TYPE = "full" ]; then
	    PARENTID=$ID
	fi
	
	QUERY="UPDATE mysql_backup SET filename='$FILENAME', size=$SIZE, error=$ret, status='$STATUS', lsn=$LSN, md5sum='$SUM', report_ts=now(), parentid=$PARENTID WHERE backupid=$ID"
	$MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"
	if [ $? -ne 0 ]; then
	    log_job_message "Failed to update backup record" 1
	    log_job 'FAILED' 'backup failed' 1
	    echo "Failed to upate backup record"
	    exit 1
	fi
	if [ $ret -eq 0 ]; then
 	    write_email "Backup finished" "s9s_backupc finished ok - check Backups for status."
   	    log_job_message "Backup finished - check Backups for status" 0
  	    log_job 'FINISHED' 'Command OK' 0
	else
 	    write_email "Backup failed" "s9s_backupc failed - check Backups and the backup log for status. If backup log is empty, then check diskspace."
   	    log_job_message "Backup failed - check Backups for status" 1
  	    log_job 'FAILED' 'Backup failed' 1
	fi
	rm $LOCKFILE
	trap - INT TERM EXIT
	exit $ret
    else
 	    write_email "Backup failed" "s9s_backupc failed - Lock file $LOCKFILE exists which indicates another backup may be running?" 
	    log_job_message "Lock file $LOCKFILE exists which indicates another backup may be running?" 1
	    log_job 'FAILED' 'Backup failed' 1
    fi	
}


function purge_backups
{    

    args=`getopt p:i:r: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -r)
		RETENTION="$2"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_backupc  --purge -i <clusterid> missing"
	exit 1
    fi
    init
    
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    HNAME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" | awk '{print $2;}'`
    if [ -z "$RETENTION" ]; then
	RETENTION=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='BACKUP_RETENTION' AND cid=$CLUSTER_ID"`
	if [ $? -ne 0 ]; then
            RETENTION=32
	fi   
    fi
    
    if [ $RETENTION -eq 0 ]; then
	echo "Retention period is 0, no retenetion. Set in Cluster Setup or do"
	echo "s9s_backupc --purge -r <retention days>"
	echo "Backups older than <retention days> will be purged"
	exit 0
    fi
    
    echo "Round 1: purge failed backups - retention $RETENTION"
    
    VICTIMS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select group_concat(concat(backupid, ',', directory,'/', filename) SEPARATOR ' ') from mysql_backup where status='failed' AND cc_storage=1 AND backup_method='xtrabackup' AND cid=$CLUSTER_ID"`
    if [ $? -ne 0 ]; then
      echo "Failed to get files to purge"
      exit 1
    fi
    
    if [ -z "$VICTIMS" ]; then
	echo "No failed backups to purge"
	exit 0
    fi 
    if [  "$VICTIMS" = "NULL" ]; then
	echo "No failed backups to purge"
    fi 
    
    for f in $VICTIMS 
    do
	backupid=`echo $f | awk -F ',' '{print $1;}'`
	backuppath=`echo $f | awk -F ',' '{print $2;}'`
	if [ -f $backuppath ]; then
            echo "Purging $backuppath"
            rm -rf $backuppath
	fi
	`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID"`
    done
  ## TODO: clean up purging (now ugly dupliated) but no time
    echo "Round 2: looking for candidates matching > retention period ($RETENTION)."
    VICTIMS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select group_concat(concat(backupid, ',', directory,'/', filename) SEPARATOR ' ') from mysql_backup where date_sub(now(), INTERVAL $RETENTION day) > report_ts AND backup_method='xtrabackup' AND cc_storage=1  AND cid=$CLUSTER_ID"`
    if [ $? -ne 0 ]; then
	echo "Failed to get files to purge"
	exit 1
    fi
    
    if [ -z "$VICTIMS" ]; then
	echo "Nothing to purge"
	exit 0
    fi 
    
    
    if [  "$VICTIMS" = "NULL" ]; then
	echo "Nothing to purge"
	exit 0
    fi 
    
    for f in $VICTIMS 
  do
	backupid=`echo $f | awk -F ',' '{print $1;}'`
	backuppath=`echo $f | awk -F ',' '{print $2;}'`
	if [ -f $backuppath ]; then
            echo "Purging $backuppath"
            rm -rf $backuppath
            if [ $? -eq 0 ];  then
    		`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID"`
            fi
	else
    	    `$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID"`
	fi
    done
    
    
    echo "Round 3: removing failed mysqldumps."
    VICTIMS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select group_concat(concat(backupid, ',', directory,'/', filename) SEPARATOR ' ') from mysql_backup where status='failed' AND backup_method='mysqldump' AND cid=$CLUSTER_ID"`
    if [ $? -ne 0 ]; then
	echo "Failed to get files to purge"
	exit 1
    fi
    
    if [ -z "$VICTIMS" ]; then
	echo "Nothing to purge"
	exit 0
    fi 
    

    if [  "$VICTIMS" = "NULL" ]; then
	echo "Nothing to purge"
      exit 0
    fi 
    
    for f in $VICTIMS 
    do
	backupid=`echo $f | awk -F ',' '{print $1;}'`
	`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "delete from mysql_backup where backupid=$backupid AND cid=$CLUSTER_ID"`
    done
}

function restore
{    
    args=`getopt h:i:b:t:e: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -b)
		BACKUPID="$2"; shift;
		shift;;
            -t)
		TARGETDIR="$2"; shift;
		shift;;
            -e)
		EXCLUDE_BACKUPID="$2"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_backupc  --restore -i <clusterid> missing"
	exit 1
    fi
    init
    if [ -z "$BACKUPID" ]; then
	echo "s9s_backupc  --restore -b <backup set> missing"
	exit 1
    fi
    if [ -z "$TARGETDIR" ]; then
	echo "s9s_backupc  --restore -t <targetdir> missing"
	exit 1
    fi
    
  if ! test -f $MYSQL_BIN; then
      if ! test -f $MYSQL_BIN2; then
	  echo "Could not find mysql client binary"
	  echo "Change MYSQL_BIN in beginning of the scipt"
	  exit 1
      fi
      MYSQL_BIN=$MYSQL_BIN2
  fi

  load_opts $CLUSTER_ID
  MYCNF=${CONFIGDIR}/my.cnf
    
  if [ -z "$MYCNF" ]; then
      MYCNF=/etc/my.cnf
  fi
  
  
  $SUDO killall -q -15 nc
  sleep 2
  x=`which innobackupex 2>/dev/null`
  if [ "$x" = ""  ]; then     
      $SUDO /usr/bin/s9s_backupc --install -h 127.0.0.1 
  fi
  
  rm -rf $TARGETDIR/$BACKUPID
  mkdir -p $TARGETDIR/$BACKUPID
  
  if [ ! -f $MYCNF ]; then
      echo "./s9s_backupc :  $MYCNF not found"
      exit 1
  fi
  
  LIST=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "set group_concat_max_len=1024*1024;select group_concat(concat(parentid ,',', backup_type, ',', concat(directory,'/', filename), ',' , status, ',', size, ',\'' ,  date_format(report_ts, \"%Y-%M-%d_%H:%i:%s\") , '\'', ',', directory, ',', filename, ',', md5sum, ',', compressed) SEPARATOR ' ') from mysql_backup where cid=$CLUSTER_ID and parentid=$BACKUPID and backup_type='full' and lsn>0 and status='completed' order by parentid, backupid"`
  if [ $? -ne 0 ]; then
      echo "Failed to read backups"
      exit 1
  fi
  
  if [ -z "$LIST" ]; then
      echo "No backups found"
      exit 0
  fi

  if [ "$LIST" = "NULL" ]; then
      echo "No backups found"
      exit 0
  fi
  mkdir -p  $TARGETDIR/tmp/
  
  for f in $LIST
  do
      backupid=`echo $f | awk -F ',' '{print $1;}'`
      backup_type=`echo $f | awk -F ',' '{print $2;}'`
      location=`echo $f | awk -F ',' '{print $3;}'`
      status=`echo $f | awk -F ',' '{print $4;}'`
      size=`echo $f | awk -F ',' '{print $5;}'`
      report_ts=`echo $f | awk -F ',' '{print $6;}'`
      directory=`echo $f | awk -F ',' '{print $7;}'`
      filename=`echo $f | awk -F ',' '{print $8;}'`
      md5=`echo $f | awk -F ',' '{print $9;}'`
      compressed=`echo $f | awk -F ',' '{print $10;}'`
      echo "Restoring into $TARGETDIR/$BACKUPID/:"

      SUM=`md5sum $directory/$filename | awk '{print $1;}'`
      
      if [ -n "$md5" ]; then
	  if [ "$SUM" != "$md5" ]; then
	      echo "Md5sum does not match! The md5sum of $directory/$filename does not match md5sum stored in cmon db."
              echo "Refusing to restore."
	      exit 1
	  fi
      fi
      if [ $compressed -eq 1 ]; then
	  tar -xizf $directory/$filename -C $TARGETDIR/$BACKUPID/
	  ret=$?
      else
	  tar -xif $directory/$filename -C $TARGETDIR/$BACKUPID/
	  ret=$?
      fi
      if [ $ret -ne 0 ]; then
        echo "untar $directory/$filename failed"
	  exit 1
      fi
      $XTRABACKUP_BIN --defaults-file=$MYCNF  --apply-log --redo-only $TARGETDIR/$BACKUPID/
      ret=$?
      if [ $ret -ne 0 ]; then
        echo "restore full $directory/$filename failed"
	exit 1
      fi
  done

  LIST=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "set group_concat_max_len=1024*1024;select group_concat(concat(parentid ,',', backup_type, ',', concat(directory,'/', filename), ',' , status, ',', size, ',\'' ,  date_format(report_ts, \"%Y-%M-%d_%H:%i:%s\") , '\'', ',', directory, ',', filename, ',', md5sum, ',', compressed) SEPARATOR ' ') from mysql_backup where cid=$CLUSTER_ID and parentid=$BACKUPID and backup_type='incremental' and status='completed' and lsn>0 order by parentid, backupid"`
  if [ $? -ne 0 ]; then
      echo "Failed to read backups"
      echo "Restore FAILED"
      exit 1
  fi
  
  if [ -z "$LIST" ]; then
      echo "No increments found"
      echo "Restore completed OK"
      exit 0
  fi
  
  
  if [ "$LIST" = "NULL" ]; then
      echo "No increments found"
      echo "Restore completed OK"
      echo "To copy back the restored data into your datadir of mysqld do:"
      echo "* Shutdown the cmon controller to prevent automatic recovery"
      echo "* Shutdown the cmon agent on this host to prevent automatic recovery"
      echo "* Shutdown all the mysql servers in the cluster. "
      echo "* Copy $TARGETDIR/$BACKUPID/  to all the mysql servers in the cluster, e.g: "
      echo "*   scp -r $TARGETDIR/$BACKUPID/  <user>@<target_server>:"
      echo "*   On the target_server"
      echo "*   innobackupex --copy-back $BACKUPID/"
      echo "    and don't forget to:"
      echo "*   chwon mysql:mysql -R <mysqld datadir>"
      echo "Start up the cluster again."
      echo ""
      exit 0
  fi


  for f in $LIST
  do      
      backupid=`echo $f | awk -F ',' '{print $1;}'`
      if [ -n "$EXCLUDE_BACKUPID" ]; then	  
	  if [ $backupid -ge $EXCLUDE_BACKUPID ];  then
	      continue
	  fi
      fi

      backup_type=`echo $f | awk -F ',' '{print $2;}'`
      location=`echo $f | awk -F ',' '{print $3;}'`
      status=`echo $f | awk -F ',' '{print $4;}'`
      size=`echo $f | awk -F ',' '{print $5;}'`
      report_ts=`echo $f | awk -F ',' '{print $6;}'`
      directory=`echo $f | awk -F ',' '{print $7;}'`
      filename=`echo $f | awk -F ',' '{print $8;}'`
      md5=`echo $f | awk -F ',' '{print $9;}'`
      compressed=`echo $f | awk -F ',' '{print $10;}'`
      echo "Restoring INCREMENTS $directory/$filename into  $TARGETDIR/$BACKUPID/:"
      SUM=`md5sum $directory/$filename | awk '{print $1;}'`      
      if [ -n "$md5" ]; then
	  if [ "$SUM" != "$md5" ]; then
	      echo "Md5sum does not match! The md5sum of $directory/$filename does not match md5sum stored in cmon db."
              echo "Refusing to restore."
	      exit 1
	  fi
      fi

      rm -rf $TARGETDIR/tmp/*
      if [ $compressed -eq 1 ]; then
	  zcat $directory/$filename  |xbstream -x -C  $TARGETDIR/tmp/
	  ret=$?
      else
	  cat $directory/$filename  |xbstream -x -C  $TARGETDIR/tmp/
	  ret=$?
      fi

      if [ $ret -ne 0 ]; then
        echo "xbstream failed"
	echo "Restore FAILED"
	exit 1
      fi
      
      $XTRABACKUP_BIN --defaults-file=$MYCNF   --apply-log --redo-only --incremental-dir=$TARGETDIR/tmp/ /$TARGETDIR/$BACKUPID/ 
      ret=$?
      if [ $ret -ne 0 ]; then
          echo "restore incremental $directory/$filename failed"
	  echo "Restore FAILED"
	  exit 1
      fi
  done
  $XTRABACKUP_BIN --defaults-file=$MYCNF   --apply-log /$TARGETDIR/$BACKUPID/ 
  rm -rf $TARGETDIR/tmp/
  echo "Restore OK"
  
  echo "To copy back the restored data into your datadir of mysqld do:"
  echo "* Shutdown the cmon controller to prevent automatic recovery"
  echo "* Shutdown the cmon agent on this host to prevent automatic recovery"
  echo "* Shutdown all the mysql servers in the cluster. "
  echo "* Copy $TARGETDIR/$BACKUPID/  to all the mysql servers in the cluster, e.g: "
  echo "*   scp -r $TARGETDIR/$BACKUPID/*  <user>@<target_server>:"
  echo "* On the target_server"
  echo "* innobackupex --copy-back $BACKUPID/"
  echo "and don't forget to:"
  echo "* chwon mysql:mysql -R <mysqld datadir>"
  echo " Start up the cluster again."
  exit 0
}


function load_opts 
{
    local CLUSTER_ID=$1
    echo "load opts $CLUSTER_ID"
    OS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS' AND cid=$CLUSTER_ID" 2>/dev/null`
    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    HTTP_PROXY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='HTTP_PROXY' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    GALERA_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='GALERA_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_BASEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_BASEDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    MYSQL_SCRIPTDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SCRIPTDIR' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_IDENTITY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_IDENTITY' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_USER=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_USER' AND cid=$CLUSTER_ID" 2>/dev/null`
    SSH_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    SUDO=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SUDO' AND cid=$CLUSTER_ID" 2>/dev/null`
    S9S_TMPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='STAGING_DIR' AND cid=$CLUSTER_ID"`
    OS_USER_HOME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS_USER_HOME' AND cid=$CLUSTER_ID" 2>/dev/null`
    DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon.cluster_config where variable='datadir' and cid=$CLUSTER_ID order by id asc limit 1" 2>/dev/null`
    
    SSH_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10"
    if [ "$SSH_IDENTITY" = "" ]; then
	SSH_IDENTITY="-oIdentityFile=${OS_USER_HOME}/.ssh/id_rsa"
    else
	SSH_IDENTITY="-oIdentityFile=$SSH_IDENTITY"
    fi

    SSH_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    SCP_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    if [ "$SSH_USER" != "root" ]; then
	SSH_OPTS="-q -t -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
	SCP_OPTS="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
	if [ "$SUDO" = "" ] || [ "$SUDO" = "NULL" ];  then
           SUDO=sudo
        fi
    fi

    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi

    if [ "$SSH_PORT" = "" ] || [ "$SSH_PORT" = "NULL" ];  then
        SSH_PORT="22"
    fi
    SSH_OPTS="-p$SSH_PORT $SSH_OPTS -n"    
    SSH_OPTS2="-q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY -p$SSH_PORT"    
    if [ "$SSH_USER" != "root" ]; then
	SSH_OPTS="$SSH_OPTS -t"    
    fi
    SCP_OPTS="-P$SSH_PORT $SCP_OPTS"    
    if [ "$S9S_TMPDIR" = "" ] || [ "$TMPDIR" = "NULL" ];  then
	S9S_TMPDIR="/tmp/"
	TMPDIR="/tmp/"
    fi
}

function list_backups
{    
    args=`getopt p:i: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_backupc  --list -i <clusterid> missing"
	exit 1
    fi
    init
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the script"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    STORAGE_HNAME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" | awk '{print $2;}'`

    QUERY="set group_concat_max_len=1024*1024;select group_concat(concat(parentid ,',', backup_type, ',', concat(directory,'/', filename), ',' , status, ',', size, ',\'' ,  date_format(report_ts, \"%Y-%M-%d_%H:%i:%s\") , '\'', ',',lsn, ',', md5sum, ',', hostname , ',' , compressed) order by parentid, backupid SEPARATOR ' '  ) from mysql_backup where cid=$CLUSTER_ID and backup_method='xtrabackup' and cc_storage=1 and parentid>0 order by parentid, backupid"

    LIST=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`
    if [ $? -ne 0 ]; then
	echo "Failed to backups"
	exit 1
    fi
    
    if [ -z "LIST" ]; then
	echo "No backups found"
	exit 0
    fi

    for f in $LIST
    do
	backupid=`echo $f | awk -F ',' '{print $1;}'`
	backup_type=`echo $f | awk -F ',' '{print $2;}'`
	location=`echo $f | awk -F ',' '{print $3;}'`
	status=`echo $f | awk -F ',' '{print $4;}'`
	size=`echo $f | awk -F ',' '{print $5;}'`
	report_ts=`echo $f | awk -F ',' '{print $6;}'`
	lsn=`echo $f | awk -F ',' '{print $7;}'`
	md5=`echo $f | awk -F ',' '{print $8;}'`
	backup_hostname=`echo $f | awk -F ',' '{print $9;}'`
	compressed=`echo $f | awk -F ',' '{print $10;}'`
	VERIFICATION_PARENT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select count(backupid) from mysql_backup where cid=$CLUSTER_ID and backup_method='xtrabackup' and cc_storage=1 and backupid=$backupid order by parentid, backupid"`
	if [ $VERIFICATION_PARENT -gt 0 ]; then	  
	    if [ "$backup_type" = "full" ];  then
		echo "==========================================================================="
		echo "Full backup taken $report_ts:"
		echo "   backup server:      $backup_hostname"
		echo "   storage server:     $STORAGE_HNAME"
		echo "   backup set:         $backupid"
		echo "   status:             $status"
		echo "   location:           $location ($size bytes)"
		echo "   compressed:         $compressed"
		echo "   lsn:                $lsn"
		echo "   md5:                $md5"
	    else
		echo "   >> incremental backup taken $report_ts:"
		echo "      backup server:      $backup_hostname"
		echo "      storage server:     $STORAGE_HNAME"
		echo "      parent:             $backupid"
		echo "      status:             $status"
		echo "      location:           $location ($size bytes)"	
		echo "      compressed:         $compressed"
		echo "      lsn:                $lsn"
		echo "      md5:                $md5"
	    fi
	else
            echo "==========================================================================="
	  echo "WARNING! Incremental backupid $backupid is incomplete, no FULL backup found"
	fi
    done
}




function install
{    
    args=`getopt h:i: $*`
    set -- $args
    for i
    do
	case "$i" in
            -p)
		CMON_PASSWORD="$2"; shift;
		shift;;
            -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            -h)
		BACKUP_HOSTNAME="$2"; shift;
		shift;;
            --)
		shift; break;;
		esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_backup  --install -i <clusterid> missing"
      exit 1
    fi
    init 
    if [ -z "$BACKUP_HOSTNAME" ]; then
      echo "s9s_backup  --install -h <backup hostname> missing"
      exit 1
    fi

    load_opts $CLUSTER_ID

    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi

    MYCNF=${CONFIGDIR}/my.cnf
    
    if [ -z "$OS" ]; then
	log_job_message "Could not start s9s_backup - OS is not defined in cmon_configuration" 1
	echo "./s9s_backup --install <my.cnf> <crond optional>" 
        exit 1
    fi
    
    if [ -z "$MYCNF" ]; then
	MYCNF=/etc/my.cnf
    fi        
    
    if [ ! -f $MYCNF ]; then
	echo "$MYCNF not found!"
	log_job_message "Could not start s9s_backup - $MYCNF not found" 1
	echo "./s9s_backup --install <my.cnf> <crond optional>"
	exit 1
    fi
    
    chown mysql:mysql  $MYCNF 
    log_job_message "Installing xtrabackup" 0
    case $OS in
	redhat)
            remote_cmd $BACKUP_HOSTNAME "$HTTP_PROXY rpm -Uhv http://www.percona.com/downloads/percona-release/percona-release-0.0-1.x86_64.rpm"
            remote_cmd $BACKUP_HOSTNAME "$HTTP_PROXY yum -y install xtrabackup  "
	    remote_cmd $BACKUP_HOSTNAME "$HTTP_PROXY yum -y install perl-Time-HiRes"
	    remote_cmd $BACKUP_HOSTNAME "$HTTP_PROXY yum -y install nc "
	    ;;
	debian)
	    LSB=`lsb_release -c -s`
	    if [ -z "$LSB" ]; then
		echo "'lsb_release -c -s' return empty"
                log_job_message "lsb_release -c -s return empty" 1
		exit 1
	    fi
	    remote_cmd_nofail $BACKUP_HOSTNAME "mkdir -p $S9S_TMPDIR"
	    remote_cmd $BACKUP_HOSTNAME "chown -R $SSH_USER:$SSH_USER $S9S_TMPDIR"
	    remote_cmd $BACKUP_HOSTNAME "$HTTP_PROXY wget --no-check-certificate --tries=5  -O ${S9S_TMPDIR}/RPM-GPG-KEY-percona http://www.percona.com/redir/downloads/RPM-GPG-KEY-percona"
	    remote_cmd $BACKUP_HOSTNAME "gpg --import ${S9S_TMPDIR}/RPM-GPG-KEY-percona"
	    remote_cmd $BACKUP_HOSTNAME "sh -c 'gpg -a --export CD2EFD2A | apt-key add -'"
	    remote_cmd $BACKUP_HOSTNAME "sed -i.bak '/percona/d' /etc/apt/sources.list"
	    remote_cmd $BACKUP_HOSTNAME "sh -c 'echo \"deb http://repo.percona.com/apt $LSB main\" >> /etc/apt/sources.list'"
	    remote_cmd $BACKUP_HOSTNAME "sh -c 'echo \"deb-src http://repo.percona.com/apt $LSB main\" >> /etc/apt/sources.list'"
	    remote_cmd $BACKUP_HOSTNAME "$HTTP_PROXY apt-get -q -y update"
	    remote_cmd $BACKUP_HOSTNAME "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -o Dpkg::Options::='--force-confnew' -y -q install percona-xtrabackup "
	    
	    remote_cmd $BACKUP_HOSTNAME "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -o Dpkg::Options::='--force-confnew' -y -q install netcat"
	    remote_cmd_nofail $BACKUP_HOSTNAME "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -o Dpkg::Options::='--force-confnew' -y -q install netcat-openbsd"
	    remote_cmd_nofail $BACKUP_HOSTNAME "cp  $CONFIGDIR/my.cnf.dpkg-old $CONFIGDIR/my.cnf"
	    ;;
	*)
	    echo "Unrecognized OS, must be 'redhat' or 'debian' class"
	    echo "./s9s_backup --install <my.cnf> <crond optional>"
	    exit 1;	           	     
    esac
    
    
#    if [ "$PWD" != "/usr/bin/" ]; then
#    echo "You must run this from /usr/bin"
#    echo "cd /usr/bin"
#    echo "./s9s_backup --install <package mgr=yum|apt> <my.cnf> <crond optional>"
#    exit 1
#    fi
#
# Full backup every SUNDAY at 0300
# Incremental backup every other day at 0300
#
    if [ -n "$CROND" ]; then
	cat > /etc/cron.d/s9s_backup <<EOF
0 3 * * 0 root /usr/bin/s9s_backup --backup full $MYCNF
0 3 * * 1 root /usr/bin/s9s_backup --backup incremental $MYCNF
0 3 * * 2 root /usr/bin/s9s_backup --backup incremental $MYCNF
0 3 * * 3 root /usr/bin/s9s_backup --backup incremental $MYCNF
0 3 * * 4 root /usr/bin/s9s_backup --backup incremental $MYCNF
0 3 * * 5 root /usr/bin/s9s_backup --backup incremental $MYCNF
0 3 * * 6 root /usr/bin/s9s_backup --backup incremental $MYCNF
0 6 * * 0 root /usr/bin/s9s_backup --purge 

EOF
    fi
    log_job_message "Installation of xtrabackup completed" 0
    echo "Install complete"
}

function remote_cmd_nofail()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s'" "$desthost" "$xcommand"
   ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand "  >> $HOME/s9s_deploy.log 2>&1
   ret=$?
   printf "\033[32m[ok]\033[0m\n"
   return $ret
}

function remote_cmd()
{
   desthost=$1
   xcommand=$2
   MAX_RETRIES=1
   printf "%-4s: Executing '%s' " "$desthost" "$xcommand"
   retry=0
   while [ $retry -lt $MAX_RETRIES ]; 
   do
      x=`ssh -q $SSH_OPTS  $SSH_USER@$desthost "$SUDO $xcommand " 2>&1  >> $HOME/s9s_deploy.log`
      if [ $? -eq 0 ]; then
        printf "\033[32m[ok]\033[0m\n"
        return 0
      fi
      retry=`expr $retry + 1`
      printf "\033[31m[failed: retrying ${retry}/${MAX_RETRIES}]\033[0m\n"
      ssh -q $SSH_OPTS  $SSH_USER@$desthost " sync " 2>&1  >> $HOME/s9s_deploy.log
      sleep 1
   done
   
   printf "\033[31m[failed]\033[0m\n"
   echo $x
   echo 'The following command failed:'
   echo "ssh -q $SSH_OPTS $SSH_USER@$desthost \"$SUDO $xcommand \""
   echo 'Try running the command on the line above again, contact http://support.severalnines.com/ticket/new, attach the output from deploy.sh and the error from running the command to the Support issue.'
   exit 1
}


function checkenv
{
    echo "Checking environment:"
    if ! test -f $XTRABACKUP_BIN ; then
	echo "$XTRABACKUP_BIN not found."
	echo "Change XTRABACKUP_BIN in beginning of the scipt"
    fi
    echo "Found: $XTRABACKUP_BIN"
    if ! test -f $MYSQL_BIN; then
	if ! test -f $MYSQL_BIN2; then
	    echo "Could not find mysql client binary"
	    echo "Change MYSQL_BIN in beginning of the scipt"
	    exit 1
	fi
	MYSQL_BIN=$MYSQL_BIN2
    fi
    echo "Found: $MYSQL_BIN"
    echo "Environment check is ok!"
    exit 0
}

function check_mysql_client()
{
   if [ ! -f $MYSQL_BIN ]; then
        # Try normal locations:
        MYSQL_BIN="/usr/bin/mysql"
        if [ ! -f $MYSQL_BIN ]; then
             MYSQL_BIN="/usr/local/mysql/bin/mysql"
             if [ ! -f $MYSQL_BIN ]; then
		 echo "The MySQL client binary could not be found"         
		 if [ "$mysql_basedir" = "" ]; then
		     echo "mysql_basedir in /etc/cmon.cnf is not set. Add it to /etc/cmon.cnf"
		     exit 1		     
		 fi
             fi
	fi
   fi
}

ulimit -n $ULIMIT
ARG=$1
shift

case $ARG in
    --backup)
	start_backup $*
	;;
    --purge)
	purge_backups $*
	;;
    --list)
	list_backups  $*
	;;
    --install)
	install $*
	;;
    --restore)
	restore $*
	;;
    --checkenv)
	checkenv $*
	;;
    *)
    echo "Usage:"
    echo "bash ./s9s_backup <--backup|--list|--install|--purge|--restore|--checkenv> <options follows>"
    echo "e.g:"
    echo "./s9s_backup --backup"
    echo "will print out additional arguments needed"
    exit 1
    ;;
esac
