#!/bin/bash

# All files in this package is subject to the GPL v2 license
# More information is in the COPYING file in the top directory of this package.
# Copyright (C) 2011 Severalnines

# Also see CREDITS, many thanks to Timothy Der (randomorzero.wordpress.com)
# for the work on the makecfg, mysqlckk, and haproxy.cfg.tmpl
JOBID=0
HAPROXY_MYSQL_LISTEN_PORT="33306"
HAPROXY_LOG='/tmp/s9s_haproxy.log'
HAPROXY_LOG2='/tmp/s9s_haproxy2.log'
LOCKFILE="/tmp/s9s_haproxy.lock"
function init
{    
    FILES=`ls /etc/cmon.cnf 2>&1`
    FILES2=`ls /etc/cmon.d/*.cnf 2>&1`
    FILES="$FILES $FILES2"
    configfile=""
    for f in $FILES
    do
	X=`grep -l cluster_id=${CLUSTER_ID} $f 2>&1 `
	if [ $? -eq 0 ]; then
            source $f
            configfile=$f
        fi
    done

    if [ -z "$configfile" ]; then
        echo "No matching configuration file found having cluster_id=${CLUSTER_ID}"
        exit 1
    fi

    source $configfile;

    CMON_DB_HOST=127.0.0.1
    CMON_DB_PORT=$mysql_port
    CMON_USER=cmon
    CMON_DB_DB=cmon
    CMON_PASSWORD=$mysql_password
    MYSQL_BIN=$mysql_basedir/bin/mysql
    CONNECT_TIMEOUT=10
    CLUSTER_TYPE=$type
    MYSQL_OPTS="--connect-timeout=$CONNECT_TIMEOUT"
    OSUSER=$USER
    SUFFIX="1"
    PREFIX="s9s"
    HAPROXY_OPTS="-f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -st \$(cat /var/run/haproxy.pid)"
    LB_ADMIN_USER='admin'
    LB_ADMIN_PORT="9600"
    LB_ADMIN_PASSWORD='admin'

    if [ "$OSUSER" != "root" ]; then
	echo "must be executed as 'root' or with 'sudo'"
	exit 1
    fi
    echo "" > $HAPROXY_LOG
    echo "" > $HAPROXY_LOG2
}

function write_email
{
  SUBJECT=$1
  MSG=$2
  cat $HAPROXY_LOG | sed  -e "s/'/\\\'/g" -e 's/"/\\"/g' > /tmp/s9s_haproxy_log_escaped
  MSG="$MSG \nInstallation log follows:\n `cat /tmp/s9s_haproxy_log_escaped`" 
  QUERY="INSERT INTO simple_email(cid,subject,message) VALUES($CLUSTER_ID, \"$SUBJECT\", \"$MSG\")"
  $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
}



function log_job_message
{    
    MSG=$1
    EXIT_CODE=$2
    if [ -z "$EXIT_CODE" ]; then
	EXIT_CODE=1
    fi
    QUERY="INSERT INTO cmon_job_message(cid, jobid,message,exit_code,report_ts) VALUES($CLUSTER_ID,$JOBID,\"$MSG\",$EXIT_CODE,now())"
    if [ $JOBID -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
	if [ $EXIT_CODE -ne 0 ]; then
	    echo "Failed: $MSG" >> $HAPROXY_LOG
	else
	    echo "$MSG" >> $HAPROXY_LOG
	fi
    else
	if [ $EXIT_CODE -ne 0 ]; then
	    echo "Failed: $MSG" >> $HAPROXY_LOG
	    echo "Failed: $MSG"
	else
	    echo "$MSG" >> $HAPROXY_LOG
	    echo "$MSG" 
	fi
    fi
}

function log_message
{    
    MSG=$1
    echo "$MSG" >> $HAPROXY_LOG2
    echo "$MSG"
}

function log_job
{    
    STATUS=$1
    STATUS_TXT=$2
    EXIT_CODE=$3
    QUERY="UPDATE cmon_job SET status='$STATUS', status_txt='$STATUS_TXT', exit_code=$EXIT_CODE, report_ts=NOW()  WHERE cid=$CLUSTER_ID AND jobid=$JOBID"
    if [ $JOBID -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    fi
}

function write_haproxy_cfg_tmpl_rhel()
{
cat << 'EOF' > $TMPDIR/haproxy.cfg.tmpl.rhel
        global
        pidfile /var/run/haproxy.pid
        daemon
        user nobody
        group nobody
        stats socket /tmp/haproxy.socket user nobody group nobody mode 600 level admin
        node NODE_NAME
        description NODE_DESC

        #* Performance Tuning
        maxconn 40000
        spread-checks 3
        quiet
defaults
        #log    global
        mode    tcp
        option  dontlognull
        option tcp-smart-accept
        option tcp-smart-connect
        #option dontlog-normal
        retries 3
        option redispatch
        maxconn 40000
        timeout check   3500ms
        timeout queue   3500ms
        timeout connect 3500ms
        timeout client  10000ms
        timeout server  10000ms

userlist STATSUSERS
        group admin users admin
        user ADMIN_USER insecure-password ADMIN_PASSWORD
        user stats insecure-password PASSWORD

listen admin_page 0.0.0.0:ADMIN_PORT
        mode http
        stats enable
        stats refresh 60s
        stats uri /
        acl AuthOkay_ReadOnly http_auth(STATSUSERS)
        acl AuthOkay_Admin http_auth_group(STATSUSERS) admin
        stats http-request auth realm admin_page unless AuthOkay_ReadOnly
        #stats admin if AuthOkay_Admin
EOF
}



function write_keepalived_cfg_tmpl()
{
  echo "Writing $TMPDIR/keepalived.conf.$1"
cat << 'EOF' > $TMPDIR/keepalived.conf.$1
vrrp_script chk_haproxy {
   script "killall -0 haproxy"   # verify the pid existance
   interval 2                    # check every 2 seconds
   weight 2                      # add 2 points of prio if OK
}
 
vrrp_instance VI_1 {
   interface ETHINTERFACE                # interface to monitor
   state MASTER
   virtual_router_id 51          # Assign one ID for this route
   priority PRIORITY                   # 101 on master, 100 on backup
   virtual_ipaddress {
       VIRTUAL_IP                        # the virtual IP
   }
   track_script {
       chk_haproxy
   }
}
EOF

sed -i.bak "s#PRIORITY#$3#g" $TMPDIR/keepalived.conf.$1
sed -i.bak "s#VIRTUAL_IP#$2#g" $TMPDIR/keepalived.conf.$1
sed -i.bak "s#ETHINTERFACE#$4#g" $TMPDIR/keepalived.conf.$1
}	


function write_haproxy_cfg_tmpl_debian()
{
cat << 'EOF' > $TMPDIR/haproxy.cfg.tmpl.debian
global
        pidfile /var/run/haproxy.pid
        daemon
        user www-data
        group www-data
        stats socket /tmp/haproxy.socket user www-data group www-data mode 600 level admin
        node NODE_NAME
        description NODE_DESC

        #* Performance Tuning
        maxconn 40000
        spread-checks 3
        quiet
defaults
        #log    global
        mode    tcp
        option  dontlognull
        option tcp-smart-accept
        option tcp-smart-connect
        #option dontlog-normal
        retries 3
        option redispatch
        maxconn 40000
        timeout check   3500ms
        timeout queue   3500ms
        timeout connect 3500ms
        timeout client  120s
        timeout server  120s

userlist STATSUSERS
        group admin users admin
        user ADMIN_USER insecure-password ADMIN_PASSWORD
        user stats insecure-password USER_PASSWORD

listen admin_page 0.0.0.0:ADMIN_PORT
        mode http
        stats enable
        stats refresh 60s
        stats uri /
        acl AuthOkay_ReadOnly http_auth(STATSUSERS)
        acl AuthOkay_Admin http_auth_group(STATSUSERS) admin
        stats http-request auth realm admin_page unless AuthOkay_ReadOnly
        #stats admin if AuthOkay_Admin
EOF
}	

function write_makecfg
{
cat << 'EOF' > $TMPDIR/makecfg.sh
#!/bin/bash
DC="$1"
SUFFIX="$2"
TMPDIR2="$3"
MAX_CONNECTIONS="$4"
LISTEN_PORT="$5"
TEMPLATE=${TMPDIR2}/haproxy.cfg.tmpl
if ([ -z "$DC" ] || [ -z "$SUFFIX" ]); then
        echo "specify PREFIX and SUFFIX: $0 <prefix> <suffix>; ie: $0 dt 1"
        exit 1
fi
BACKENDFILES=`ls ${TMPDIR2}/${DC}_*.backend`
if [ -z "$BACKENDFILES" ]; then
    echo "No backend files found for $DC
                *.backend files are lists of hostnames in files with naming convention  <prefix>_<port>_<name>.backend
                example: dt_9906_production.backend
                the list in the file is seperated by whitespace and each server can optionally have ',disabled' 
                appended to its name to add it as disabled by default.
                example: slave1 slave2,disabled"
        exit 1
fi
NNAME="HAPROXY-$DC$SUFFIX"
NDESC="$NNAME"
if [ ! -f "$TEMPLATE" ]; then
        echo "$TEMPLATE not found!"
        exit 1;
fi
sed -e 's/NODE_NAME/'$NNAME'/g' -e 's/NODE_DESC/'$NDESC'/g'  "$TEMPLATE"
for I in $BACKENDFILES; do
        NAME=`basename $I .backend`
        PORT=`echo $NAME | awk -F'_' '{ print $2 }'`
        DESC=`echo $NAME | sed -e 's/^'$DC'_/'$DC$SUFFIX'_/'`
echo -n "
listen  "$DESC"
        bind *:$LISTEN_PORT
        mode tcp
        timeout client  60000ms
        timeout server  60000ms
        balance leastconn 
        option httpchk
        option allbackups
        default-server port 9200 inter 2s downinter 5s rise 3 fall 2 slowstart 60s maxconn $MAX_CONNECTIONS maxqueue 128 weight 100
	"
        for J in `cat $I`; do
                SPLIT=(`echo $J | tr ',' ' '`);
                STAT=""
                if [ "${SPLIT[1]}" == "disabled" ]; then
                        STAT="disabled"
                fi
                echo -e "        server ${SPLIT[0]} ${SPLIT[0]}:3306 check $STAT"
        done
done
exit
EOF
chmod u+x $TMPDIR/makecfg.sh
}

function write_mysqlchk_galera
{
cat << 'EOF' > $TMPDIR/mysqlchk.sh.galera
#!/bin/bash
#
# This script checks if a mysql server is healthy running on localhost. It will
# return:
# "HTTP/1.x 200 OK\r" (if mysql is running smoothly)
# - OR -
# "HTTP/1.x 500 Internal Server Error\r" (else)
#
# The purpose of this script is make haproxy capable of monitoring mysql properly
#

MYSQL_HOST="127.0.0.1"
MYSQL_PORT=$MYSQL_PORT
MYSQL_USERNAME="cmon"
MYSQL_PASSWORD=$CMON_PASSWORD
MYSQL_OPTS="-N -q -A"
TMP_FILE="/dev/shm/mysqlchk.$$.out"
ERR_FILE="/dev/shm/mysqlchk.$$.err"
FORCE_FAIL="/dev/shm/proxyoff"
MYSQL_BIN=$MYSQL_BIN
CHECK_QUERY="show global status where variable_name='wsrep_local_state'"
CHECK_QUERY2="show global variables where variable_name='wsrep_sst_method'"
preflight_check()
{
    for I in "$TMP_FILE" "$ERR_FILE"; do
        if [ -f "$I" ]; then
            if [ ! -w $I ]; then
                echo -e "HTTP/1.1 503 Service Unavailable\r\n"
                echo -e "Content-Type: Content-Type: text/plain\r\n"
                echo -e "\r\n"
                echo -e "Cannot write to $I\r\n"
                echo -e "\r\n"
                exit 1
            fi
        fi
    done
}
return_ok()
{
    echo -e "HTTP/1.1 200 OK\r\n"
    echo -e "Content-Type: text/html\r\n"
    echo -e "Content-Length: 43\r\n"
    echo -e "\r\n"
    echo -e "<html><body>MySQL is running.</body></html>\r\n"
    echo -e "\r\n"
    rm $ERR_FILE $TMP_FILE
    exit 0
}
return_fail()
{
    echo -e "HTTP/1.1 503 Service Unavailable\r\n"
    echo -e "Content-Type: text/html\r\n"
    echo -e "Content-Length: 42\r\n"
    echo -e "\r\n"
    echo -e "<html><body>MySQL is *down*.</body></html>\r\n"
    sed -e 's/\n$/\r\n/' $ERR_FILE
    echo -e "\r\n"
    rm $ERR_FILE $TMP_FILE
    exit 1
}
preflight_check
if [ -f "$FORCE_FAIL" ]; then
        echo "$FORCE_FAIL found" > $ERR_FILE
        return_fail;
fi
$MYSQL_BIN $MYSQL_OPTS --host=$MYSQL_HOST --port=$MYSQL_PORT --user=$MYSQL_USERNAME --password=$MYSQL_PASSWORD -e "$CHECK_QUERY" > $TMP_FILE 2> $ERR_FILE
if [ $? -ne 0 ]; then
        return_fail;
fi
status=`cat  $TMP_FILE | awk '{print $2;}'`

if [ $status -eq 2 ] || [ $status -eq 4 ] ; then
   if [ $status -eq 2 ]; then
     $MYSQL_BIN $MYSQL_OPTS --host=$MYSQL_HOST --port=$MYSQL_PORT --user=$MYSQL_USERNAME --password=$MYSQL_PASSWORD -e "$CHECK_QUERY2" > $TMP_FILE 2> $ERR_FILE
     if [ $? -ne 0 ]; then
        return_fail;
     fi
     method=`cat  $TMP_FILE | awk '{print $2;}'`
     if [ -z "$method" ] || [ "$method" = "rsync" ] || [ "$method" = "mysqldump" ]; then
         return_fail;
     fi
   fi
   return_ok;
fi

return_fail;

EOF

}

function test_ssh()
{
  tmphost=$1
  printf "Testing ssh to $tmphost: "
  ssh $SSH_OPTS -oNumberOfPasswordPrompts=0 $SSH_USER@$tmphost "$SUDO ls -al /usr " 2>&1 > /dev/null
  echo "ssh $SSH_OPTS -oNumberOfPasswordPrompts=0 $SSH_USER@$tmphost $SUDO ls -al /usr "
  if [ $? -eq 0 ]; then
        printf "[32m[ok][0m\n"
        return 0
  else
       printf "[32m[failed][0m\n"
       echo "ssh test failed, it was not possible to ssh without passwords or password-less sudo (for non-root users) failed"
       echo "You must setup shared keys and or verify IDENTITY in .s9s/config"
       echo "If you need help please read how to setup password less sudo and ssh:"
       echo "   http://support.severalnines.com/entries/20614858-server-requirements-on-premise-amis-other-images"
       echo "   or file a support request at:"
       echo "   http://support.severalnines.com/tickets/new"
       log_job_message 'Could not ssh to remote host, you must setup shared keys to the host first.' 1
       log_job 'FAILED' 's9s_haproxy failed' 1
       write_email 's9s_haproxy installation failed' 'Could not ssh to remote host, you must setup shared keys to the host first.' 
       exit 1
       return 1
  fi
}

function remote_cmd_getreply()
{
   desthost=$1
   xcommand=$2
   x=`ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand"`
   if [ $? -ne 0 ]; then
       echo ""
       return 0
   fi
   x=`echo $x | grep -v "password"`
   echo $x
}

function remote_copy()
{
   srcfile=$1
   desthost=$2
   destfile=$3
   printf "%-4s: Copying '%s' " "$desthost" "$srcfile"
   scp $SSH_OPTS2 $srcfile $SSH_USER@$desthost:$destfile > $HOME/s9s_cmd.log 2>&1
   if [ $? -eq 0 ]; then
      log_job_message "copying file $srcfile --> $desthost:$destfile" 0
      printf "\033[32m[ok]\033[0m\n"
      return 0
   else
      log_job_message "copying file $srcfile --> $desthost:$destfile" 1
      log_job 'FAILED' 's9s_haproxy failed' 1
      printf "\033[31m[failed]\033[0m\n"
      cat $HOME/s9s_cmd.log
      exit 1
   fi
}

function remote_cmdx()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s'" "$desthost" "$xcommand"
   ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand "
   if [ $? -eq 0 ]; then
      log_job_message "$desthost: Executed $xcommand" 0
      printf "\033[32m[ok]\033[0m\n"
      return 0
   else
      log_job_message "command failed: $xcommand" 1
      log_job 'FAILED' 's9s_haproxy failed' 1
      printf "\033[31m[failed]\033[0m\n"
      exit 1
   fi
}

function remote_cmd_nofail()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s'" "$desthost" "$xcommand"
   ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand "  >> $HOME/s9s_deploy.log 2>&1
   ret=$?
   log_job_message "$desthost: Executed $xcommand" 0
   printf "\033[32m[ok]\033[0m\n"
   return $ret
}

function remote_cmd()
{
   desthost=$1
   xcommand=$2
   MAX_RETRIES=3
   printf "%-4s: Executing '%s' " "$desthost" "$xcommand"
   retry=0
   while [ $retry -lt $MAX_RETRIES ]; 
   do
      x=`ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand " 2>&1  >> $HOME/s9s_deploy.log`
      if [ $? -eq 0 ]; then
	  log_job_message "$desthost: Executed $xcommand" 0
          printf "\033[32m[ok]\033[0m\n"
          return 0
      fi
      retry=`expr $retry + 1`
      printf "\033[31m[failed: retrying ${retry}/${MAX_RETRIES}]\033[0m\n"
      ssh -q $SSH_OPTS $SSH_USER@$desthost " sync " 2>&1  >> $HOME/s9s_deploy.log
      sleep 1
   done
   log_job_message "FAILED: ${desthost} : ${xcommand}" 1
   log_job 'FAILED' 's9s_haproxy failed' 1
   write_email 's9s_haproxy installation failed' "FAILED: ${desthost} : ${xcommand}"
   printf "\033[31m[failed]\033[0m\n"
   echo $x
   echo 'The following command failed:'
   echo "ssh -q $SSH_OPTS $SSH_USER@$desthost \"$SUDO $xcommand \""
   echo 'Try running the command on the line above again, contact http://support.severalnines.com/ticket/new, attach the output from deploy.sh and the error from running the command to the Support issue.'
   exit 1
}


function remote_cmd3()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s' " "$desthost" "$xcommand"
   ssh -nq -tt $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand "   >> $HOME/s9s_deploy.log 2>&1
   if [ $? -eq 0 ]; then
       log_job_message "$desthost: Executed $xcommand" 0
       printf "\033[32m[ok]\033[0m\n"
       return 0
   else
       printf "\033[31m[failed]\033[0m\n"
       log_job_message "FAILED: ${desthost}@${xcommand}" 1
       log_job 'FAILED' 's9s_haproxy failed' 1
       write_email 's9s_haproxy installation failed' "FAILED: ${desthost} : ${xcommand}"
       exit 1
   fi
}

function write_mysqlchk_mc
{
cat << 'EOF' > $TMPDIR/mysqlchk.sh.mysqlcluster
#!/bin/bash
#
# This script checks if a mysql server is healthy running on localhost. It will
# return:
# "HTTP/1.x 200 OK\r" (if mysql is running smoothly)
# - OR -
# "HTTP/1.x 500 Internal Server Error\r" (else)
#
# The purpose of this script is make haproxy capable of monitoring mysql properly
#

MYSQL_HOST="127.0.0.1"
MYSQL_PORT=$MYSQL_PORT
MYSQL_USERNAME="cmon"
MYSQL_PASSWORD=$CMON_PASSWORD
MYSQL_OPTS="-N -q -A"
TMP_FILE="/dev/shm/mysqlchk.$$.out"
ERR_FILE="/dev/shm/mysqlchk.$$.err"
FORCE_FAIL="/dev/shm/proxyoff"
MYSQL_BIN=$MYSQL_BIN
CHECK_QUERY="select 1"
preflight_check()
{
    for I in "$TMP_FILE" "$ERR_FILE"; do
        if [ -f "$I" ]; then
            if [ ! -w $I ]; then
                echo -e "HTTP/1.1 503 Service Unavailable\r\n"
                echo -e "Content-Type: Content-Type: text/plain\r\n"
                echo -e "\r\n"
                echo -e "Cannot write to $I\r\n"
                echo -e "\r\n"
                exit 1
            fi
        fi
    done
}
return_ok()
{
    echo -e "HTTP/1.1 200 OK\r\n"
    echo -e "Content-Type: text/html\r\n"
    echo -e "Content-Length: 43\r\n"
    echo -e "\r\n"
    echo -e "<html><body>MySQL is running.</body></html>\r\n"
    echo -e "\r\n"
    rm $ERR_FILE $TMP_FILE
    exit 0
}
return_fail()
{
    echo -e "HTTP/1.1 503 Service Unavailable\r\n"
    echo -e "Content-Type: text/html\r\n"
    echo -e "Content-Length: 42\r\n"
    echo -e "\r\n"
    echo -e "<html><body>MySQL is *down*.</body></html>\r\n"
    sed -e 's/\n$/\r\n/' $ERR_FILE
    echo -e "\r\n"
    rm $ERR_FILE $TMP_FILE
    exit 1
}
preflight_check
if [ -f "$FORCE_FAIL" ]; then
        echo "$FORCE_FAIL found" > $ERR_FILE
        return_fail;
fi
$MYSQL_BIN $MYSQL_OPTS --host=$MYSQL_HOST --port=$MYSQL_PORT --user=$MYSQL_USERNAME --password=$MYSQL_PASSWORD -e "$CHECK_QUERY" > $TMP_FILE 2> $ERR_FILE
if [ $? -ne 0 ]; then
        return_fail;
fi
return_ok;


EOF

}


function write_xinetd
{
cat << 'EOF' > $TMPDIR/xinetd_mysqlchk
# default: on
# description: mysqlchk
service mysqlchk
{
        flags           = REUSE
        socket_type     = stream
        port            = 9200
        wait            = no
        user            = nobody
        server          = /usr/local/bin/mysqlchk.sh
        log_on_failure  += USERID
        disable         = no
#        only_from       = 0.0.0.0/0
# recommended to put the IPs that need
# to connect exclusively (security purposes)
        per_source      = UNLIMITED
# Recently added (May 20, 2010)
# Prevents the system from complaining
# about having too many connections open from
# the same IP. More info:
# http://www.linuxfocus.org/English/November2000/article175.shtml
}
EOF
}

function load_opts
{
    local CLUSTER_ID=$1
    echo "load opts $CLUSTER_ID"
    OS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS' AND cid=$CLUSTER_ID"`
    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID"`
    MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_PORT' AND cid=$CLUSTER_ID"`
    GALERA_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='GALERA_PORT' AND cid=$CLUSTER_ID"`
    MYSQL_BASEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_BASEDIR' AND cid=$CLUSTER_ID"`
    MYSQL_SCRIPTDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SCRIPTDIR' AND cid=$CLUSTER_ID"`
    SSH_IDENTITY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_IDENTITY' AND cid=$CLUSTER_ID"`
    SSH_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_PORT' AND cid=$CLUSTER_ID"`
    SSH_USER=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_USER' AND cid=$CLUSTER_ID"`
    SSH_OPTSX=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_OPTS' AND cid=$CLUSTER_ID"`
    SUDO=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SUDO' AND cid=$CLUSTER_ID"`
    TMPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='STAGING_DIR' AND cid=$CLUSTER_ID"`
    HTTP_PROXY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='HTTP_PROXY' AND cid=$CLUSTER_ID"`
    S9S_TMPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='STAGING_DIR' AND cid=$CLUSTER_ID"`
    VENDOR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='VENDOR' AND cid=$CLUSTER_ID" 2>/dev/null`
    OS_USER_HOME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS_USER_HOME' AND cid=$CLUSTER_ID"`
    DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon.cluster_config where variable='datadir' and cid=$CLUSTER_ID order by id asc limit 1"`
    MYCNF=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select data  from cmon.cluster_configuration where cid=$CLUSTER_ID limit 1"`


    MONITORED_MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONITORED_MYSQL_PORT' AND cid=$CLUSTER_ID" 2>/dev/null`
    
    if [ "$MONITORED_MYSQL_PORT" = "" ] || [ "$MONITORED_MYSQL_PORT" = "NULL" ];  then
        MONITORED_MYSQL_PORT="3306"
    fi
    

    if [ $? -ne 0 ]; then 
        log_job_message "load opts failed"     
	log_job 'FAILED' 's9s_haproxy failed' 1  
        exit 1
    fi

    if [ -z "$MYSQL_PORT" ]; then 
        log_job_message "MYSQL_PORT not set in Cluster Settings"
	log_job 'FAILED' 's9s_haproxy failed' 1
	write_email 's9s_haproxy installation failed' "MYSQL_PORT not set in Cluster Settings"
        exit 1
    fi

    if [ -z "$SSH_USER" ]; then 
        log_job_message "SSH_USER not set in Cluster Settings"
	write_email 's9s_haproxy installation failed' "SSH_USER not set in Cluster Settings"
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi

    if [ "$S9S_TMPDIR" = "" ] || [ "$TMPDIR" = "NULL" ];  then
	S9S_TMPDIR="/tmp/"
	TMPDIR="/tmp/"
    fi
    
    if [ -z "$OS" ]; then 
        log_job_message "OS not set in Cluster Settings"
	write_email 's9s_haproxy installation failed' "OS not set in Cluster Settings"
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi


    SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10"
    if [ "$SSH_IDENTITY" = "" ]; then
	SSH_IDENTITY="-oIdentityFile=${OS_USER_HOME}/.ssh/id_rsa"
    else
	SSH_IDENTITY="-oIdentityFile=$SSH_IDENTITY"
    fi

    SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    SSH_OPTS2="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    if [ "$SSH_USER" != "root" ]; then
	#SSH_OPTS_EXTRA="-ft"
	if [ -z "$SSH_OPTSX" ] || [ "SSH_OPTSX" = "NULL" ]; then
	    SSH_OPTSX="-tt"
	fi

	SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
	if [ "$SUDO" = "" ] || [ "$SUDO" = "NULL" ];  then
           SUDO="sudo"
        fi
    fi
    if [ "$SSH_PORT" = "" ] || [ "$SSH_PORT" = "NULL" ];  then
        SSH_PORT="22"
    fi

    if [ "$S9S_TMPDIR" = "" ] || [ "$TMPDIR" = "NULL" ];  then
	S9S_TMPDIR="/tmp/"
	TMPDIR="/tmp/"
    fi
    SSH_OPTS="$SSH_OPTSX -p$SSH_PORT $SSH_OPTS"    
    SSH_OPTS2="-P$SSH_PORT $SSH_OPTS2"    



    mkdir -p $TMPDIR/haproxy
    TMPDIR=$TMPDIR/haproxy
    chown -R $SSH_USER:$SSH_USER $TMPDIR
}

function install_mysqlchk
{
    local h=$1
    #if [ "$h" = "$LB_HOSTNAME" ]; then
    #    echo "ERROR: Trying to install HaProxy on a database host is prohibited."
    #    exit 1
    #fi
    remote_cmd $h "mkdir -p $S9S_TMPDIR"
    remote_cmd $h "chown -R $SSH_USER:$SSH_USER $S9S_TMPDIR"
    remote_cmd_nofail $h  "sed  -i  '/mysqlchk        9200\/tcp/d' /etc/services"
    remote_cmd $h "sh -c 'echo \"mysqlchk        9200/tcp\" >> /etc/services'"
    remote_copy $TMPDIR/mysqlchk.sh $h $S9S_TMPDIR
    remote_cmd $h "mv -f $S9S_TMPDIR/mysqlchk.sh /usr/local/bin"
    remote_cmd $h "chmod 777 /usr/local/bin/mysqlchk.sh"
    remote_cmd $h "$PKG_MGR xinetd"
    remote_copy $TMPDIR/xinetd_mysqlchk $h $S9S_TMPDIR
    remote_cmd $h "mv -f $S9S_TMPDIR/xinetd_mysqlchk /etc/xinetd.d/mysqlchk"
    remote_cmd $h "/etc/init.d/xinetd restart"
    if [ $OS = "rhel" ] || [ $OS = "centos6" ]; then
        remote_cmd $h "/sbin/chkconfig --add xinetd"
    else
        remote_cmd $h "/usr/sbin/update-rc.d xinetd defaults"
    fi
    echo $h >>  $TMPDIR/${PREFIX}_${HAPROXY_MYSQL_LISTEN_PORT}_${LB_NAME}.backend
    echo "Configuration of $h completed "
}

function prepare
{
    write_haproxy_cfg_tmpl_rhel
    write_haproxy_cfg_tmpl_debian
    write_makecfg
    write_mysqlchk_galera
    write_mysqlchk_mc
    write_xinetd

}

function prepare_keepalived
{
    write_keepalived_cfg_tmpl $1 $2 $3 $4
}

function install_haproxy
{
    LB_NODE_LIST=""
    MAX_CONNECTIONS=256
    args=`getopt p:P:h:i:N:n:j:sf:c: $*`
    set -- $args
    for i
    do
        case "$i" in
            -p)
                CMON_PASSWORD="$2"; shift;
                shift;;
            -P)
		HAPROXY_MYSQL_LISTEN_PORT="$2"; shift;
                shift;;
            -i)
                CLUSTER_ID="$2"; shift;
                shift;;
            -h)
                LB_HOSTNAME="$2"; shift;
                shift;;
            -n)
                LB_NODE_LIST="$2"; shift;
                shift;;
            -s)
                BUILD_FROM_SRC="1"; 
                shift;;
            -f)
                BUILD_FROM_SRC_FILE="$2"; 
                shift;;
            -c)
                MAX_CONNECTIONS="$2"; shift;
                shift;;
            -N)
                LB_NAME="$2"; shift;
                shift;;
            -j)
                JOBID="$2"; shift;
                shift;;
            --)
                shift; break;;
                esac
    done

    if [ -z "$CLUSTER_ID" ]; then
	log_job_message "s9s_haproxy --install -i <cluster id>  missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
      exit 1
    fi
    
    init
    log_job 'RUNNING' 'Job is running' 0

    if [ -z "$CMON_PASSWORD" ]; then
      log_job_message  "s9s_haproxy --install -p  missing" 1
      log_job 'FAILED' 's9s_haproxy failed' 1
      exit 1
    fi

    if [ -z "$LB_HOSTNAME" ]; then
        log_job_message "s9s_haproxy --install  -h <loadbalancer host> missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi
    
    if [ -z "$LB_NAME" ]; then
        LB_NAME="production"
    fi
    sleep 1
    log_job 'RUNNING' 'Job is running' 0
    sleep 2
    log_job 'RUNNING' 'Job is running' 0
    load_opts $CLUSTER_ID
    log_job 'RUNNING' 'Job is running' 0
    test_ssh $LB_HOSTNAME
    sleep 2
    log_job 'RUNNING' 'Job is running' 0
    rm -rf $TMPDIR/haproxy/*
    prepare
    chown -R $SSH_USER:$SSH_USER $TMPDIR
    sleep 1
    log_job 'RUNNING' 'Job is running' 0
    case $OS in
        rhel|redhat)
            PKG_MGR="$HTTP_PROXY yum -y install "
            PKG_MGR2="$HTTP_PROXY yum -y install "
            PHP_CURL="php-curl"
	    BUILD="gcc make"
            CENTOS=$(remote_cmd_getreply $LB_HOSTNAME 'test -f /etc/redhat-release && cat /etc/redhat-release  | grep -i centos |grep "6"')
	    if [ -z "$BUILD_FROM_SRC" ]; then
		if [ -n "$CENTOS" ]; then
		    remote_cmd $LB_HOSTNAME "$HTTP_PROXY yum -y install centos-release-cr"
		    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  's/enabled=1/enabled=0/g' /etc/yum.repos.d/CentOS-CR.repo"
		    PKG_MGR2="$HTTP_PROXY yum -y --enablerepo=cr install "
		    PKG_MGR="$HTTP_PROXY yum -y install "
		    PHP_CURL="php-curl"
		    OS=rhel
		else
		    OS=rhel
		fi
	    fi
	    OS=rhel		
            ;;
	centos6)
	    BUILD="gcc make"
	    OS=centos6
	    ;;	
        debian|ubuntu)
            PKG_MGR="$HTTP_PROXY apt-get install -y "
            PKG_MGR2="$HTTP_PROXY apt-get install -y "
            PHP_CURL="php5-curl"
	    BUILD="gcc make"
            OS=debian
            ;;
        *)
            echo "OS $OS not supported"
	    log_job 'FAILED' 's9s_haproxy failed' 1
            exit
            ;;
    esac
    log_job 'RUNNING' 'Job is running' 0
    case $CLUSTER_TYPE in
        galera)
        ;;
        mysqlcluster)
        ;;
        *)
        log_job_message "Cluster $CLUSTER_TYPE not supported" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
	exit 1
        ;;
    esac
    log_job 'RUNNING' 'Job is running' 0
    if [ -n "$LB_NODE_LIST" ]; then
	LB_NODE_LIST2=`echo $LB_NODE_LIST | sed "s/,/','/g" | sed "s/^/'/g" | sed "s/$/'/g" |sed "s/[ ]*//g"`
	
	QUERY="select group_concat(h.hostname SEPARATOR ' ') from mysql_server m, hosts h WHERE m.id=h.id and h.cid=m.cid and h.ping_status>0 and (h.hostname in ($LB_NODE_LIST2) or h.ip in ($LB_NODE_LIST2)  ) and connected=1 and m.cid=$CLUSTER_ID"
        hostnames=`$MYSQL_BIN -B -N --host=$CMON_DB_HOST --port=$CMON_DB_PORT --user=cmon --password=$CMON_PASSWORD --database=$CMON_DB_DB -e "$QUERY"`
    else
	QUERY="select group_concat(h.hostname SEPARATOR ' ') from mysql_server m, hosts h WHERE m.id=h.id and h.cid=m.cid and h.ping_status>0 and connected=1 and m.cid=$CLUSTER_ID"
        hostnames=`$MYSQL_BIN -B -N --host=$CMON_DB_HOST --port=$CMON_DB_PORT --user=cmon --password=$CMON_PASSWORD --database=$CMON_DB_DB -e "$QUERY"`
    fi

    if [ -z "$hostnames" ]; then
       echo "No hostnames found."
       log_job 'FAILED' 's9s_haproxy failed' 1
       write_email 's9s_haproxy installation failed' "No hostnames found"
       exit 1
    fi

    if [ "$hostnames" = "NULL" ]; then
       write_email 's9s_haproxy installation failed' "No hostnames found"
       log_job 'FAILED' 's9s_haproxy failed' 1
       exit 1
    fi
    log_job 'RUNNING' 'Job is running' 0
    echo "cp $TMPDIR/haproxy.cfg.tmpl.$OS $TMPDIR/haproxy.cfg.tmpl"
    cp $TMPDIR/haproxy.cfg.tmpl.$OS $TMPDIR/haproxy.cfg.tmpl
    sed -i "s#ADMIN_PASSWORD#$LB_ADMIN_PASSWORD#g" $TMPDIR/haproxy.cfg.tmpl
    sed -i "s#ADMIN_USER#$LB_ADMIN_USER#g" $TMPDIR/haproxy.cfg.tmpl
    sed -i "s#ADMIN_PORT#$LB_ADMIN_PORT#g" $TMPDIR/haproxy.cfg.tmpl
    cp $TMPDIR/mysqlchk.sh.$CLUSTER_TYPE $TMPDIR/mysqlchk.sh 


    rm -rf  $TMPDIR/${PREFIX}*.backend  2>/dev/null
    sed -i "s#MYSQL_PASSWORD=.*#MYSQL_PASSWORD='$CMON_PASSWORD'#g" $TMPDIR/mysqlchk.sh
    sed -i "s#MYSQL_USERNAME=.*#MYSQL_USERNAME=\"cmon\"#g" $TMPDIR/mysqlchk.sh
    sed -i "s#MYSQL_PORT=.*#MYSQL_PORT=\"$MONITORED_MYSQL_PORT\"#g" $TMPDIR/mysqlchk.sh
    sed -i "s#MYSQL_BIN=.*#MYSQL_BIN=\"$MYSQL_BIN\"#g" $TMPDIR/mysqlchk.sh

    
    echo "*************************************************************"
    echo "* installing xinetd, mysqlchk.sh, and creating backend file *"
    echo "*************************************************************"
    for h in $hostnames
    do
	    log_job 'RUNNING' 'Job is running' 0
	    install_mysqlchk $h
    done
    log_job 'RUNNING' 'Job is running' 0
    
    echo ""
    echo "*************************************************************"
    echo "* writing $TMPDIR/${PREFIX}_${LB_NAME}_haproxy.cfg               *"
    echo "*************************************************************"
    $TMPDIR/makecfg.sh $PREFIX $SUFFIX $TMPDIR $MAX_CONNECTIONS $HAPROXY_MYSQL_LISTEN_PORT > $TMPDIR/${PREFIX}_${LB_NAME}_haproxy.cfg
    echo ""
    echo "*************************************************************"
    echo "* Installing haproxy on $LB_HOSTNAME                            *"
    echo "*************************************************************"

    remote_cmd $LB_HOSTNAME "$PKG_MGR wget"

    if [ $OS = "rhel" ] || [ $OS = "centos6" ]; then
	remote_cmd_nofail $LB_HOSTNAME "$HTTP_PROXY wget  --no-check-certificate --tries=5 -O ${S9S_TMPDIR}/epel-release-6-8.noarch.rpm  http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm"
	remote_cmd_nofail $LB_HOSTNAME "rpm -Uvh --force ${S9S_TMPDIR}/epel-release-6-8.noarch.rpm"
	remote_cmd_nofail $LB_HOSTNAME "rpm --import http://apt.sw.be/RPM-GPG-KEY.dag.txt"
	remote_cmd_nofail $LB_HOSTNAME "$HTTP_PROXY rpm -Uvh http://packages.sw.be/rpmforge-release/rpmforge-release-0.5.2-2.el6.rf.x86_64.rpm"
    fi
    remote_cmd_nofail $LB_HOSTNAME "$PKG_MGR $PHP_CURL"
    if [ -n "$BUILD_FROM_SRC" ]; then
	HAPROXY_SRC="haproxy-1.4.23.tar.gz"
	if [ -n "$BUILD_FROM_SRC_FILE" ]; then
	    if [ ! -f $BUILD_FROM_SRC_FILE ]; then
		echo "$BUILD_FROM_SRC_FILE could not be found"
		log_job 'FAILED' 's9s_haproxy failed' 1
		write_email 's9s_haproxy installation failed' "$BUILD_FROM_SRC_FILE could not be found"
		exit 1
	    fi
	    HAPROXY_SRC=`basename $BUILD_FROM_SRC_FILE`
	    remote_copy $BUILD_FROM_SRC_FILE $LB_HOSTNAME /tmp/
	else
	    remote_cmd $LB_HOSTNAME "sh -c '$HTTP_PROXY wget -O/tmp/${HAPROXY_SRC}  http://haproxy.1wt.eu/download/1.4/src/${HAPROXY_SRC}'"	 	
	    
	fi
	remote_cmd $LB_HOSTNAME "$PKG_MGR $BUILD"
	remote_cmd $LB_HOSTNAME "tar xvfz /tmp/${HAPROXY_SRC} -C /tmp/"
	remote_cmd_nofail $LB_HOSTNAME "killall -q  -9 haproxy"	
	remote_cmd $LB_HOSTNAME "sh -c 'cd /tmp/`basename ${HAPROXY_SRC} .tar.gz` && make TARGET=linux26 && cp haproxy /usr/sbin/'"
    else
	remote_cmd $LB_HOSTNAME "$PKG_MGR2 haproxy"
    fi
    remote_copy $TMPDIR/${PREFIX}_${LB_NAME}_haproxy.cfg $LB_HOSTNAME /tmp
    remote_cmd_nofail $LB_HOSTNAME "mkdir -p /etc/haproxy/"
    remote_cmd $LB_HOSTNAME "rm -f /etc/haproxy/haproxy.cfg"
    remote_cmd $LB_HOSTNAME "mv -f /tmp/${PREFIX}_${LB_NAME}_haproxy.cfg /etc/haproxy/haproxy.cfg"
    #remote_cmd $LB_HOSTNAME "/usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid -st \$(cat /var/run/haproxy.pid"                                                                                         
    remote_cmd $LB_HOSTNAME "/usr/sbin/haproxy ${HAPROXY_OPTS}"
    if [ -z "$BUILD_FROM_SRC" ]; then
	echo ""
	echo "** Adding init.d/haproxy auto start**"
	echo ""
	
	if [ $OS = "rhel" ] || [ $OS = "centos6"  ] ; then
            remote_cmd $LB_HOSTNAME "/sbin/chkconfig --add haproxy"
            remote_cmd $LB_HOSTNAME "/sbin/chkconfig haproxy on"
	else
            remote_cmd $LB_HOSTNAME "sed -i 's#ENABLED=.*#ENABLED=1#g' /etc/default/haproxy"
            remote_cmd $LB_HOSTNAME "/usr/sbin/update-rc.d haproxy defaults"
	fi
    fi
    echo ""
    echo "** Tuning Network **"
    echo ""
    log_job 'RUNNING' 'Job is running' 0
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.ip_nonlocal_bind.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.ip_nonlocal_bind=1\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.tcp_tw_reuse.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.tcp_tw_reuse=1\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.ip_local_port_range.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.ip_local_port_range = 1024 65023\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.tcp_max_syn_backlog.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.tcp_max_syn_backlog=40000\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.tcp_max_tw_buckets.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.tcp_max_tw_buckets=400000\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.tcp_max_orphans.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.tcp_max_orphans=60000\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.tcp_synack_retries.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.tcp_synack_retries=3\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.core.somaxconn.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.core.somaxconn=40000\" >> /etc/sysctl.conf'"
    remote_cmd_nofail $LB_HOSTNAME "sed  -i.bak  '/net.ipv4.tcp_fin_timeout.*/d' /etc/sysctl.conf"
    remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.tcp_fin_timeout = 5\" >> /etc/sysctl.conf'"
    
    QUERY="select count(column_name) from information_schema.columns where table_schema='cmon' and table_name='haproxy_server' and column_name='server_addr'"
    CNT=`$MYSQL_BIN -A -B -N  -ucmon -p$CMON_PASSWORD --host=$CMON_DB_HOST --database=$CMON_DB_DB --port=${CMON_DB_PORT} -e "$QUERY" 2>&1`
    if [ $CNT -eq 0 ]; then
        QUERY="ALTER TABLE $CMON_DB.haproxy_server ADD COLUMN server_addr VARCHAR(255) DEFAULT ''"
        $bindir/mysql -B -N  -ucmon -p$CMON_PASSWORD --database=$CMON_DB_DB -h$CMON_DB_HOST -P${CMON_DB_PORT} -e "$QUERY" 2>&1 >/tmp/err.log
        if [  $? -ne 0 ]; then
            log_job_message "Query failed: $QUERY"
            echo ""
            cat /tmp/err.log           
	    log_job 'FAILED' 's9s_haproxy failed' 1
	    write_email 's9s_haproxy installation failed' "Query failed: $QUERY"
	    exit 1
        fi
    fi

    QUERY="REPLACE INTO haproxy_server(cid, lb_host,lb_name,lb_port,lb_admin,lb_password,server_addr) VALUES ($CLUSTER_ID, '$LB_HOSTNAME','$LB_NAME', '$LB_ADMIN_PORT', '$LB_ADMIN_USER', '$LB_ADMIN_PASSWORD', '$LB_HOSTNAME')"
    $MYSQL_BIN --host=$CMON_DB_HOST --port=$CMON_DB_PORT --user=cmon --password=$CMON_PASSWORD --database=$CMON_DB_DB -e "$QUERY"



    QUERY="REPLACE INTO ext_proc (cid, hostname,bin, opts,cmd, proc_name, port) VALUES($CLUSTER_ID, '$LB_HOSTNAME','/usr/sbin/haproxy', \"${HAPROXY_OPTS}\", \"/usr/sbin/haproxy ${HAPROXY_OPTS}\",'haproxy', $LB_ADMIN_PORT)"
    $MYSQL_BIN -B -N  -ucmon -p$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST  --port=${CMON_DB_PORT} -e "$QUERY" 2>&1 >/tmp/err.log
    if [  $? -ne 0 ]; then
	log_job_message "Query failed: $QUERY" 1
	echo ""
	cat /tmp/err.log
	log_job 'FAILED' 's9s_haproxy failed' 1
	write_email 's9s_haproxy installation failed' "Query failed: $QUERY"
	exit 1
    fi


    log_message ""  
    log_message  "** Reboot is recommended of $LB_HOSTNAME for network settings to take effect! **"
    log_message  ""
    log_message  "FIREWALL: To access HAproxy within ClusterControl, you should allow the clustercontrol server to connect to $LB_HOSTNAME on port ${LB_ADMIN_PORT}, and HAProxy must be able to communicate on port 9200 and 3306 with the MySQL Servers."
    log_message  ""
    log_job_message  "**The admin interface is on http://${LB_HOSTNAME}:${LB_ADMIN_PORT} **" 0
    log_job_message  "  Login with ${LB_ADMIN_USER}/${LB_ADMIN_PASSWORD}" 0
    log_message  ""
    log_job_message  "** Connect from applications: " 0
    log_job_message  "  HAProxy listens on ${LB_HOSTNAME}:${HAPROXY_MYSQL_LISTEN_PORT} for mysql connections." 0
    log_message  ""
    log_job_message  " ** Don't forget to GRANT ${LB_HOSTNAME} on your mysql servers:" 0
    log_job_message  "  GRANT <privs> ON <db>.* TO '<user>'@'${LB_HOSTNAME}' IDENTIFIED BY '<password>'" 0
    log_message  ""
    
    cat $HAPROXY_LOG2  > /tmp/s9s_result
    cat $HAPROXY_LOG  >> /tmp/s9s_result
    mv -f /tmp/s9s_result  $HAPROXY_LOG 
    rm $HAPROXY_LOG2
    
    log_job 'FINISHED' 's9s_haproxy --install completed' 0
    write_email 's9s_haproxy installation completed' "s9s_haproxy --install completed"
    exit 0

}


function install_keepalived
{
    LB_NODE_LIST=""
    KEEPALIVED_NAME="keepalived"
    args=`getopt p:P:h:i:N:n:j:sf:c:x:y:v:I: $*`
    set -- $args
    for i
    do
        case "$i" in
            -p)
                CMON_PASSWORD="$2"; shift;
                shift;;
            -v)
		VIRTUAL_IP="$2"; shift;
                shift;;
            -i)
                CLUSTER_ID="$2"; shift;
                shift;;
            -I)
                ETHINTERFACE="$2"; shift;
                shift;;
            -x)
                LB_HOSTNAME1="$2"; shift;
                shift;;
            -y)
                LB_HOSTNAME2="$2"; shift;
                shift;;
            -N)
                KEEPALIVED_NAME="$2"; shift;
                shift;;
            -j)
                JOBID="$2"; shift;
                shift;;
            --)
                shift; break;;
                esac
    done

    if [ -z "$CLUSTER_ID" ]; then
	log_job_message "s9s_haproxy --install-ka -i <cluster id>  missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
	exit 1
    fi
    init

    
    if [ -z "$CMON_PASSWORD" ]; then
      log_job_message  "s9s_haproxy --install-ka -p  missing" 1
      log_job 'FAILED' 's9s_haproxy failed' 1
      exit 1
    fi

    if [ -z "$ETHINTERFACE" ]; then
	ETHINTERFACE="eth0"
    fi

    if [ -z "$LB_HOSTNAME1" ]; then
        log_job_message "s9s_haproxy --install-ka  -x <loadbalancer host> missing" 1
	write_email 's9s_haproxy keepalived installation failed' "s9s_haproxy --install-ka  -x <loadbalancer host> missing" 
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi

    if [ -z "$LB_HOSTNAME2" ]; then
	write_email 's9s_haproxy keepalived installation failed' "s9s_haproxy --install-ka  -y <loadbalancer host> missing" 
        log_job_message "s9s_haproxy --install-ka  -y <loadbalancer host> missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi

    if [ -z "$VIRTUAL_IP" ]; then
	write_email 's9s_haproxy keepalived installation failed' "s9s_haproxy --install-ka  -v <virtual ip> missing" 
        log_job_message "s9s_haproxy --install-ka  -v <virtual ip> missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi
    

    if [ -z "$JOBID" ]; then
	JOBID=0
    fi


    log_job 'RUNNING' 'Job is running' 1
    load_opts $CLUSTER_ID
    prepare_keepalived $LB_HOSTNAME1 $VIRTUAL_IP 101 $ETHINTERFACE
    prepare_keepalived $LB_HOSTNAME2 $VIRTUAL_IP 100 $ETHINTERFACE
    case $OS in
        rhel|redhat)
            PKG_MGR="$HTTP_PROXY yum -y install "
            PHP_CURL="php-curl"
	    OS=rhel
            ;;
	centos6)
	    OS=centos6
	    ;;	
        debian|ubuntu)
            PKG_MGR="$HTTP_PROXY apt-get install -y "	    
            OS=debian
            ;;
        *)
        echo "OS $OS not supported"
        exit
        ;;
    esac

    case $CLUSTER_TYPE in
        galera)
            ;;
        mysqlcluster)
            ;;
        *)
            log_job_message "Cluster $CLUSTER_TYPE not supported" 1
	    log_job 'FAILED' 's9s_haproxy failed' 1
	    exit 1
            ;;
    esac

    for LB_HOSTNAME in $LB_HOSTNAME1 $LB_HOSTNAME2
    do    
	echo "Installing keepalived on $LB_HOSTNAME"
	remote_cmd_nofail $LB_HOSTNAME "killall -q -15 keepalived"
	remote_cmd $LB_HOSTNAME "$PKG_MGR keepalived"
	remote_copy $TMPDIR/keepalived.conf.$LB_HOSTNAME $LB_HOSTNAME /tmp/
	remote_cmd $LB_HOSTNAME "mkdir -p /etc/keepalived/"
	remote_cmd $LB_HOSTNAME "rm -f /etc/keepalived/keepalived.conf"
	remote_cmd $LB_HOSTNAME "mv -f /tmp/keepalived.conf.$LB_HOSTNAME /etc/keepalived/keepalived.conf"
	echo ""
	echo "** Adding init.d/keepalived auto start**"
	echo ""
	
	if [ $OS = "rhel" ] || [ $OS = "centos6"  ] ; then
            remote_cmd $LB_HOSTNAME "/sbin/chkconfig --add keepalived"
            remote_cmd $LB_HOSTNAME "/sbin/chkconfig keepalived on"
	else
            remote_cmd $LB_HOSTNAME "/usr/sbin/update-rc.d keepalived defaults"
	fi
	echo ""
	echo "** Tuning Network **"
	echo ""    
	remote_cmd $LB_HOSTNAME  "/bin/sh -c 'echo \"net.ipv4.ip_nonlocal_bind=1\" >> /etc/sysctl.conf'"
	remote_cmd $LB_HOSTNAME  "/sbin/sysctl -e -p"    
	

	QUERY="REPLACE INTO keepalived(cid, keepalived_addr, virtual_ip ,haproxy_addr1,haproxy_addr2, name) VALUES ($CLUSTER_ID, '$LB_HOSTNAME','$VIRTUAL_IP','$LB_HOSTNAME1','$LB_HOSTNAME2','$KEEPALIVED_NAME')"
	$MYSQL_BIN --host=$CMON_DB_HOST --port=$CMON_DB_PORT --user=cmon --password=$CMON_PASSWORD --database=$CMON_DB_DB -e "$QUERY"
	if [  $? -ne 0 ]; then
	    log_job_message "Query failed: $QUERY"
	    echo ""
	    cat /tmp/err.log
	    write_email 's9s_haproxy keepalived installation failed' "s9s_haproxy --install-ka query failed: $QUERY"
	    log_job 'FAILED' 's9s_haproxy --install-ka failed' 1
	    exit 1
	fi

	QUERY="REPLACE INTO ext_proc (cid, hostname,bin, opts,cmd, proc_name, port) VALUES($CLUSTER_ID, '$LB_HOSTNAME','/usr/sbin/keepalived', \" \", \"nohup nice /usr/sbin/keepalived\",'keepalived', 0)"
	$MYSQL_BIN -B -N  -ucmon -p$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST  --port=${CMON_DB_PORT} -e "$QUERY" 2>&1 >/tmp/err.log
	if [  $? -ne 0 ]; then
	    log_job_message "Query failed: $QUERY"
	    echo ""
	    cat /tmp/err.log
	    log_job 'FAILED' 's9s_haproxy --install-ka failed' 1
	    exit 1
	fi
    done
    for LB_HOSTNAME in $LB_HOSTNAME1 $LB_HOSTNAME2
    do        
	echo "** starting keepalived on $LB_HOSTNAME **"
	remote_cmd $LB_HOSTNAME "nohup nice /usr/sbin/keepalived"
    done
    log_job_message "Access your cluster on $VIRTUAL_IP and the listening port ${HAPROXY_MYSQL_LISTEN_PORT} " 0
    log_job 'FINISHED' 's9s_haproxy --install-ka completed' 0
    write_email 's9s_haproxy keepalived installation completed' "Access your cluster on $VIRTUAL_IP and the listening port of HAProxy (${HAPROXY_MYSQL_LISTEN_PORT} default)" 
    exit 0

}




function delete_keepalived
{
    LB_NODE_LIST=""
    KEEPALIVED_NAME="keepalived"
    args=`getopt p:P:h:i:N:n:j:sf:c:x:y:v: $*`
    set -- $args
    for i
    do
        case "$i" in
            -p)
                CMON_PASSWORD="$2"; shift;
                shift;;
            -v)
		VIRTUAL_IP="$2"; shift;
                shift;;
            -i)
                CLUSTER_ID="$2"; shift;
                shift;;
            -x)
                LB_HOSTNAME1="$2"; shift;
                shift;;
            -y)
                LB_HOSTNAME2="$2"; shift;
                shift;;
            -N)
                KEEPALIVED_NAME="$2"; shift;
                shift;;
            -j)
                JOBID="$2"; shift;
                shift;;
            --)
                shift; break;;
                esac
    done
    if [ -z "$CLUSTER_ID" ]; then
	log_job_message "s9s_haproxy --delete-ka -i <cluster id>  missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
      exit 1
    fi
    init
    
    if [ -z "$CMON_PASSWORD" ]; then
      log_job_message  "s9s_haproxy --delete-ka -p  missing" 1
      log_job 'FAILED' 's9s_haproxy failed' 1
      exit 1
    fi

    if [ -z "$LB_HOSTNAME1" ]; then
        log_job_message "s9s_haproxy --delete-ka  -x <loadbalancer host> missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi

    if [ -z "$LB_HOSTNAME2" ]; then
        log_job_message "s9s_haproxy --delete-ka  -y <loadbalancer host> missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi

    if [ -z "$VIRTUAL_IP" ]; then
        log_job_message "s9s_haproxy --delete-ka  -v <virtual ip> missing" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
        exit 1
    fi
    

    if [ -z "$JOBID" ]; then
	JOBID=0
    fi


    log_job 'RUNNING' 'Job is running' 1
    load_opts $CLUSTER_ID
    case $OS in
        rhel|redhat)
            PKG_MGR="$HTTP_PROXY yum -y remove "
            PHP_CURL="php-curl"
	    OS=rhel
            ;;
	centos6)
	    OS=centos6
	    ;;	
        debian|ubuntu)
            PKG_MGR="$HTTP_PROXY apt-get remove -y "	    
            OS=debian
            ;;
        *)
        echo "OS $OS not supported"
        exit
        ;;
    esac

    case $CLUSTER_TYPE in
        galera)
        ;;
        mysqlcluster)
        ;;
        *)
        log_job_message "Cluster $CLUSTER_TYPE not supported" 1
	log_job 'FAILED' 's9s_haproxy failed' 1
	exit 1
        ;;
    esac

    for LB_HOSTNAME in $LB_HOSTNAME1 $LB_HOSTNAME2
    do    
	echo "Removing keepalived on $LB_HOSTNAME"
	remote_cmd_nofail $LB_HOSTNAME "killall -q -15 keepalived"
	remote_cmd_nofail $LB_HOSTNAME "$PKG_MGR keepalived"
	remote_cmd_nofail $LB_HOSTNAME "rm -f /etc/keepalived/keepalived.conf"
	
	if [ $OS = "rhel" ] || [ $OS = "centos6"  ] ; then
            remote_cmd_nofail $LB_HOSTNAME "/sbin/chkconfig --del keepalived"
            remote_cmd_nofail $LB_HOSTNAME "/sbin/chkconfig keepalived off"
	else
            remote_cmd_nofail $LB_HOSTNAME "/usr/sbin/update-rc.d -f keepalived remove"
	fi
	
	QUERY="DELETE FROM keepalived WHERE cid=${CLUSTER_ID} AND virtual_ip='$VIRTUAL_IP' AND name='$KEEPALIVED_NAME'"
	$MYSQL_BIN --host=$CMON_DB_HOST --port=$CMON_DB_PORT --user=cmon --password=$CMON_PASSWORD --database=$CMON_DB_DB -e "$QUERY"
	if [  $? -ne 0 ]; then
	    log_job_message "Query failed: $QUERY"
	    echo ""
	    cat /tmp/err.log
	    log_job 'FAILED' 's9s_haproxy --delete-ka failed' 1
	    exit 1
	fi

	QUERY="DELETE FROM ext_proc WHERE cid=${cluster_id} AND hostname='$LB_HOSTNAME' AND proc_name='keepalived'"
	$MYSQL_BIN -B -N  -ucmon -p$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST  --port=${CMON_DB_PORT} -e "$QUERY" 2>&1 >/tmp/err.log
	if [  $? -ne 0 ]; then
	    log_job_message "Query failed: $QUERY" 1
	    echo ""
	    cat /tmp/err.log
	    log_job 'FAILED' 's9s_haproxy --delete-ka failed' 1
	    exit 1
	fi
    done
    log_job 'FINISHED' 's9s_haproxy --delete-ka completed' 0
    exit 0

}



function delete_haproxy
{
    LB_NODE_LIST=""
    args=`getopt p:s:P:h:i:N:n:j: $*`
    set -- $args
    for i
    do
        case "$i" in
            -p)
                CMON_PASSWORD="$2"; shift;
                shift;;
            -i)
                CLUSTER_ID="$2"; shift;
                shift;;
            -h)
                LB_HOSTNAME="$2"; shift;
                shift;;
            -n)
                LB_NODE_LIST="$2"; shift;
                shift;;
            -N)
                LB_NAME="$2"; shift;
                shift;;
            -j)
                JOBID="$2"; shift;
                shift;;
            --)
                shift; break;;
                esac
    done
    if [ -z "$CLUSTER_ID" ]; then
      log_job_message "s9s_haproxy --delete -i <cluster id>  missing" 1
      log_job "FAILED" "s9s_haproxy failed" 1
      exit 1
    fi
    init
    
    if [ -z "$CMON_PASSWORD" ]; then
      log_job_message "s9s_haproxy --delete -p  missing" 1
      log_job "FAILED" "s9s_haproxy failed" 1
      exit 1
    fi

    if [ -z "$LB_HOSTNAME" ]; then
        log_job_message "s9s_haproxy --delete  -h <loadbalancer host> missing" 1
	log_job "FAILED" "s9s_haproxy failed" 1
        exit 1
    fi
    

    if [ -z "$JOBID" ]; then
	JOBID=0
    fi

    log_job 'RUNNING' 'Job is running' 1
    load_opts $CLUSTER_ID
    rm -rf $TMPDIR/haproxy/*
    prepare

    case $OS in
        rhel|redhat|centos6)
            PKG_MGR="$HTTP_PROXY yum -y remove "
            OS=rhel
            ;;
        debian|ubuntu)
            PKG_MGR="$HTTP_PROXY apt-get -y remove "
            OS=debian
            ;;
        *)
        log_job_message "OS $OS not supported" 1
	log_job "FAILED" "s9s_haproxy failed" 1
        exit 1
        ;;
    esac

    case $CLUSTER_TYPE in
        galera)
        ;;
        mysqlcluster)
        ;;
        *)
        log_job_message "Cluster $CLUSTER_TYPE not supported" 1
	log_job "FAILED" "s9s_haproxy failed" 1
        exit 1
        ;;
    esac


    remote_cmd_nofail $LB_HOSTNAME "$PKG_MGR haproxy"
    remote_cmd_nofail $LB_HOSTNAME "rm -f /etc/haproxy/haproxy.cfg"
    remote_cmd_nofail $LB_HOSTNAME "killall -9 haproxy"
    echo ""
    echo "** Removing init.d/haproxy auto start**"
    echo ""

    if [ $OS = "rhel" ] || [ $OS = "centos6"  ] ; then
          remote_cmd_nofail $LB_HOSTNAME "/sbin/chkconfig --del haproxy"
          remote_cmd_nofail $LB_HOSTNAME "/sbin/chkconfig haproxy off"
    else
          remote_cmd_nofail $LB_HOSTNAME "sed -i 's#ENABLED=.*#ENABLED0=0#g' /etc/default/haproxy"
          remote_cmd_nofail $LB_HOSTNAME "/usr/sbin/update-rc.d -f haproxy remove"
    fi

    QUERY="DELETE FROM haproxy_server WHERE cid=$CLUSTER_ID AND lb_host='$LB_HOSTNAME'"
    $MYSQL_BIN --host=$CMON_DB_HOST --port=$CMON_DB_PORT --user=cmon --password=$CMON_PASSWORD --database=$CMON_DB_DB -e "$QUERY"
    
    QUERY="DELETE FROM ext_proc WHERE cid=$CLUSTER_ID AND hostname='$LB_HOSTNAME' AND proc_name='haproxy'"
    $MYSQL_BIN -B -N  -ucmon -p$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST  --port=${CMON_DB_PORT} -e "$QUERY" 2>&1 >/tmp/err.log
    if [  $? -ne 0 ]; then
        log_job_message "Query failed: $QUERY" 1
       echo ""
       cat /tmp/err.log
       log_job "FAILED" "s9s_haproxy failed" 1
    fi
    log_job_message "Removed $LB_HOSTNAME" 0
    echo ""
    log_job "FINISHED" "s9s_haproxy --delete completed" 0    
    exit 0
}



function remove_node
{
    args=`getopt p:s:P:h:i:N:n:j: $*`
    set -- $args
    for i
    do
        case "$i" in
            -p)
                CMON_PASSWORD="$2"; shift;
                shift;;
            -i)
                CLUSTER_ID="$2"; shift;
                shift;;
            -h)
                LB_HOSTNAME="$2"; shift;
                shift;;
            -n)
                NODE="$2"; shift;
                shift;;
            -N)
                LB_NAME="$2"; shift;
                shift;;
            -j)
                JOBID="$2"; shift;
                shift;;
            --)
                shift; break;;
                esac
    done
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_haproxy --remove-node -i <cluster id>  missing"
      exit 1
    fi
    init
    
    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_haproxy --remove-node -p  missing"
      exit 1
    fi

    if [ -z "$LB_HOSTNAME" ]; then
        echo "s9s_haproxy --remote-node  -h <loadbalancer host> missing"
        exit 1
    fi

    if [ -z "$NODE" ]; then
        echo "s9s_haproxy --remote-node  -n <serveraddr to remove from lb > missing"
        exit 1
    fi
    
    if [ -z "$LB_NAME" ]; then
        LB_NAME="production"
    fi

    if [ -z "$JOBID" ]; then
	JOBID=0
    fi

    load_opts $CLUSTER_ID
    prepare

    case $OS in
        rhel|redhat)
            PKG_MGR="$HTTP_PROXY yum install -y "
            PHP_CURL="php-curl"
            OS=rhel
            ;;
        debian|ubuntu)
            PKG_MGR="$HTTP_PROXY apt-get install -y "
            PHP_CURL="php5-curl"
            OS=debian
            ;;
        *)
        echo "OS $OS not supported"
        exit
        ;;
    esac

    case $CLUSTER_TYPE in
        galera)
        ;;
        mysqlcluster)
        ;;
        *)
        echo "Cluster $CLUSTER_TYPE not supported"
        exit 1
        ;;
    esac

    REMOVE_STR="server $NODE"
    remote_cmd $LB_HOSTNAME "sed -i.bak '/$REMOVE_STR/d'  /etc/haproxy/haproxy.cfg"
    remote_cmd $LB_HOSTNAME "/usr/sbin/haproxy ${HAPROXY_OPTS}"    

}

function add_node
{
    args=`getopt p:s:P:h:i:N:n:j: $*`
    set -- $args
    for i
    do
        case "$i" in
            -p)
                CMON_PASSWORD="$2"; shift;
                shift;;
            -i)
                CLUSTER_ID="$2"; shift;
                shift;;
            -h)
                LB_HOSTNAME="$2"; shift;
                shift;;
            -n)
                NODE="$2"; shift;
                shift;;
            -N)
                LB_NAME="$2"; shift;
                shift;;
            -j)
                JOBID="$2"; shift;
                shift;;
            --)
                shift; break;;
                esac
    done
    if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_haproxy --remove-node -i <cluster id>  missing"
      exit 1
    fi
    init
    
    if [ -z "$CMON_PASSWORD" ]; then
      echo "s9s_haproxy --remove-node -p  missing"
      exit 1
    fi

    if [ -z "$LB_HOSTNAME" ]; then
        echo "s9s_haproxy --remove-node  -h <loadbalancer host> missing"
        exit 1
    fi

    if [ -z "$NODE" ]; then
        echo "s9s_haproxy --remove-node  -n <serveraddr to add to lb > missing"
        exit 1
    fi
    
    if [ -z "$LB_NAME" ]; then
        LB_NAME="production"
    fi

    if [ -z "$JOBID" ]; then
	JOBID=0
    fi

    load_opts $CLUSTER_ID
    prepare

    case $OS in
        rhel|redhat)
            OS=rhel
            ;;
        debian|ubuntu)
            OS=debian
            ;;
        *)
        echo "OS $OS not supported"
        exit
        ;;
    esac

    case $OS in
        rhel|redhat)
            PKG_MGR="$HTTP_PROXY yum install -y "
            PHP_CURL="php-curl"
            OS=rhel
            ;;
        debian|ubuntu)
            PKG_MGR="$HTTP_PROXY apt-get install -y "
            PHP_CURL="php5-curl"
            OS=debian
            ;;
        *)
        echo "OS $OS not supported"
        exit
        ;;
    esac


    case $CLUSTER_TYPE in
        galera)
        ;;
        mysqlcluster)
        ;;
        *)
        echo "Cluster $CLUSTER_TYPE not supported"
        exit 1
        ;;
    esac
    cp $TMPDIR/mysqlchk.sh.$CLUSTER_TYPE $TMPDIR/mysqlchk.sh 
    sed -i "s#MYSQL_PASSWORD=.*#MYSQL_PASSWORD=\"$CMON_PASSWORD\"#g" $TMPDIR/mysqlchk.sh
    sed -i "s#MYSQL_USERNAME=.*#MYSQL_USERNAME=\"cmon\"#g" $TMPDIR/mysqlchk.sh
    sed -i "s#MYSQL_PORT=.*#MYSQL_PORT=\"$MONITORED_MYSQL_PORT\"#g" $TMPDIR/mysqlchk.sh
    sed -i "s#MYSQL_BIN=.*#MYSQL_BIN=\"$MYSQL_BIN\"#g" $TMPDIR/mysqlchk.sh
    install_mysqlchk $NODE
    REMOVE_STR="server $NODE"
    remote_cmd_nofail $LB_HOSTNAME "sed -i.bak '/$REMOVE_STR/d'  /etc/haproxy/haproxy.cfg"
    remote_cmd $LB_HOSTNAME "echo '        server $NODE ${NODE}:3306 check' >> /etc/haproxy/haproxy.cfg"
    remote_cmd $LB_HOSTNAME "/usr/sbin/haproxy ${HAPROXY_OPTS}"    

}


if [ ! -e $LOCKFILE ]; then
    trap "rm -f $LOCKFILE; exit" INT TERM EXIT
    ARG=$1
    shift
    touch $LOCKFILE    
    case $ARG in
	--prepare)
    	    prepare
	    ;;
	--install)
	    install_haproxy $*
	    ;;
	--install-keepalived)
	    install_keepalived $*
	    ;;
	--delete-keepalived)
	    delete_keepalived $*
	    ;;
	--add-node)	
	    add_node $*
	    ;;
	--remove-node)
	    remove_node $*
	    ;;
	--delete)
	    delete_haproxy $*
	    ;;
	*)
	    echo "Usage: "
	    echo "s9s_haproxy --[install|delete|add-node|remove-node|install-keepalived|delete-keepalived]: "
	    exit 1
	    ;;
    esac
    rm $LOCKFILE
    trap - INT TERM EXIT
fi

