#!/bin/bash
# Copyright 2013 Severalnines AB
#
# MODIFY THE BELOW TO SUIT YOU ENV:

JOBID=0
function init
{    
    FILES=`ls /etc/cmon.cnf 2>&1`
    FILES2=`ls /etc/cmon.d/*.cnf 2>&1`    
    FILES="$FILES $FILES2"
    configfile=""
    for f in $FILES 
    do
	X=`grep -l cluster_id=${CLUSTER_ID} $f 2>&1 `	
	if [ $? -eq 0 ]; then 
	    source $f
            configfile=$f
	fi
    done

    if [ -z "$configfile" ]; then
	echo "No matching configuration file found having cluster_id=${CLUSTER_ID}"
	exit 1
    fi
   
    source $configfile


    CID=$cluster_id
    CMON_USER=cmon
    CMON_DB_PORT=$mysql_port
    CMON_PASSWORD=$mysql_password
    CMON_DB_HOST=$mysql_hostname
    CMON_DB_DB=cmon
    MYSQL_BIN=$mysql_basedir/bin//mysql
    MYSQL_BIN2=$mysql_bindir/mysql
    CLUSTER_STATE_TIMEOUT=3600
    START_BALANCER_TIMEOUT=3600
    BACKUP_LOGFILE="/tmp/s9s_mongodb_backup.log"
### NO MODS BELOW UNLESS YOU KNOW WHAT YOU DO:
    
    export LC_ALL=C
    OSUSER=$USER
    
    
    if [ "$OSUSER" != "root" ]; then
	echo "must be executed as 'root' or with 'sudo'"
	exit 1
    fi
}

if ! test -f $MYSQL_BIN; then
    if ! test -f $MYSQL_BIN2; then
	echo "Could not find mysql client binary"
	echo "Change MYSQL_BIN in beginning of the scipt"
	exit 1
    fi
    MYSQL_BIN=$MYSQL_BIN2
fi


function log_job_message
{    
    MSG=$1
    EXIT_CODE=$2
    QUERY="INSERT INTO cmon_job_message(cid, jobid,message,exit_code,report_ts) VALUES($CLUSTER_ID,$JOBID,\"$MSG\",$EXIT_CODE,now())"
    echo $QUERY >> /tmp/queries
    if [ $JOBID -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    else
	if [ $EXIT_CODE -eq 0 ]; then
   	   echo "$MSG"
	else
   	   echo "Failed: $MSG"
	fi
    fi
}

function create_job
{    
    MSG=$1
    STATUS=$2
    EXIT_CODE=$3
    QUERY="INSERT INTO cmon_job(cid, jobid, jobspec,status,status_txt,exit_code,report_ts) VALUES($CLUSTER_ID,$JOBID,'internal','$STATUS',\"$MSG\",$EXIT_CODE,now())"
    echo $QUERY >> /tmp/queries
    if [ $JOBID -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    else
	if [ $EXIT_CODE -eq 0 ]; then
   	   echo "$MSG"
	else
   	   echo "Failed: $MSG"
	fi
    fi
}

function log_job
{    
    STATUS=$1
    STATUS_TXT=$2
    EXIT_CODE=$3
    QUERY="UPDATE cmon_job SET status='$STATUS', status_txt='$STATUS_TXT', exit_code=$EXIT_CODE, report_ts=NOW()  WHERE cid=$CLUSTER_ID AND jobid=$JOBID"
    if [ $JOBID -ne 0 ]; then 
	$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
    fi
}

function remote_copy()
{
   srcfile=$1
   desthost=$2
   destfile=$3
   printf "%-4s: Copying '%s' " "$desthost" "$srcfile"
   scp $SSH_OPTS2 $srcfile $SSH_USER@$desthost:$destfile >> $HOME/s9s_deploy.log  2>/dev/null
   echo "scp $SSH_OPTS2 $srcfile $SSH_USER@$desthost:$destfile >> $HOME/s9s_deploy.log  2>/dev/null"
   if [ $? -eq 0 ]; then
      log_job_message "copying file $srcfile --> $desthost:$destfile" 0
      printf "\033[32m[ok]\033[0m\n"
      return 0
   else
       echo "FAILED: scp $SSH_OPTS2 $srcfile $SSH_USER@$desthost:$destfile >> $HOME/s9s_deploy.log  2>/dev/null"
       echo ""
      log_job_message "copying file $srcfile --> $desthost:$destfile" 1
      log_job 'FAILED' 's9s_haproxy failed' 1
      printf "\033[31m[failed]\033[0m\n"
      exit 1
   fi
}

function remote_cmd_nofail()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s'" "$desthost" "$xcommand"
   ssh $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand "  >> $HOME/s9s_deploy.log 2>&1
   ret=$?
   printf "\033[32m[ok]\033[0m\n"
   return $ret
}


function remote_cmd3()
{
   desthost=$1
   xcommand=$2
   printf "%-4s: Executing '%s' " "$desthost" "$xcommand"
   ssh $SSH_OPTS $SSH_OPTS_EXTRA $SSH_USER@$desthost "$SUDO $xcommand " &
   if [ $? -eq 0 ]; then
      printf "\033[32m[ok]\033[0m\n"
      log_job_message "$desthost: Executed $xcommand" 0
      return 0
   else
      printf "\033[31m[failed]\033[0m\n"
      log_job_message "FAILED: ${desthost}@${xcommand}" 1
      exit 1
   fi
}

function remote_cmd_getreply()
{
   desthost=$1
   xcommand=$2
   x=`ssh $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand " 2>&1`
   echo $x
}


function remote_cmd()
{
   desthost=$1
   xcommand=$2
   MAX_RETRIES=1
   printf "%-4s: Executing '%s' " "$desthost" "$xcommand"
   retry=0
   while [ $retry -lt $MAX_RETRIES ]; 
   do
      x=`ssh -q $SSH_OPTS $SSH_USER@$desthost "$SUDO $xcommand " > $HOME/s9s_cmd.log 2>&1`
      if [ $? -eq 0 ]; then
        printf "\033[32m[ok]\033[0m\n"
	log_job_message "Executed on $desthost : $xcommand" 0
        return 0
      fi
      retry=`expr $retry + 1`
      printf "\033[31m[failed: retrying ${retry}/${MAX_RETRIES}]\033[0m\n"
      ssh -q $SSH_OPTS $SSH_USER@$desthost " sync " 2>&1  >> $HOME/s9s_deploy.log
      sleep 1
   done
   
   log_job_message "Command failed: $xcommand" 1
   log_job 'FAILED' 's9s_clone --create failed' 1
   printf "\033[31m[failed]\033[0m\n"
   echo $x
   echo 'The following command failed:'
   echo "ssh -q $SSH_OPTS $SSH_USER@$desthost \" $SUDO  $xcommand \""
   cat $HOME/s9s_cmd.log
   echo 'Try running the command on the line above again, contact http://support.severalnines.com/ticket/new, attach the output from deploy.sh and the error from running the command to the Support issue.'
   exit 1
}

function write_email
{
  SUBJECT=$1
  MSG=$2
  cat $BACKUP_LOGFILE | sed  -e "s/'/\\\'/g" -e 's/"/\\"/g' > /tmp/s9s_backup_log_escaped
  MSG="$MSG \nBackup log follows:\n `cat /tmp/s9s_backup_log_escaped`" 
  QUERY="INSERT INTO simple_email(cid,subject,message) VALUES($CLUSTER_ID, \"$SUBJECT\", \"$MSG\")"
  $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
}


function load_opts 
{
    local CLUSTER_ID=$1
    echo "load opts $CLUSTER_ID"
    OS=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS' AND cid=$CLUSTER_ID"`
    CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='CONFIGDIR' AND cid=$CLUSTER_ID"`
    MYSQL_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_PORT' AND cid=$CLUSTER_ID"`
    HTTP_PROXY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='HTTP_PROXY' AND cid=$CLUSTER_ID"`
    GALERA_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='GALERA_PORT' AND cid=$CLUSTER_ID"`
    MYSQL_BASEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MYSQL_BASEDIR' AND cid=$CLUSTER_ID"`
    MONGODB_BASEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONGODB_BASEDIR' AND cid=$CLUSTER_ID"`
    MYSQL_SCRIPTDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SCRIPTDIR' AND cid=$CLUSTER_ID"`
    SSH_IDENTITY=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_IDENTITY' AND cid=$CLUSTER_ID"`
    SSH_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_PORT' AND cid=$CLUSTER_ID"`
    SSH_USER=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_USER' AND cid=$CLUSTER_ID"`
    SSH_OPTSX=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SSH_OPTS' AND cid=$CLUSTER_ID"`
    SUDO=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='SUDO' AND cid=$CLUSTER_ID"`
    VENDOR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='VENDOR' AND cid=$CLUSTER_ID"`
    S9S_TMPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='STAGING_DIR' AND cid=$CLUSTER_ID"`
    CLUSTER_TYPE=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select type FROM cluster WHERE id=$CLUSTER_ID"`
    if [ "${CLUSTER_TYPE}" != "mongodb" ]; then
	log_job_message "This is not a mongodb cluster, aborting. " 1
	exit 1
    fi

    BACKUPDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='BACKUPDIR' AND cid=$CLUSTER_ID"`

    MONGODB_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONGODB_PORT' AND cid=$CLUSTER_ID"`
    MONGOS_PORT=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONGOS_PORT' AND cid=$CLUSTER_ID"`
    MONGODB_DATADIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select dbpath from  mongodb_server where node_type=0 AND dbpath<>'' and cid=$CLUSTER_ID limit 1"`
    MONGODB_PIDFILEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select pidfilepath from  mongodb_server where node_type=0 and pidfilepath<>'' and cid=$CLUSTER_ID limit 1"`
    MONGODB_PIDFILEPATH=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select pidfilepath from  mongodb_server where node_type=0 and pidfilepath<>'' and cid=$CLUSTER_ID limit 1"`
    if [ -z ${MONGODB_PIDFILEPATH} ]; then
	log_job_message "Failed to MONGODB_PIDFILEPATH " 1
	exit 1
    fi
    if [ -z ${MONGODB_PIDFILEDIR} ]; then
	log_job_message "Failed to MONGODB_PIDFILEDIR " 1
	exit 1
    fi
    MONGODB_PIDFILEDIR=`dirname $MONGODB_PIDFILEDIR`
    MONGODB_LOGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select logpath from  mongodb_server where node_type=0 and logpath<>'' and cid=$CLUSTER_ID limit 1"`
    if [ -z ${MONGODB_LOGDIR} ]; then
	log_job_message "Failed to MONGODB_LOGDIR " 1
	exit 1
    fi
    MONGODB_LOGDIR=`dirname $MONGODB_LOGDIR`
    MONGODB_BASEDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONGODB_BASEDIR' AND cid=$CLUSTER_ID"`
    MONGODB_CONFIGDIR=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='MONGODB_CONFIGDIR' AND cid=$CLUSTER_ID"`
#    MONGODB_CNF=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select data  from cmon.cluster_configuration_templates where cid=${CLUSTER_ID} limit 1"`
    MONGODB_CNF=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select data  from cmon.cluster_configuration_templates where cid=$CLUSTER_ID and filename='mongodb.0.conf' limit 1" 2>/dev/null`

    if [ -z "$MONGODB_PORT" ]; then
       MONGODB_PORT=27018
    fi
    if [ -z "$MONGOS_PORT" ]; then
       MONGOS_PORT=27017
    fi
    if [ -z "$MONGOCFG_PORT" ]; then
       MONGOCFG_PORT=27019
    fi

    if [ -z "$MONGODB_CONFIGDIR" ]; then
       MONGODB_CONFIGDIR="/etc/"
    fi

    if [ -z "$MONGODB_PIDFILEDIR" ]; then
	MONGODB_PIDFILEDIR="/var/run/mongodb/"
    fi
    
    mkdir -p $S9S_TMPDIR
    echo $MONGODB_CNF |  sed 's#\\n#\n\r#g' > $S9S_TMPDIR/mongodb.conf

    ########

    OS_USER_HOME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select value from cmon_configuration where param='OS_USER_HOME' AND cid=$CLUSTER_ID"`

    SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10"
    if [ "$SSH_IDENTITY" = "" ]; then
	SSH_IDENTITY="-oIdentityFile=${OS_USER_HOME}/.ssh/id_rsa"
    else
	SSH_IDENTITY="-oIdentityFile=$SSH_IDENTITY"
    fi
    SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    SSH_OPTS2="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
    if [ "$SSH_USER" != "root" ]; then
	if [ -z "$SSH_OPTSX" ] || [ "SSH_OPTSX" = "NULL" ]; then
	    SSH_OPTSX="-tt"
	fi
	SSH_OPTS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -oNumberOfPasswordPrompts=0 -oConnectTimeout=10 $SSH_IDENTITY"
	if [ "$SUDO" = "" ] || [ "$SUDO" = "NULL" ];  then
           SUDO=sudo
        fi
    fi
    if [ "$SSH_PORT" = "" ] || [ "$SSH_PORT" = "NULL" ];  then
        SSH_PORT="22"
    fi

    SSH_OPTS="$SSH_OPTSX -p$SSH_PORT $SSH_OPTS"    
    SSH_OPTS2="-P$SSH_PORT $SSH_OPTS2 -q"     

    if [ "$BACKUPDIR" = "" ]; then
        BACKUPDIR="$HOME/backups/"
    fi

}


function wait_for_ok
{
   desthost=$1
   port=$2
   timeout=$3
   retry=0
   echo "Trying to connect to mongo for $timeout seconds to check it starts up"
   while [ $retry -lt $timeout ];
   do
      echo -n "."
      ok=`$MONGODB_BASEDIR/bin/mongo $desthost:$port --quiet --eval "printjson(db.serverStatus().ok)" 2>&1`
      if [ "$ok" = "1" ];  then
         echo ""
         echo "Node started"
         return 0
      fi
      sleep 1
      retry=`expr $retry + 1`
   done
   echo "Node failed to start in $timeout seconds."
   exit 1
}
function wait_for_ok_rs
{
   desthost=$1
   port=$2
   timeout=$3
   retry=0   
   echo "Trying to connect to mongo for $timeout seconds to check it is done creating replica set"
   while [ $retry -lt $timeout ];
   do
      ok=`$MONGODB_BASEDIR/bin/mongo $desthost:$port --quiet --eval "printjson(rs.status().members)"  |grep stateStr | grep -e 'PRIMARY\|SECONDARY'  |wc -l 2>&1`
      echo  "Check ${retry}/${timeout} : $ok PRIMARY/SECONDARY(s) initialized out of ${RS_SIZE}"
      if [ "$ok" = "${RS_SIZE}" ];  then
          echo ""
          echo "PRIMARY and SECONDARY(s) detected ok!"
          return 0
      fi
      sleep 1
      retry=`expr $retry + 1`
   done
   echo "Failed to detect PRIMARY and SECONDARY(s) in $timeout seconds."
   exit 1
}

function remove_lockfile
{        
    args=`getopt h:t:i: $*`
    set -- $args
    for i
    do
	case "$i" in
            -h)
		XHOSTNAME="$2"; shift;
		shift;;
	    -t)
		TYPE="$2"; shift;
		shift;;
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    

  case $TYPE in 
      configsvr)
	  ;;      
      shardsvr)
	  ;;
      *)
	  echo "s9s_mongodb_admin --remove-lockfile -i <cluster id> -h <hostname> -t <configsvr|shardsvr>"
	  exit 1
  esac
  
  if [ -z "$TYPE" ]; then
      echo "s9s_mongodb_admin --remove-lockfile -i <cluster id> -h <hostname> -t <configsvr|shardsvr>"
      exit 1
  fi

  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --remove-lockfile -i <cluster id> -h <hostname> -t <configsvr|shardsvr>"
      exit 1
  fi
  init 
  if [ -z "$XHOSTNAME" ]; then
      echo "s9s_mongodb_admin --remove-lockfile -i <cluster id> -h <hostname> -t <configsvr|shardsvr>"
      exit 1
  fi
  
  load_opts $CLUSTER_ID

  QUERY="select dbpath from mongodb_server m, hosts h,mongodb_nodetype_map n  where h.id=m.serverid and m.cid=$CLUSTER_ID and (h.hostname='$XHOSTNAME' or h.ip='$XHOSTNAME') and m.node_type=n.id and n.name='$TYPE'"
  
  DBPATH=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`

  if [ -z "$DBPATH" ]; then
      echo "No 'dbpath' found for $XHOSTNAME, type $TYPE"
      exit 1
  fi
  
#  echo "Are you sure you want to remove the lockfile $DBPATH/mongod.lock? Enter YES to remove it."
#  read ANSWER

 # if [ "$ANSWER" = "YES" ]; then
      remote_cmd $XHOSTNAME "rm -f $DBPATH/mongod.lock"
#  else
 #     echo "$ANSWER was pressed ($ANSWER != YES ), not removing file"
  #    exit 0
  #fi    
}

function add_shard
{
    RESTART_CMON=1
    args=`getopt i:a:b:c:N:m:j:f: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
            -a)
		SERVER1="$2"; shift;
		shift;;
	    -b)
		SERVER2="$2"; shift;
		shift;;
	    -c)
		SERVER3="$2"; shift;
		shift;;
            -N)
		RSNAME="$2"; shift;
		shift;;
            -f)
		TGZ_FILE="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    

  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --add-shard -i <cluster id>"
      exit 1
  fi
  init 
  if [ -z "$SERVER1" ]; then
      echo "s9s_mongodb_admin --add-shard -a <replica set member address>"
      exit 1
  fi
  if [ -z "$SERVER2" ]; then
      echo "s9s_mongodb_admin --add-shard -b <replica set member address>"
      exit 1
  fi
  if [ -z "$SERVER3" ]; then
      RS_SIZE=2
  else
      RS_SIZE=3
  fi
  
  echo "$RS_SIZE replica set members"
  
  if [ -z "$RSNAME" ]; then
      echo "s9s_mongodb_admin --add-shard -N <replica set name>"
      exit 1
  fi

  MONGOS_SERVER=$(get_mongos_server)
  if [ -z "$MONGOS_SERVER" ]; then
      echo "s9s_mongodb_admin --add-shard : no running mongos server found"
      exit 1
  fi

  RS_MEMBERS="$SERVER1 $SERVER2 $SERVER3"

  echo $CLUSTER_ID
  load_opts $CLUSTER_ID
  
  if [ $VENDOR = 'tokutek' ]; then 
      if [ -z "$TGZ_FILE" ]; then
	  echo "s9s_mongodb_admin --add-shard -f <path to tokumx .tgz file>"
	  exit 1
      fi
  fi

  for R in $RS_MEMBERS $MONGOS_SERVER
  do
      remote_cmd $R "echo 'ssh test'"
      remote_cmd_nofail $R "mkdir -p $S9S_TMPDIR"
      remote_cmd_nofail $R "chown -R $SSH_USER:$SSH_USER  $S9S_TMPDIR"
      if [ "$OS"="redhat" ]; then
	  remote_cmd_nofail $R "/usr/sbin/setenforce 0"
	  remote_cmd_nofail $R "echo '0' > /selinux/enforce "
	  remote_cmd_nofail $R "sed -i.bak 's#SELINUX=enforcing#SELINUX=permissive#g' /etc/selinux/config"
      fi
  done
  
  for R in $RS_MEMBERS
  do
      install_mongodb $R  $TGZ_FILE
  done
  start_rs
  create_rs
  create_shard

  

  QUERY="SELECT GROUP_CONCAT(concat(hostname,':',port) ORDER BY serverid) FROM mongodb_server WHERE cid=$CLUSTER_ID and node_type=0"
  SERVERS=`$MYSQL_BIN $MYSQL_OPTS -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`    
  if [ $? -ne 0 ]; then
      log_job_message "Failed to get mysql_server list" 1
      exit 1
  fi
  if [ $RS_SIZE -eq 3 ]; then
     SERVERS="$SERVERS,$SERVER1:$MONGODB_PORT,$SERVER2:$MONGODB_PORT,$SERVER3:$MONGODB_PORT"  
  fi
  if [ $RS_SIZE -eq 2 ]; then
     SERVERS="$SERVERS,$SERVER1:$MONGODB_PORT,$SERVER2:$MONGODB_PORT"  
  fi
  sed -i.bak "s#mongodb_server_addresses=.*#mongodb_server_addresses=${SERVERS}#g" $configfile
  
  if [ $RESTART_CMON -eq 1 ]; then
      service cmon restart
  fi


}

function upgrade_mongodb
{
    RESTART_CMON=1
    args=`getopt h:i:a:b:c:N:m:j:f: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
            -h)
		XHOST="$2"; shift;
		shift;;
            -f)
		TGZ_FILE="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    
    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_mongodb_admin --upgrade-mongodb -i <cluster id>"
	exit 1
    fi
    init
    load_opts $CLUSTER_ID

    if [ -z $XHOST ]; then	
	SERVER_LIST=$(get_all_servers)
	if [ "$SERVER_LIST" = "NULL" ]; then
	    echo "Could not get servers";
	    exit 1
	fi
    else
	SERVER_LIST=$XHOST
    fi
    for r in $SERVER_LIST
    do
	xhost=`echo $r | awk -F ',' '{print $1};'`
	echo "Upgrading $xhost"
	case $VENDOR in
	    tokutek)	    
		if [ -z "$TGZ_FILE" ]; then
		    echo "s9s_mongodb_admin --upgrade-mongodb -f <path to tokumx..tgz"
		    exit 1
		fi
		if [ ! -f $TGZ_FILE ]; then
		    echo "Could not find : $TGZ_FILE"
		    echo "s9s_mongodb_admin --upgrade-mongodb -f <path to tokumx..tgz"
		    exit 1
		fi	    
		remote_copy $TGZ_FILE $xhost $S9S_TMPDIR/ 
		remote_cmd $xhost  "tar xfz $S9S_TMPDIR/`basename $TGZ_FILE` -C /usr/local"
		remote_cmd $xhost  "rm -rf /usr/local/tokumx"
		remote_cmd $xhost  "ln -s --force /usr/local/`basename $TGZ_FILE .tgz`  /usr/local/tokumx" 
		remote_cmd $xhost  "test -f /usr/local/tokumx/bin/mongod" 
		remote_cmd_nofail $xhost "rm -rf $S9S_TMPDIR/`basename $TGZ_FILE`"
		;;
	    10gen)
		case $OS in
		    redhat)
			remote_cmd $xhost "$HTTP_PROXY yum -y install mongo-10gen mongo-10gen-server"
			;;
		    debian)
	   		remote_cmd $xhost "$HTTP_PROXY apt-get -q -y update"
			remote_cmd $xhost "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -o Dpkg::Options::='--force-confnew' -y -q install mongodb-10gen"
			;;
		    *)
			echo "unknown OS"
			exit 1
			;;
		esac
		;;
	    *)
		echo "unknown vendor" 
		exit 1
		;;
	esac
    done
    return 0
}


function rolling_restart
{
    RESTART_CMON=1
    args=`getopt h:i:a:b:c:N:m:j:f: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
            -h)
		XHOST="$2"; shift;
		shift;;
            -f)
		TGZ_FILE="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    
    
    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_mongodb_admin --upgrade-mongodb -i <cluster id>"
	exit 1
    fi
    init
    load_opts $CLUSTER_ID
    
    echo "Rolling restart"
    TYPE="shardsvr"
    SERVER_LIST=$(get_servers $TYPE)
    if [ "$SERVER_LIST" != "NULL" ]; then
	for r in $SERVER_LIST
	do
	    HNAME=`echo $r | awk -F ',' '{print $1};'`
	    NODEID=`echo $r | awk -F ',' '{print $2};'`
	    echo "Restarting $HNAME"
	    remote_cmd $HNAME "killall -9 mongod"
	    sleep 1
	    STATE=$(check_node_up $HNAME $NODEID)
	    if [ "$STATE" = "DOWN" ]; then
		start_node $HNAME $NODEID
	    fi
	    sleep 2
	    wait_for_cluster_state "STARTED"
	done
    fi
    
    
    TYPE="configsvr"
    SERVER_LIST=$(get_servers $TYPE)
    if [ "$SERVER_LIST" != "NULL" ]; then
      for r in $SERVER_LIST
      do
	  HNAME=`echo $r | awk -F ',' '{print $1};'`
	  NODEID=`echo $r | awk -F ',' '{print $2};'
	  echo "Restarting $HNAME"`
	  remote_cmd $HNAME "killall -9 mongod"
	  sleep 1
	  STATE=$(check_node_up $HNAME $NODEID)
	  if [ "$STATE" = "DOWN" ]; then
	      start_node $HNAME $NODEID
	  fi
	  sleep 2
	  wait_for_cluster_state "STARTED"
      done
    fi
    
    TYPE="mongos"
    SERVER_LIST=$(get_servers $TYPE)
    if [ "$SERVER_LIST" != "NULL" ]; then
	for r in $SERVER_LIST
	do
	    HNAME=`echo $r | awk -F ',' '{print $1};'`
	    NODEID=`echo $r | awk -F ',' '{print $2};'`
	    echo "Restarting $HNAME"
	    remote_cmd $HNAME "killall -9 mongos"
	    sleep 2
	    STATE=$(check_node_up $HNAME $NODEID)
	    if [ "$STATE" = "DOWN" ]; then
		start_node $HNAME $NODEID
	    fi
	    sleep 2
	    wait_for_cluster_state "STARTED"
	done
    fi        
}


function install_mongodb
{
    RESTART_CMON=1
    args=`getopt i:h:N:m:j:f: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
            -h)
		SERVER1="$2"; shift;
		shift;;
            -f)
		TGZ_FILE="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    

    if [ -z "$CLUSTER_ID" ]; then
	echo "s9s_mongodb_admin --install-mongodb -i <cluster id>"
	exit 1
    fi
    init 
    if [ -z "$SERVER1" ]; then
	echo "s9s_mongodb_admin --install-mongodb -h <hostname>"
	exit 1
    fi
    load_opts $CLUSTER_ID
    case $VENDOR in
	tokutek)
	    if [ -z "$TGZ_FILE" ]; then 
		echo "s9s_mongodb_admin --install-mongodb -f <path to tokumx tar.gz file is missing>"
		exit 1
	    fi
	    ;;
	*)
	    ;;
    esac
    

    local xhost=$SERVER1

    remote_cmd_nofail $xhost "killall -9 cmon mongod"
    remote_cmd_nofail $xhost "rm -rf /etc/mongodb.conf"
    remote_cmd_nofail $xhost "rm -rf /var/lib/mongodb"
    remote_cmd_nofail $xhost "rm -rf /var/log/mongodb"
    remote_cmd_nofail $xhost "rm -rf /etc/init.d/mongod"
    remote_cmd_nofail $xhost "rm -rf ${MONGODB_DATADIR}"
    case $VENDOR in
	tokutek)
	    if [ -z "$TGZ_FILE" ]; then
		if [ -z "$PKG_NAME" ]; then
		## default to MYSQL_INITIAL_DEPLOY if not set
		    PKG_NAME="MONGODB_INITIAL_DEPLOY"
		fi
		QUERY="SELECT packageid FROM ${CMON_DB}.cmon_sw_package WHERE name='$PKG_NAME' AND cid=$CLUSTER_ID"
		PID=`$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY"`	
		if [ -z "$PID" ]; then
		    log_job_message "Invalid package name $PKG_NAME" 1
		    log_job 'FAILED' 'job failed' 1
		    write_email "s9s_mongodb_admin failed" "Invalid package name $PKG_NAME. Check job messages."
		    exit 1
		fi
		QUERY="SELECT group_concat(concat(path,',', filename) SEPARATOR ' ') FROM ${CMON_DB}.cmon_uploads WHERE packageid='$PID' AND cid=$CLUSTER_ID"
		FILELIST=`$MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY"`	    
		if [ -z "$FILELIST" ]; then
		    log_job_message "No files in package $PKG_NAME" 1
		    log_job 'FAILED' 'job failed' 1
		    write_email "s9s_sw_update failed" "No files in $PKG_NAME. Check job messages."
		    exit 1	    
		fi
		
		if [ "$FILELIST" = "NULL" ]; then
		    log_job_message "No files in package $PKG_NAME" 1
		    log_job 'FAILED' 'job failed' 1
		    write_email "s9s_sw_update failed" "No files in $PKG_NAME. Check job messages."
		    exit 1	    
		fi
		
		
		remote_cmd $XHOSTNAME "mkdir -p $S9S_TMPDIR"
		remote_cmd $XHOSTNAME "chown  -R $SSH_USER:$SSH_USER $S9S_TMPDIR"
		
		for f in $FILELIST 
		do
		    path=`echo $f | awk -F, '{print $1;}'`
		    filename=`echo $f | awk -F, '{print $2;}'`	 
		    TGZ_FILE="$path/$filename $XHOSTNAME $S9S_TMPDIR"
		done
		
	    fi
	    
	    LN_NAME=`basename $TGZ_FILE .tar.gz`
	    ## strip the -main..
	    LN_NAME=`basename $LN_NAME -main`
	    remote_copy $TGZ_FILE $xhost $S9S_TMPDIR/ 
	    remote_cmd_nofail $xhost "chown -R $SSH_USER:$SSH_USER  $S9S_TMPDIR"
	    remote_cmd $xhost  "tar xfz $S9S_TMPDIR/`basename $TGZ_FILE` -C /usr/local"
	    remote_cmd $xhost  "rm -rf /usr/local/tokumx"
	    remote_cmd $xhost  "ln -s --force /usr/local/${LN_NAME}  /usr/local/tokumx" 
	    remote_cmd $xhost  "test -f /usr/local/tokumx/bin/mongod" 
	    ;;
	10gen)
	    case $OS in
		redhat)
		    cat << 'EOF' > $S9S_TMPDIR/10gen.repo
[mongodb]
name=MongoDB Repository
baseurl=http://downloads-distro.mongodb.org/repo/redhat/os/x86_64/
gpgcheck=0
enabled=1
EOF
		    remote_copy $S9S_TMPDIR/10gen.repo $xhost $S9S_TMPDIR/ 
		    remote_cmd $xhost "mv $S9S_TMPDIR/10gen.repo /etc/yum.repos.d/"
		    remote_cmd_nofail $xhost "$HTTP_PROXY yum -y remove mongodb*"
		    remote_cmd_nofail $xhost "$HTTP_PROXY yum -y install mongodb-org"
		    remote_cmd $xhost  "chown 755 /etc/init.d/mongod"
		    remote_cmd $xhost  "/sbin/chkconfig --add mongod"
		    ;;
		debian)
		    remote_cmd $xhost "apt-key adv --keyserver keyserver.ubuntu.com --recv 7F0CEB10"
		    remote_cmd_nofail $xhost "touch /etc/init.d/mongodb"
		    remote_cmd_nofail $xhost "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -y remove mongo*"
		    remote_cmd_nofail $xhost "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -y remove mongodb-10gen"
		    remote_cmd_nofail $xhost "rm -rf /etc/init.d/mongo*"
		    remote_cmd_nofail $xhost "sed -i.bak '/10gen/d'  /etc/apt/sources.list"
		    remote_cmd $xhost "/bin/sh -c 'echo \"deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen\" >> /etc/apt/sources.list'"
		    remote_cmd $xhost "$HTTP_PROXY apt-get -q -y update"
		    remote_cmd $xhost "LC_ALL=en_US.utf8 DEBIAN_FRONTEND=noninteractive $HTTP_PROXY apt-get -o Dpkg::Options::='--force-confnew' -y -q install mongodb-10gen"
		    ;;
		*)
		    echo "OS is unknonwn / not supported" 		  
		    exit 1
		    ;;
	    esac
	    ;;
	*)
	    echo "Unknown vendor: $VENDOR"
	    exit 1
	    ;;
    esac
    
    remote_cmd_nofail ${xhost} "killall -9 mongod mongos cmon"
    echo "Creating mongodb datadirs in ${mongodb_datadir} if not already exists ..."
    remote_cmd ${xhost} "mkdir -p ${MONGODB_DATADIR}"
    remote_cmd ${xhost} "mkdir -p ${MONGODB_DATADIR}/config"
    remote_cmd ${xhost} "mkdir -p ${MONGODB_LOGDIR}"
    remote_cmd ${xhost} "mkdir -p ${MONGODB_PIDFILEDIR}"
    remote_cmd_nofail $xhost "/usr/sbin/groupadd mongodb"
    remote_cmd_nofail $xhost "/usr/sbin/useradd mongodb -g mongodb"
    remote_cmd $xhost "chown mongodb:mongodb -R $MONGODB_DATADIR"
    remote_cmd $xhost "chown mongodb:mongodb -R $MONGODB_LOGDIR"
    remote_cmd $xhost "chown mongodb:mongodb -R ${MONGODB_PIDFILEDIR}"
}

function start_rs
{
    sed -i.bak "s/smallfiles = false//g" $S9S_TMPDIR/mongodb.conf
    sed -i.bak "s/replSet = .*/replSet = $RSNAME/g" $S9S_TMPDIR/mongodb.conf
    sed -i.bak "s/\r//g" $S9S_TMPDIR/mongodb.conf
    RS_MEMBERS="$SERVER1 $SERVER2 $SERVER3"   
    for xhost in ${RS_MEMBERS}
    do
	echo "Copying configuation"
	TMP_DATADIR=`grep dbpath $S9S_TMPDIR/mongodb.conf |grep -v '#'  |tr -d ' ' | awk -F = '{print $2}'`
	if [ -n "$TMP_DATADIR" ]; then
	    MONGODB_DATADIR=$TMP_DATADIR
	fi
	remote_copy $S9S_TMPDIR/mongodb.conf ${xhost} $S9S_TMPDIR/mongodb.${MONGODB_PORT}.conf
	remote_cmd ${xhost} "mv -f $S9S_TMPDIR/mongodb.${MONGODB_PORT}.conf $MONGODB_CONFIGDIR"
	start_mongodb $xhost
    done
}

function start_mongodb
{
    local xhost=$1
    remote_cmd_nofail ${xhost} "killall -9 mongod"
    echo "Starting mongodb shard server, this may take a while"
    remote_cmd $xhost "mkdir -p $MONGODB_PIDFILEDIR"
    remote_cmd $xhost "mkdir -p $MONGODB_DATADIR"       
    remote_cmd3 $xhost "LC_ALL=C $MONGODB_BASEDIR/bin//mongod -f ${MONGODB_CONFIGDIR}/mongodb.${MONGODB_PORT}.conf  --logpath /var/log//mongodb_${MONGODB_PORT}.log --pidfilepath ${MONGODB_PIDFILEDIR}/mongodb_${MONGODB_PORT}.pid --shardsvr"
    wait_for_ok $xhost ${MONGODB_PORT} 300
}


function create_rs
{
    local xhost=$SERVER1
    if [ $RS_SIZE -eq 3 ]; then
	cat << 'EOF' > $S9S_TMPDIR/init.js
    rs.initiate({_id: 'RSNAME', members: [
        {_id: 0, host: 'SERVER1:MONGOPORT'},
        {_id: 1, host: 'SERVER2:MONGOPORT'},
        {_id: 2, host: 'SERVER3:MONGOPORT'}]
});
EOF
    else
		cat << 'EOF' > $S9S_TMPDIR/init.js
    rs.initiate({_id: 'RSNAME', members: [
        {_id: 0, host: 'SERVER1:MONGOPORT'},
        {_id: 1, host: 'SERVER2:MONGOPORT'}]
});
EOF
    fi
	sed -i.bak "s#RSNAME#$RSNAME#g" $S9S_TMPDIR/init.js
    sed -i.bak "s#SERVER1#$SERVER1#g" $S9S_TMPDIR/init.js
    sed -i.bak "s#SERVER2#$SERVER2#g" $S9S_TMPDIR/init.js
    if [ $RS_SIZE -eq 3 ]; then
       sed -i.bak "s#SERVER3#$SERVER3#g" $S9S_TMPDIR/init.js
    fi
    sed -i.bak "s#MONGOPORT#$MONGODB_PORT#g" $S9S_TMPDIR/init.js
    remote_copy $S9S_TMPDIR/init.js  ${xhost} $S9S_TMPDIR/init.js
    remote_cmd ${xhost} "sync"
    remote_cmd $xhost "$MONGODB_BASEDIR/bin/mongo  127.0.0.1:${MONGODB_PORT}/test --quiet $S9S_TMPDIR/init.js"
    echo "creating replica set"
    wait_for_ok_rs $xhost ${MONGODB_PORT} 300
}


function create_shard
{
    local xhost=$MONGOS_SERVER
    cat << EOF > $S9S_TMPDIR/init_shard.js
    sh.addShard( 'RSNAME/SERVER:MONGOPORT' ) 
EOF
    
    sed -i.bak "s#RSNAME#$RSNAME#g" $S9S_TMPDIR/init_shard.js
    sed -i.bak "s#SERVER#$SERVER1#g" $S9S_TMPDIR/init_shard.js
    sed -i.bak "s#MONGOPORT#$MONGODB_PORT#g" $S9S_TMPDIR/init_shard.js
    remote_copy $S9S_TMPDIR/init_shard.js  ${xhost} $S9S_TMPDIR
    remote_cmd ${xhost} "sync"
    remote_cmd $xhost "$MONGODB_BASEDIR/bin/mongo  127.0.0.1:${MONGOS_PORT} --quiet $S9S_TMPDIR/init_shard.js"
    echo "creating shard"
    sleep 10
}


function start_node
{
    local xHNAME=$1
    local xNODEID=$2    
    local cmdline=$(get_field $xNODEID "cmdline")
    local pidfile=$(get_field $xNODEID "pidfilepath")
    remote_cmd_nofail $xHNAME "sh -c '$cmdline'"
    set_nodestate $xNODEID 3	    
}


function stop_node
{
    local xHNAME=$1
    local xNODEID=$2    
    local cmdline=$(get_field $xNODEID "cmdline")
    local pidfile=$(get_field $xNODEID "pidfilepath")

    if [ -z "$cmdline" ]; then 
	echo "could not read cmdline argument"
	exit 1
    fi

    if [ -z "$pidfile" ]; then 
	echo "could not read pidfile argument"
	exit 1
    fi

    PID=$(remote_cmd_getreply $xHNAME "cat $pidfile")
    if [ -n "$PID" ]; then
	remote_cmd_nofail $xHNAME "kill -15 $PID"
    fi
    remote_cmd_nofail $xHNAME "pkill -15 -f \"${cmdline}.*\""
    max_retries=60
    retry=0
    while [ $retry -lt $max_retries  ];
    do
	STATE=$(check_node_up $xHNAME $xNODEID)
	if [ "$STATE" = "DOWN" ]; then
	    set_nodestate $xNODEID 12
	    return
	else
	    retry=`expr $retry + 1`		  		  
	fi
	sleep 1
    done
    echo "failed to stop node $cmdline"
    exit 1
}


function set_nodestate
{
    NODEID=$1
    STATE=$2
    if [ -n "$STATE" ]; then
	QUERY="UPDATE mongodb_server SET status=$STATE WHERE cid=$CLUSTER_ID AND nodeid=$NODEID"  
	`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT --auto-vertical-output -Bse "$QUERY"`
	if [ $? -ne 0 ]; then
	    echo "setting state failed"
	    exit 1
	fi
    fi
}


function get_field
{
    NODEID=$1
    FIELD=$2
    QUERY="SELECT $FIELD FROM mongodb_server WHERE cid=$CLUSTER_ID AND nodeid=$NODEID"  
    DATA=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`
    if [ $? -ne 0 ]; then
	echo "getting field failed"
	exit 1
    fi
    echo "$DATA"
}



function check_node_up
{
    local xHNAME=$1
    local xNODEID=$2    
    local cmdline=$(get_field $xNODEID "cmdline")
    local pidfile=$(get_field $xNODEID "pidfilepath")
    if [ "$cmdline" != "" ]; then	
	PID=$(remote_cmd_getreply $xHNAME "pgrep -f  '${cmdline}'")
	if [ -n "$PID" ]; then
	    echo "UP"
	    return
	fi
	echo "DOWN"
    else
	## If we dont have a cmdline, then assume the node is up
	echo "UP"
    fi
}

function check_node_up2
{
    local xHNAME=$1
    local xCOND=$2
    PID=$(remote_cmd_getreply $xHNAME "pgrep -f \"${xCOND}\"")
    if [ -n "$PID" ]; then
	echo "UP"
	return
    fi
    echo "DOWN"
}

function get_cluster_state
{
    local QUERY="SELECT status FROM cluster_state WHERE id=$CLUSTER_ID"
    local STATE=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$STATE"
}

function wait_for_cluster_state
{
    local STATE=$1
    retry=0
    while [ $retry -lt $CLUSTER_STATE_TIMEOUT ];
    do
	XSTATE=$(get_cluster_state)
	if [ "$STATE" = "$XSTATE" ]; then
	    echo $XSTATE
	    return 0
	fi
	retry=`expr $retry + 1`
	sleep 1
    done
    echo "Failed to reach cluster state $STATE - timed out after $CLUSTER_STATE_TIMEOUT (CLUSTER_STATE_TIMEOUT)"
    exit 1
}


function get_servers
{
    local TYPE=$1
    local QUERY="select group_concat(concat(m.hostname, ',', m.nodeid, ',', m.port) SEPARATOR ' ') from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID  and m.node_type=n.id and n.name in ('$TYPE') and (m.status=1 or m.status=2 or m.status=7)"
    local SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$SERVER_LIST"
}


function get_all_servers
{
    local QUERY="select group_concat(concat(m.hostname, ',', m.nodeid, ',', m.port) SEPARATOR ' ') from mongodb_server m WHERE m.cid=$CLUSTER_ID"
    local SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$SERVER_LIST"
}

function get_nodeid
{
    local QUERY="select m.nodeid from mongodb_server m WHERE m.cid=$CLUSTER_ID AND m.hostname='$1' AND m.port=$2"
    local SERVER=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$SERVER"
}

function get_servers2
{
    local TYPE=$1
    local QUERY="select group_concat(concat(m.hostname, ',', m.nodeid, ',', m.port) SEPARATOR ' ') from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID  and m.node_type=n.id and n.name in ($TYPE) and (m.status=1 or m.status=2 or m.status=7)"
    local SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$SERVER_LIST"
}

function get_servers_in_state
{
    local TYPE=$1
    local STATE=$2
    local QUERY="select group_concat(concat(m.hostname, ',', m.nodeid, ',', m.port) SEPARATOR ' ') from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID  and m.node_type=n.id and n.name='$TYPE' and m.status in ($STATE)"
    local SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$SERVER_LIST"
}

function get_count_up
{
    local QUERY="select count(nodeid) from mongodb_server m WHERE m.cid=$CLUSTER_ID  and (m.status=1 or m.status=2 )"
    local CNT=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$CNT"
}

function get_count_all
{
    local QUERY="select count(nodeid) from mongodb_server m WHERE m.cid=$CLUSTER_ID AND m.status<>7"
    local CNT=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$CNT"
}


function get_count_type
{
    local TYPE=$1
    local QUERY="select count(m.serverid) from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID  and m.node_type=n.id and n.name='$TYPE'"
    local CNT=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$CNT"
}


function get_rs_victims
{
    local QUERY="select group_concat(concat(hostname, ',', nodeid, ',', port, ',', rs_name, ',', dbpath) SEPARATOR ' ') from (select m.hostname, m.nodeid, m.port, m.rs_name, m.dbpath from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID and m.node_type=n.id and n.name='shardsvr' and m.status=2 group by rs_name) t"  
    local RS_SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$RS_SERVER_LIST"
}

function get_rs_primary
{
    XRS_NAME=$1
    local QUERY="select group_concat(concat(hostname, ',', nodeid, ',', port, ',', rs_name, ',', dbpath) SEPARATOR ' ') from (select m.hostname, m.nodeid, m.port, m.rs_name, m.dbpath from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID and m.node_type=n.id and n.name='shardsvr' and m.status=1 and rs_name='$XRS_NAME') t"  
    local RS_SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$RS_SERVER_LIST"
}

function get_configsvr
{
    local QUERY="select group_concat(concat(m.hostname, ',', m.nodeid,',', m.port) SEPARATOR ' ') from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID  and m.node_type=n.id and n.name='configsvr' and m.status=1 LIMIT 1"  
    local CFG_SERVER_LIST=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$CFG_SERVER_LIST"
}



function get_mongos_server
{
    local QUERY="select m.hostname from mongodb_server m,mongodb_nodetype_map n WHERE m.cid=$CLUSTER_ID  and m.node_type=n.id and n.name='mongos' and m.status=1 LIMIT 1"  
    local MONGOS_SERVER=`$MYSQL_BIN $MYSQL_OPTS --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=127.0.0.1 --port=$CMON_DB_PORT -N -Bse "$QUERY"`   
    if [ $? -ne 0 ]; then
	echo "getting server list failed"
	exit 1
    fi
    echo "$MONGOS_SERVER"
}


function stop_cluster
{        
    args=`getopt i: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    


  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --stop-cluster -i <cluster id>"
      exit 1
  fi
  init
  load_opts $CLUSTER_ID
  
  echo "Are you sure you want to stop the cluster? Enter YES to stop the cluster."
  read ANSWER

  if [ "$ANSWER" != "YES" ]; then
      echo "$ANSWER was pressed ($ANSWER != YES ), not removing file"
      exit 0
  fi    
  

  TYPE="mongos"
  SERVER_LIST=$(get_servers $TYPE)
  if [ "$SERVER_LIST" != "NULL" ]; then
      for r in $SERVER_LIST
      do
	  HNAME=`echo $r | awk -F ',' '{print $1};'`
	  NODEID=`echo $r | awk -F ',' '{print $2};'`
	  stop_node $HNAME $NODEID
      done
  fi

  TYPE="configsvr"
  SERVER_LIST=$(get_servers $TYPE)
  if [ "$SERVER_LIST" != "NULL" ]; then
      for r in $SERVER_LIST
      do
	  HNAME=`echo $r | awk -F ',' '{print $1};'`
	  NODEID=`echo $r | awk -F ',' '{print $2};'`
	  stop_node $HNAME $NODEID	  
      done
  fi

  TYPE="shardsvr"
  SERVER_LIST=$(get_servers $TYPE)
  if [ "$SERVER_LIST" != "NULL" ]; then
      for r in $SERVER_LIST
      do
	  HNAME=`echo $r | awk -F ',' '{print $1};'`
	  NODEID=`echo $r | awk -F ',' '{print $2};'`
	  stop_node $HNAME $NODEID
      done
  fi
}



function start_cluster
{        
    args=`getopt i: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    


  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --start-cluster -i <cluster id>"
      exit 1
  fi
  init
  load_opts $CLUSTER_ID
  
  echo "Are you sure you want to start the cluster? Enter YES to start the cluster."
  read ANSWER

  if [ "$ANSWER" != "YES" ]; then
      echo "$ANSWER was pressed ($ANSWER != YES ), not removing file"
      exit 0
  fi    
  
  TYPE="shardsvr"
  SERVER_LIST=$(get_servers_in_state $TYPE "8,11,12")
  if [ "$SERVER_LIST" != "NULL" ]; then
      for r in $SERVER_LIST
      do
	  HNAME=`echo $r | awk -F ',' '{print $1};'`
	  NODEID=`echo $r | awk -F ',' '{print $2};'`
	  start_node $HNAME $NODEID
      done
  fi

  TYPE="configsvr"
  SERVER_LIST=$(get_servers_in_state $TYPE "8,11,12")
  if [ "$SERVER_LIST" != "NULL" ]; then
      for r in $SERVER_LIST
      do
	  HNAME=`echo $r | awk -F ',' '{print $1};'`
	  CONFIG_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  start_node $HNAME $CONFIG_NODEID
      done
  fi
  TYPE="mongos"
  SERVER_LIST=$(get_servers_in_state $TYPE "8,11,12")
  if [ "$SERVER_LIST" != "NULL" ]; then
      for r in $SERVER_LIST
      do
	  HNAME=`echo $r | awk -F ',' '{print $1};'`
	  NODEID=`echo $r | awk -F ',' '{print $2};'`
	  start_node $HNAME $NODEID
      done
  fi
  set_nodestate $CONFIG_NODEID 8
  
}

function set_backup_status
{
    local status=$1
    local error=$2
    cat $BACKUP_LOGFILE | sed  -e "s/'/\\\'/g" -e 's/"/\\"/g' > /tmp/s9s_mongodb_backup_log_escaped
    LOG_CONTENT=`cat /tmp/s9s_mongodb_backup_log_escaped`
    QUERY="UPDATE mongodb_backup SET status='$status', error=$error, logfile='$LOG_CONTENT' WHERE backupid=$BACKUP_ID and cid=$CLUSTER_ID"
    $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"
   SIZE=`stat --printf='%s' /tmp/s9s_mongodb_backup_log_escaped`
   CC_NAME=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD  --host=127.0.0.1 --port=$CMON_DB_PORT -e "SHOW GLOBAL VARIABLES LIKE 'hostname'" | awk '{print $2;}'`
   QUERY="REPLACE INTO cmon_host_log(cid, hostname, filename, result_len, result, report_ts,description, tag) VALUES ($CID,'$CC_NAME', '$BACKUP_LOGFILE', $SIZE, \"$LOG_CONTENT\", NOW(), '$BACKUP_LOGFILE', 's9s_mongodb_backup')"
   $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log

}

function set_backup_size
{
    local storagesize=$1
    QUERY="UPDATE mongodb_backup SET size='$storagesize' WHERE backupid=$BACKUP_ID and cid=$CLUSTER_ID"
    $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"
}



function backup_cluster
{        
    args=`getopt i:b:j: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
	    -b)
		XBACKUPDIR="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    


  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --backup -i <cluster id>"
      exit 1
  fi
  init
  load_opts $CLUSTER_ID

  if [ -n "$XBACKUPDIR" ]; then
      BACKUPDIR=$XBACKUPDIR;
  fi
  echo $BACKUPDIR
  
  if [ ! -f $MONGODB_BASEDIR/bin/mongo ]; then
      log_job_message "Could not find $MONGODB_BASEDIR/bin/mongo on the controller, please install the mongo client utilities" 1
      log_job "FAILED" "Backup Failed" 1
      write_email "Backup failed" "Could not find $MONGODB_BASEDIR/bin/mongo on the controller, please install the mongo client utilities"
      exit 1
  fi

  echo "starting mongodb shard backup" > $BACKUP_LOGFILE
  sleep 5
  log_job_message "Starting backup" 0
  log_job "RUNNING" "Backup started" 0
  

  cat << 'EOF' > /tmp/startBalancer.js
   db=db.getMongo().getDB("config"); 
   db.settings.update( { _id: "balancer" }, { $set : { stopped: false } } , true );
EOF

  cat << 'EOF' > /tmp/stopBalancer.js
   db=db.getMongo().getDB("config"); 
   db.settings.update( { _id: "balancer" }, { $set : { stopped: true } } , true );
EOF

  cat << 'EOF' > /tmp/lock.js
   db.fsyncLock();
EOF

  cat << 'EOF' > /tmp/unlock.js
   db.fsyncUnlock();
EOF
 
  CNT_UP=$(get_count_up)
  CNT_ALL=$(get_count_all)
  if [ $CNT_UP -ne $CNT_ALL ]; then
      log_job_message "All servers are not started - refusing to start backup." 1
      log_job "FAILED" "Backup Failed" 1
      write_email "Backup failed" "All servers are not started - refusing to start backup"
      exit 1
  fi

  CNT_CFG=$(get_count_type "configsvr")
  CNT_MONGOS=$(get_count_type "mongos")
  REPLICASET_BACKUP=0
  if [ $CNT_CFG -eq 0 ]; then
      REPLICASET_BACKUP=1
      log_job_message "Backing up a replica set" 0
  else
      REPLICASET_BACKUP=0
      log_job_message "Backing up a shard cluster" 0
  fi

  BACKUP_ID=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "select ifnull(max(backupid)+1,1) from mongodb_backup"`


  curr_time=`date +%Y-%m-%d-%H-%M-%S`;  
  BACKUPDIR=$BACKUPDIR/$curr_time  

  QUERY="UPDATE mongodb_backup SET status='failed', error=1 WHERE status='running'"
  $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"
 
  QUERY="INSERT INTO mongodb_backup(backupid, cid, directory, error, status, report_ts, backup_type, backup_method, logfile, cc_storage, storage_host) VALUES($BACKUP_ID, $CID, '$BACKUPDIR','','running',now(), 'full', 'mongodump','', 1, '$HOSTNAME')"
  
  $MYSQL_BIN --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"
  if [ $? -ne 0 ]; then
      log_job_message  "Failed to create backup record" 1
      log_job "FAILED" "Backup Failed" 1
      write_email "Backup failed" "Failed to create backup record"
      exit 1
  fi


  ##  STOP CONFIG SVR
  if [ $REPLICASET_BACKUP -eq 0 ]; then      
      TYPE="configsvr"
      log_job_message "Stopping config server" 0
      SERVER_LIST=$(get_servers $TYPE)
      if [ "$SERVER_LIST" != "NULL" ]; then
	  for r in $SERVER_LIST
	  do	  
	      CONFIG_SVR_HNAME=`echo $r | awk -F ',' '{print $1};'`
	      CONFIG_SVR_NODEID=`echo $r | awk -F ',' '{print $2};'`
	      stop_node $CONFIG_SVR_HNAME $CONFIG_SVR_NODEID	 
	      break
	  done
      else
	  set_backup_status 'failed' 1
	  log_job_message "No cfg servers found" 1
	  log_job "FAILED" "Backup Failed" 1
	  write_email "Backup failed" "No configuration servers found."
	  exit 1
      fi
  fi

  TYPE="shardsvr"
  RS_VICTIM_LIST=$(get_rs_victims $TYPE)
  if [ "$RS_VICTIM_LIST" != "NULL" ]; then
      for r in $RS_VICTIM_LIST
      do	 
	  RS_HNAME=`echo $r | awk -F ',' '{print $1};'`
	  RS_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  RS_PORT=`echo $r | awk -F ',' '{print $3};'`
          echo "Locking shard server $RS_HNAME"
	  #stop_node $RS_HNAME $RS_NODEID	
	  log_job_message "Locking shard server $RS_NAME" 0 
	  echo "LC_ALL=C  $MONGODB_BASEDIR/bin/mongo $RS_HNAME:$RS_PORT"
          LC_ALL=C $MONGODB_BASEDIR/bin/mongo $RS_HNAME:$RS_PORT /tmp/lock.js >> $BACKUP_LOGFILE  2>&1
          if [ $? -ne 0 ];  then
	      echo "Failed to lock shard server" >> $BACKUP_LOGFILE
	      log_job_message "Failed to lock shard  $RS_HNAME:$RS_PORT" 1
	      write_email "Backup failed" "Failed to lock shard  $RS_HNAME:$RS_PORT" 
	      set_backup_status 'failed' 1
	      exit 1
	  fi	  
      done
  else
      log_job_message "No rs servers" 1
      set_backup_status 'failed' 1
      log_job "FAILED" "Backup Failed" 1
      write_email "Backup failed" "No replica set servers found." 
      exit 1
  fi

  if [ $REPLICASET_BACKUP -eq 0 ]; then      
      
      TYPE="configsvr"
      CFG_VICTIM_LIST=$(get_configsvr $TYPE)
      if [ "$CFG_VICTIM_LIST" != "NULL" ]; then
	  for r in $CFG_VICTIM_LIST
	  do	  
	      CFG_HNAME=`echo $r | awk -F ',' '{print $1};'`
	      CFG_NODEID=`echo $r | awk -F ',' '{print $2};'`
	      CFG_PORT=`echo $r | awk -F ',' '{print $3};'`
	      log_job_message "Backing up config server $CFG_HNAME:$CFG_PORT" 0
	      mkdir -p $BACKUPDIR/config/
	      if [ "$VENDOR" = "tokutek" ]; then
		  LC_ALL=C  $MONGODB_BASEDIR/bin/mongodump --db config -h $CFG_HNAME:$CFG_PORT -o $BACKUPDIR  >> $BACKUP_LOGFILE  2>&1
	      else
		  LC_ALL=C  $MONGODB_BASEDIR/bin/mongodump --journal --db config -h $CFG_HNAME:$CFG_PORT -o $BACKUPDIR  >> $BACKUP_LOGFILE  2>&1
	      fi
	      break
	  done
      else
	  set_backup_status 'failed' 1
	  log_job_message "No cfg server to dump from" 1
	  write_email "Backup failed" "No config servers found to run mongodump on." 
	  log_job "FAILED" "Backup Failed" 1
	  exit 1
      fi
  fi

  if [ "$RS_VICTIM_LIST" != "NULL" ]; then
      for r in $RS_VICTIM_LIST
      do	  
	  RS_HNAME=`echo $r | awk -F ',' '{print $1};'`
	  RS_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  RS_PORT=`echo $r | awk -F ',' '{print $3};'`
	  RS_NAME=`echo $r | awk -F ',' '{print $4};'`
	  RS_DB_PATH=`echo $r | awk -F ',' '{print $5};'`
	  remote_cmd $RS_HNAME "mkdir -p $BACKUPDIR/$RS_NAME"
	  mkdir -p $BACKUPDIR/$RS_NAME
	  log_job_message "Backing up shard server $RS_HNAME" 0
	  if [ "$VENDOR" = "tokutek" ]; then
	      LC_ALL=C  $MONGODB_BASEDIR/bin/mongodump  -h $RS_HNAME:$RS_PORT -o $BACKUPDIR/$RS_NAME  >> $BACKUP_LOGFILE  2>&1  &
	  else
	      LC_ALL=C  $MONGODB_BASEDIR/bin/mongodump --journal  -h $RS_HNAME:$RS_PORT -o $BACKUPDIR/$RS_NAME  >> $BACKUP_LOGFILE  2>&1  &
	  fi
	      
      done
  else
      set_backup_status 'failed' 1
      log_job_message "No rs servers" 1
      write_email "Backup failed" "No replica set servers found to run mongodump on." 
      log_job "FAILED" "Backup Failed" 1
      exit 1
  fi
 
  
  FAIL=0
  for job in `jobs -p`
  do
      wait $job || let "FAIL+=1"
  done 
  
  if [ "$FAIL" == "0" ];
  then
      log_job_message "All shards completed backup" 0
  else
      log_job_message "$FAIL shards failed to complete backup - backup failed" 1
      log_job "FAILED" "Backup Failed" 1
  fi
  

  storagesize=`du -b -s $BACKUPDIR | awk '{print $1;}'`
  set_backup_size $storagesize
  sleep 10

  
  echo "Unlocking shard server" >> $BACKUP_LOGFILE
  log_job_message "Unlocking shard server" 0
  if [ "$RS_VICTIM_LIST" != "NULL" ]; then
      for r in $RS_VICTIM_LIST
      do	  
	  RS_HNAME=`echo $r | awk -F ',' '{print $1};'`
	  RS_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  RS_PORT=`echo $r | awk -F ',' '{print $3};'`
    	  log_job_message "Unlocking shard server $RS_HNAME" 0
          #stop_node $RS_HNAME $RS_NODEID
          LC_ALL=C  $MONGODB_BASEDIR/bin/mongo $RS_HNAME:$RS_PORT /tmp/unlock.js >> $BACKUP_LOGFILE  2>&1
          if [ $? -ne 0 ];  then
	      echo "Failed to unlock shard server" >> $BACKUP_LOGFILE
	      log_job_message "Failed to unlock shard  $RS_HNAME:$PORT" 1
	      set_backup_status 'failed' 1
	      write_email "Backup failed" "Failed to unlock shard  $RS_HNAME:$PORT" 
	      log_job "FAILED" "Backup Failed" 1
	      exit 1
          fi
	 # start_node $RS_HNAME $RS_NODEID	 
      done
  else
      set_backup_status 'failed' 1
      log_job "FAILED" "Backup Failed" 1
      exit 1
  fi

  if [ $REPLICASET_BACKUP -eq 0 ]; then
      echo "Starting config server" >> $BACKUP_LOGFILE
      
  ##  START CONFIG SVR
      start_node $CONFIG_SVR_HNAME $CONFIG_SVR_NODEID	 
      
  ## WAIT FOR CLUSTER TO BECOME STARTED"
      echo "Waiting for cluster to become STARTED" >> $BACKUP_LOGFILE
  fi
  wait_for_cluster_state "STARTED"
  sleep 30
  
  if [ $REPLICASET_BACKUP -eq 0 ]; then
  ## START BALANCER
      TYPE="mongos"
      OK=0
      MONGOS_LIST=$(get_servers $TYPE)
      if [ "$MONGOS_LIST" != "NULL" ]; then
	  echo "Starting balancer" >> $BACKUP_LOGFILE
	  for r in $MONGOS_LIST
	  do	  
	      HNAME=`echo $r | awk -F ',' '{print $1};'`
	      NODEID=`echo $r | awk -F ',' '{print $2};'`
	      PORT=`echo $r | awk -F ',' '{print $3};'`
	      while [ $retry -lt $START_BALANCER_TIMEOUT ]; 
	      do
		  LC_ALL=C  $MONGODB_BASEDIR/bin/mongo $HNAME:$PORT /tmp/startBalancer.js >> $BACKUP_LOGFILE  2>&1 
		  if [ $? -ne 0 ];  then
		      echo "Failed to start balancer" >> $BACKUP_LOGFILE
		      log_job_message "Failed to start balancer (1) - could not connect to mongos  $HNAME:$PORT" 1
		      write_email "Backup failed" "Failed to start balancer (1) - could not connect to mongos  $HNAME:$PORT" 
		      set_backup_status 'failed' 1
		      log_job "FAILED" "Backup Failed" 1
		      exit 1
		  fi
		  sleep 1
		  BAL_STATE=`$MONGODB_BASEDIR/bin/mongo $HNAME:$PORT --quiet --eval "sh.getBalancerState()"`
		  if [ $? -ne 0 ];  then
		      echo "Failed to start balancer" >> $BACKUP_LOGFILE
		      write_email "Backup failed" "Failed to start balancer (2) - could not connect to mongos  $HNAME:$PORT" 
		      log_job_message "Failed to start balancer (2) - could not connect to mongos  $HNAME:$PORT" 1
		      set_backup_status 'failed' 1
		      log_job "FAILED" "Backup Failed" 1
		      exit 1
		  fi
		  if [ "$BAL_STATE" = "true" ]; then
		      echo "Started balancer" >> $BACKUP_LOGFILE
		      log_job_message "Balancer started" 0
		      OK=1
		      break
		  else
		      retry=`expr $retry + 1`
		  fi
	      done	  
	      break
	  done
	  if [ $OK -eq 0 ]; then
	      log_job_message "Failed to start balancer (3)" 1
	      write_email "Backup failed" "Failed to start balancer (2)"
	      set_backup_status 'failed' 1
	      log_job "FAILED" "Backup Failed" 1
	      exit 1
	  fi
      fi
  fi

  if [ $FAIL -eq 0 ]; then
     set_backup_status 'completed' 0
     write_email "Backup Completed" "Backup successful"
     log_job "FINISHED" "Backup Successful" 0
  else
     set_backup_status 'failed' 1 
     write_email "Backup Failed" "Backup Failed"
     log_job "FAILED" "Backup Failed" 1
  fi
  exit 0
}



## ADD SECONDARY
function add_secondary
{        
    args=`getopt i:N:P:j:d:h:k: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
	    -N)
		RS_NAME="$2"; shift;
		shift;;
	    -h)
		XHOSTNAME="$2"; shift;
		shift;;
	    -P)
		XMONGODB_PORT="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    


  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --add-secondary -i <cluster id>"
      exit 1
  fi
  init 
  if [ -z "$RS_NAME" ]; then
      echo "s9s_mongodb_admin --add-secondary -N <replica set name to add the secondary to>"
      exit 1
  fi

  if [ -z "$XHOSTNAME" ]; then
      echo "s9s_mongodb_admin --add-secondary -h <secondary hostname>"
      exit 1
  fi

  echo $CLUSTER_ID
  load_opts $CLUSTER_ID

  if [ -n "$XMONGODB_PORT" ]; then
      MONGODB_PORT=$XMONGODB_PORT
  fi
  
  if [ -z "$MONGODB_PORT" ]; then
      echo "s9s_mongodb_admin --add-secondary -P <secondary port>"
      exit 1
  fi
  

  SERVERID=$(check_host_exists $CLUSTER_ID $XHOSTNAME)
  if [ $? -eq 1 ]; then
      log_job_message "Server $XHOSTNAME does not exists - you must add the host in the UI first" 1
      exit 1
  fi
  if [ ! -f  $S9S_TMPDIR/mongodb.conf ]; then
      log_job_message "$S9S_TMPDIR/mongodb.conf does not exist" 1
      exit 1
  fi

  if [ ! -s  $S9S_TMPDIR/mongodb.conf ]; then
      log_job_message "$S9S_TMPDIR/mongodb.conf exists but is 0 bytes" 1
      exit 1
  fi
  
  TMP_DATADIR=`grep dbpath $S9S_TMPDIR/mongodb.conf |grep -v '#'  |tr -d ' ' | awk -F = '{print $2}'`
  if [ -n "$TMP_DATADIR" ]; then
      MONGODB_DATADIR=$TMP_DATADIR
  fi
  
  remote_copy ${S9S_TMPDIR}/mongodb.conf $XHOSTNAME ${S9S_TMPDIR}/mongodb.${MONGODB_PORT}.conf
  remote_cmd $XHOSTNAME "mv -f  ${S9S_TMPDIR}/mongodb.${MONGODB_PORT}.conf  ${MONGODB_CONFIGDIR}/mongodb.${MONGODB_PORT}.conf"

  cat << 'EOF' > /tmp/addSecondary.js
   rs.add('XHOSTNAME:MONGODB_PORT')

EOF


  TYPE="shardsvr"
  RS_VICTIM_LIST=$(get_rs_primary $RS_NAME)
  if [ "$RS_VICTIM_LIST" != "NULL" ]; then
      for r in $RS_VICTIM_LIST
      do	 
	  RS_HNAME=`echo $r | awk -F ',' '{print $1};'`
	  RS_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  RS_PORT=`echo $r | awk -F ',' '{print $3};'`
      done
  else
      log_job_message "No PRIMARY found in replica set $RS_NAME" 1      
      exit 1
  fi
  
  start_mongodb $XHOSTNAME
  
  sed  -i.bak "s#XHOSTNAME#${XHOSTNAME}#g"  /tmp/addSecondary.js
  sed  -i.bak "s#MONGODB_PORT#${MONGODB_PORT}#g"  /tmp/addSecondary.js
  
    echo "Creating secondary"
    ${MONGODB_BASEDIR}/bin/mongo ${RS_HNAME}:${RS_PORT} --quiet  /tmp/addSecondary.js
    

    if [ $? -ne 0 ]; then
	echo "failed to create secondary on $XHOSTNAME"
	exit 1
    fi      
    
    
    x=`cat $configfile |grep ${XHOSTNAME}:${MONGODB_PORT}`
    if [ -z "$x" ]; then
	sed  -i.bak "s#mongodb_server_addresses=.*#mongodb_server_addresses=${mongodb_server_addresses},$XHOSTNAME:$MONGODB_PORT#g"  $configfile
	if [ $? -ne 0 ]; then
	    echo "updating $configfile failed."
	    exit 1
	fi      
    fi
    
  QUERY="REPLACE INTO mongodb_server(cid, serverid , hostname, port, report_ts, node_type, rs_name, arbiter)"
  QUERY="$QUERY VALUES ($CLUSTER_ID, $SERVERID, '$XHOSTNAME', $MONGODB_PORT, now(), 0,'$RS_NAME',0)"
  $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
  if [ $? -ne 0 ]; then
      echo "Failed: $QUERY"
      echo "You must restart cmon contoller: service cmon restart"
      exit 1
  fi
  
  
  exit 0
}

## ADD ARBITER
function add_arbiter
{        
    args=`getopt i:N:P:j:d:h:k: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
	    -N)
		RS_NAME="$2"; shift;
		shift;;
	    -h)
		ARBIT_HOSTNAME="$2"; shift;
		shift;;
	    -P)
		ARBIT_PORT="$2"; shift;
		shift;;
	    -d)
		ARBIT_DATADIR="$2"; shift;
		shift;;
	    -k)
		KEYFILE="--keyFile=$2 --auth"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    


  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --add-arbiter -i <cluster id>"
      exit 1
  fi
  init 
  if [ -z "$RS_NAME" ]; then
      echo "s9s_mongodb_admin --add-arbiter -N <replica set name to add the arbiter to>"
      exit 1
  fi

  if [ -z "$ARBIT_HOSTNAME" ]; then
      echo "s9s_mongodb_admin --add-arbiter -h <arbiter hostname>"
      exit 1
  fi

  if [ -z "$ARBIT_PORT" ]; then
      echo "s9s_mongodb_admin --add-arbiter -P <arbiter port>"
      exit 1
  fi

  if [ -z "$ARBIT_DATADIR" ]; then
      echo "s9s_mongodb_admin --add-arbiter -d <arbiter datadir>"
      exit 1
  fi



  echo $CLUSTER_ID
  load_opts $CLUSTER_ID

  SERVERID=$(check_host_exists $CLUSTER_ID $ARBIT_HOSTNAME)
  if [ $? -eq 1 ]; then
      log_job_message "Server $ARBIT_HOSTNAME does not exists - you must add it in the UI first" 1
      exit 1
  fi

    
  ARBIT_DATADIR=$ARBIT_DATADIR/$ARBIT_PORT
  
  cat << 'EOF' > /tmp/addArbiter.js
   rs.add('ARBIT_HOSTNAME:ARBIT_PORT',true)

EOF


  TYPE="shardsvr"
  RS_VICTIM_LIST=$(get_rs_primary $RS_NAME)
  if [ "$RS_VICTIM_LIST" != "NULL" ]; then
      for r in $RS_VICTIM_LIST
      do	 
	  RS_HNAME=`echo $r | awk -F ',' '{print $1};'`
	  RS_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  RS_PORT=`echo $r | awk -F ',' '{print $3};'`
      done
  else
      log_job_message "No PRIMARY found in replica set $RS_NAME" 1      
      exit 1
  fi
  
  remote_cmd $ARBIT_HOSTNAME "mkdir -p $ARBIT_DATADIR"
  remote_cmd $ARBIT_HOSTNAME "mkdir -p /var/log/mongodb"
  remote_cmd $ARBIT_HOSTNAME "mkdir -p /var/run/mongodb"
  PIDFILE="/var/run/mongodb/arbiter_${ARBIT_PORT}.pid"
  if [ "$VENDOR" != "tokutek" ]; then
     MORE_OPTS="--smallfiles"
  fi

  CMDLINE="$MONGODB_BASEDIR/bin/mongod --port $ARBIT_PORT --dbpath $ARBIT_DATADIR --replSet $RS_NAME  --logpath /var/log/mongodb/arbiter_${ARBIT_PORT}.log --pidfilepath $PIDFILE $MORE_OPTS $KEYFILE $AUTH"
  CMDLINE=`echo $CMDLINE | sed 's/[ \t]*$//'`
  remote_cmd3 $ARBIT_HOSTNAME "$CMDLINE"
  
  sed  -i.bak "s#ARBIT_HOSTNAME#${ARBIT_HOSTNAME}#g"  /tmp/addArbiter.js
  sed  -i.bak "s#ARBIT_PORT#${ARBIT_PORT}#g"  /tmp/addArbiter.js
  
    max_retries=60
    retry=0
    echo "Waiting for arbitrator to become reachable"
    sleep 1
    while [ $retry -lt $max_retries  ];
    do
	mystate=`${MONGODB_BASEDIR}/bin/mongo  $ARBIT_HOSTNAME:$ARBIT_PORT --quiet  --eval "printjson(db.serverStatus().ok)" 2>&1`
	echo $mystate
	if [ "$mystate" = "1" ]; then
	    break
	fi   
	echo -n "."
	sleep 1
	retry=`expr $retry + 1`
    done
    echo ""
    echo "Creating arbiter"
    ${MONGODB_BASEDIR}/bin/mongo ${RS_HNAME}:${RS_PORT} --quiet  /tmp/addArbiter.js
    

    if [ $? -ne 0 ]; then
	echo "failed to create arbiter on $ARBIT_HOSTNAME"
	exit 1
    fi      
    
   # max_retries=60
   # retry=0
   # echo "Waiting for arbiter to join"
   # while [ $retry -lt $max_retries  ];
   # do
#	mystate=`${MONGODB_BASEDIR}/bin//mongo  $ARBIT_HOSTNAME:$ARBIT_PORT --quiet  --eval "printjson(rs.status().myState)" 2>&1`
#	if [ "$mystate" = "7" ]; then
#	    break
#	fi   
#	echo -n "."
#	sleep 1
#	retry=`expr $retry + 1`
 #   done
  #  echo ""
    
   # if [ $mystate -ne 7 ]; then
#	echo "failed to create arbiter on $ARBIT_HOSTNAME"
#	exit 1
#    fi
    
    mystate=6

    x=`cat $configfile |grep ${ARBIT_HOSTNAME}:${ARBIT_PORT}`
    if [ -z "$x" ]; then 
	y=`cat $configfile |grep mongoarbiter_server_addresses`
	if [ -z "$y" ]; then
	    echo "mongoarbiter_server_addresses=${ARBIT_HOSTNAME}:${ARBIT_PORT}" >> $configfile
	else
	    sed  -i.bak "s#mongoarbiter_server_addresses=.*#mongoarbiter_server_addresses=$ARBIT_HOSTNAME:$ARBIT_PORT#g"  $configfile
	fi
    else
	sed  -i.bak "s#mongoarbiter_server_addresses=.*#mongoarbiter_server_addresses=${mongoarbiter_server_addresses},$ARBIT_HOSTNAME:$ARBIT_PORT#g"  $configfile
    fi
    rm ${configfile}*bak
  QUERY="REPLACE INTO mongodb_server(cid, serverid , hostname, port, report_ts, status, arbiter, hidden, votes, slave_delay, username, password, node_type, rs_name, dbpath,logpath, shardsvr, pidfilepath,cmdline)"
  QUERY="$QUERY VALUES ($CLUSTER_ID, $SERVERID, '$ARBIT_HOSTNAME', $ARBIT_PORT, now(), $mystate, 1, 0,1,0,'','',3,'$RS_NAME', '$ARBIT_DATADIR', '/var/log/mongodb/arbiter_${ARBIT_PORT}.log', 0, '$PIDFILE', '$CMDLINE')"
  $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT  -e "$QUERY" 2>&1 >/tmp/err.log
  if [ $? -ne 0 ]; then
      echo "Failed: $QUERY"
      echo "You must restart cmon contoller: service cmon restart"
      exit 1
  fi
  
  
  
  exit 0
}



## REMOVE RS MEMBER
function remove_member
{        
    args=`getopt i:N:P:j:d:h: $*`
    set -- $args
    for i
    do
	case "$i" in
	    -i)
		CLUSTER_ID="$2"; shift;
		shift;;
	    -j)
		JOBID="$2"; shift;
		shift;;
	    -N)
		RS_NAME="$2"; shift;
		shift;;
	    -h)
		XHOSTNAME="$2"; shift;
		shift;;
	    -P)
		XPORT="$2"; shift;
		shift;;
	    -d)
		ARBIT_DATADIR="$2"; shift;
		shift;;
            --)
		shift; break;;
	esac
    done    


  if [ -z "$CLUSTER_ID" ]; then
      echo "s9s_mongodb_admin --remove-member -i <cluster id>"
      exit 1
  fi
  init
  if [ -z "$RS_NAME" ]; then
      echo "s9s_mongodb_admin --remove-member -N <replica set name to add the arbiter to>"
      exit 1
  fi

  if [ -z "$XHOSTNAME" ]; then
      echo "s9s_mongodb_admin --remove-member -h <arbiter hostname>"
      exit 1
  fi

  if [ -z "$XPORT" ]; then
      echo "s9s_mongodb_admin --remove-member -P <arbiter port>"
      exit 1
  fi


  echo $CLUSTER_ID
  load_opts $CLUSTER_ID

  SERVERID=$(check_host_exists $CLUSTER_ID $XHOSTNAME)
  if [ $? -eq 1 ]; then
      log_job_message "Server $XHOSTNAME does not exists - you must add it in the UI first" 1
      exit 1
  fi
  cat << 'EOF' > /tmp/remove.js
   rs.remove("ARBIT_HOSTNAME:ARBIT_PORT")
EOF


  TYPE="shardsvr"
  RS_VICTIM_LIST=$(get_rs_primary $RS_NAME)
  if [ "$RS_VICTIM_LIST" != "NULL" ]; then
      for r in $RS_VICTIM_LIST
      do	 
	  RS_HNAME=`echo $r | awk -F ',' '{print $1};'`
	  RS_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  RS_PORT=`echo $r | awk -F ',' '{print $3};'`
      done
  else
      log_job_message "No PRIMARY found in replica set $RS_NAME" 1      
      exit 1
  fi

  TYPE="'shardsvr','arbiter'"
  SERVER_LIST=$(get_servers2 $TYPE)
  FOUND=0
#  if [ "$SERVER_LIST" != "NULL" ]; then
#      for r in $SERVER_LIST
#      do
#	  HNAME=`echo $r | awk -F ',' '{print $1};'`
#	  NODEID=`echo $r | awk -F ',' '{print $2};'`
#	  PORT=`echo $r | awk -F ',' '{print $3};'`
#	  if [ "$HNAME" = "$XHOSTNAME" ] && [ "$PORT" = "$XPORT" ]; then
#	      stop_node $HNAME $NODEID
#	      FOUND=1
#	  fi
#     done
#  else
#      exit 1
#  fi  


## if [ $FOUND -eq 0 ]; then
#     echo "$XHOSTNAME:$XPORT was not found in mongodb_server, cannot remove."
#     exit 1
# fi

  sleep 2
  TYPE="shardsvr"
  RS_VICTIM_LIST=$(get_rs_primary $RS_NAME)
  if [ "$RS_VICTIM_LIST" != "NULL" ]; then
      for r in $RS_VICTIM_LIST
      do	 
	  RS_HNAME=`echo $r | awk -F ',' '{print $1};'`
	  RS_NODEID=`echo $r | awk -F ',' '{print $2};'`
	  RS_PORT=`echo $r | awk -F ',' '{print $3};'`
      done
  else
      log_job_message "No PRIMARY found in replica set $RS_NAME" 1      
      exit 1
  fi

###
  NODEID=$(get_nodeid $XHOSTNAME $XPORT)

  if [ -z "$NODEID" ] || [ "$NODEID" = "NULL" ]; then 
      log_job_message "Failed to get NODEID" 1      
      exit 1
  fi
  STATE=$(check_node_up $XHOSTNAME $NODEID)

  if [ "$STATE" != "DOWN" ]; then
      OUT=`${MONGODB_BASEDIR}/bin/mongo ${XHOSTNAME}:${XPORT} --quiet --eval "db.shutdownServer()" `
      if [ $? -ne 0 ];  then
	  log_job_message "Could not shutdown server" 1      
	  if [ echo '$OUT' | grep 'connect failed' ]; then
	      exit 1
	  fi
      fi
  fi

  sed  -i.bak "s#ARBIT_HOSTNAME#${XHOSTNAME}#g"  /tmp/remove.js
  sed  -i.bak "s#ARBIT_PORT#${XPORT}#g"  /tmp/remove.js
  
  ${MONGODB_BASEDIR}/bin/mongo ${RS_HNAME}:${RS_PORT} /tmp/remove.js
  
  if [ $? -ne 0 ]; then
      sleep 1
      ${MONGODB_BASEDIR}/bin/mongo  ${RS_HNAME}:${RS_PORT} /tmp/remove.js
      if [ $? -ne 0 ]; then
	  ${MONGODB_BASEDIR}/bin/mongo   ${RS_HNAME}:${RS_PORT} --eval "rs.remove('$XHOSTNAME:$XPORT')"
	  if [ $? -ne 0 ]; then
	      echo "failed to remove $XHOSTNAME:$XPORT"
	      exit 1
	  fi
      fi
  fi      
  
  QUERY="DELETE FROM mongodb_server WHERE cid=$CLUSTER_ID AND hostname='$XHOSTNAME' AND port=$XPORT"
  $MYSQL_BIN  -B -N  --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"    
  if [ $? -ne 0 ]; then
      echo "Failed: $QUERY"
      echo "You must restart cmon contoller: service cmon restart"
      exit 1
  fi

  QUERY="SELECT GROUP_CONCAT(concat(hostname,':',port) ORDER BY serverid) FROM mongodb_server WHERE cid=$CLUSTER_ID and node_type=0 "
  SERVERS=`$MYSQL_BIN $MYSQL_OPTS -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`    
  if [ $? -ne 0 ]; then
      log_job_message "Failed to get mongodb_server list" 1
      exit 1
  fi
  if [ "$SERVERS" = "NULL" ]; then
      SERVERS=""
  fi
  sed -i.bak "s#mongodb_server_addresses=.*#mongodb_server_addresses=${SERVERS}#g" $configfile
  rm /etc/cmon*bak
  QUERY="SELECT GROUP_CONCAT(concat(hostname,':',port) ORDER BY serverid) FROM mongodb_server WHERE cid=$CLUSTER_ID and node_type=1"
  SERVERS=`$MYSQL_BIN $MYSQL_OPTS -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`    
  if [ $? -ne 0 ]; then
      log_job_message "Failed to get mongodb_server list" 1
      exit 1
  fi
  if [ "$SERVERS" = "NULL" ]; then
      SERVERS=""
  fi
  sed -i.bak "s#mongocfg_server_addresses=.*#mongocfg_server_addresses=${SERVERS}#g" $configfile
  rm /etc/cmon*bak


  QUERY="SELECT GROUP_CONCAT(concat(hostname,':',port) ORDER BY serverid) FROM mongodb_server WHERE cid=$CLUSTER_ID and node_type=2"
  SERVERS=`$MYSQL_BIN $MYSQL_OPTS -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`    
  if [ $? -ne 0 ]; then
      log_job_message "Failed to get mongodb_server list (2)" 1
      exit 1
  fi
  if [ "$SERVERS" = "NULL" ]; then
      SERVERS=""
  fi
  sed -i.bak "s#mongos_server_addresses=.*#mongos_server_addresses=${SERVERS}#g" $configfile
  rm /etc/cmon*bak
      

  QUERY="SELECT GROUP_CONCAT(concat(hostname,':',port) ORDER BY serverid) FROM mongodb_server WHERE cid=$CLUSTER_ID and node_type=3"
  SERVERS=`$MYSQL_BIN $MYSQL_OPTS -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`    
  if [ $? -ne 0 ]; then
      log_job_message "Failed to get mongodb_server list (3)" 1
      exit 1
  fi
  if [ "$SERVERS" = "NULL" ]; then
      SERVERS=""
  fi
  sed -i.bak "s#mongoarbiter_server_addresses=.*#mongoarbiter_server_addresses=${SERVERS}#g" $configfile
  rm /etc/cmon*bak
  exit 0
}

function check_host_exists
{
    local TARGET_CID=$1
    local TARGET_HOSTNAME=$2
    local QUERY="select id from cmon.hosts where (hostname='$TARGET_HOSTNAME' OR ip='$TARGET_HOSTNAME') AND cid=${TARGET_CID}"
    HID=`$MYSQL_BIN -N -B -A --user=$CMON_USER --password=$CMON_PASSWORD --database=$CMON_DB_DB --host=$CMON_DB_HOST --port=$CMON_DB_PORT -e "$QUERY"`
    if [ $? -ne 0 ]; then	
        log_job_message "$QUERY  - failed!" 1
        exit 1
    fi

    if [ -z "$HID" ]; then
        log_job_message "Server $TARGET_HOSTNAME does not exists - you must add it in the UI first" 1
	return 1
    fi

    if [ "$HID" = "NULL" ]; then
        log_job_message "Server $TARGET_HOSTNAME does not exists - you must add it in the UI first" 1
        exit 1
    fi
    echo $HID    
}

ARG=$1
shift
case $ARG in
    --remove-lockfile)
	remove_lockfile $*
	;;
    --stop-cluster)
	stop_cluster $*
	;;
    --start-cluster)
	start_cluster $*
	;;
    --backup)
	backup_cluster $*
	;;
    --add-secondary)
	add_secondary $*
	;;
    --add-arbiter)
	add_arbiter $*
	;;
    --remove-member)
	remove_member $*
	;;
    --add-shard)
	add_shard $*
	;;
    --install-mongodb)
	install_mongodb $*
	;;
    --upgrade-mongodb)
	upgrade_mongodb $*
	;;
    --rolling-restart)
	rolling_restart $*
	;;
    *)
    echo "Usage:"
    echo "bash ./s9s_mongodb_admin <--remove-lockfile|start-cluster|stop-cluster|--backup|--remove-member|--add-arbiter|upgrade-mongodb|--rolling-restart> <options follows>"
    exit 1
    ;;
esac
