bootstrap.sh - salt-formulas/salt-formulas-scripts - Gitiles

 #!/bin/bash -e

 # bootstrap.sh

 # Installs Salt and configure minimal SaltMaster or Minion to be used with:
 # - http://github.com/salt-formulas-scripts
 # - http://github.com/salt-formulas/salt-formula-salt (salt.master sls)

 # TODO:
 # - use PPA repository as formula source
 # - support for spm/yum


 # Source specific env vars.
 # shopt -u dotglob
 export RECLASS_ROOT=${RECLASS_ROOT:-/srv/salt/reclass}
 function source_local_envs() {
   for path in /tmp/kitchen /srv/salt . ${RECLASS_ROOT}/classes/cluster ${RECLASS_ROOT}/classes/cluster/${CLUSTER_NAME}; do
     export $(find $path -maxdepth 1 -name '*.env' 2> /dev/null | xargs --no-run-if-empty cat ) > /dev/null
   done;
 }
 source_local_envs

 ##########################################
 # Set defaults env variables

 if [[ $DEBUG =~ ^(True|true|1|yes)$ ]]; then
     set -x
     SALT_LOG_LEVEL="--state-verbose=true -ldebug"
 fi

 export MAGENTA='\033[0;95m'
 export YELLOW='\033[1;33m'
 export BLUE='\033[0;35m'
 export CYAN='\033[0;96m'
 export RED='\033[0;31m'
 export NC='\033[0m' # No Color'

 export LC_ALL=C
 export SALT_LOG_LEVEL="--state-verbose=false -lerror"
 export SALT_OPTS="${SALT_OPTS:- --timeout=120 --state-output=changes --retcode-passthrough --force-color $SALT_LOG_LEVEL }"
 export SALT_STATE_RETRY=${SALT_STATE_RETRY:-3}
 export BOOTSTRAP_SALTSTACK=${BOOTSTRAP_SALTSTACK:-True}
 export BOOTSTRAP_SALTSTACK_OPTS=${BOOTSTRAP_SALTSTACK_OPTS:- -dX stable 2016.3 }


 # salt apt repository
 test -e /etc/lsb-release && eval $(cat /etc/lsb-release)
 which lsb_release && DISTRIB_CODENAME=${DISTRIB_CODENAME:-$(lsb_release -cs)}
 #
 export APT_REPOSITORY="deb [arch=amd64] http://apt-mk.mirantis.com/${DISTRIB_CODENAME} ${DISTRIB_REVISION:-stable} salt"
 export APT_REPOSITORY_GPG=${APT_REPOSITORY_GPG:-http://apt-mk.mirantis.com/public.gpg}

 # reclass
 export RECLASS_ADDRESS=${RECLASS_ADDRESS:-https://github.com/salt-formulas/openstack-salt.git} # https/git

 # formula
 export FORMULAS_BASE=${FORMULAS_BASE:-https://github.com/salt-formulas}
 export FORMULAS_PATH=${FORMULAS_PATH:-/usr/share/salt-formulas}
 export FORMULAS_BRANCH=${FORMULAS_BRANCH:-master}
 export FORMULAS_SOURCE=${FORMULAS_SOURCE:-pkg} # pkg/git
 # essential set of formulas (known to by used on cfg01 node for most setups)
 FORMULAS_SALT_MASTER=${FORMULAS_SALT_MASTER:- $EXTRA_FORMULAS memcached openssh ntp nginx collectd sensu heka sphinx mysql grafana libvirt rsyslog glusterfs postfix xtrabackup freeipa prometheus telegraf elasticsearch kibana rundeck devops-portal}
 # minimal set of formulas for salt-master bootstrap
 declare -a FORMULAS_SALT_MASTER=(linux reclass salt git $(echo $FORMULAS_SALT_MASTER))
 export FORMULAS_SALT_MASTER

 # system / host
 export HOSTNAME=${HOSTNAME:-cfg01}
 export DOMAIN=${DOMAIN:-bootstrap.local}

 # salt
 export SALT_MASTER=${SALT_MASTER:-127.0.0.1} # ip or fqdn
 export MINION_ID=${MINION_ID:-${HOSTNAME}.${DOMAIN}}

 # saltstack
 BOOTSTRAP_SALTSTACK=${BOOTSTRAP_SALTSTACK:-True}
 BOOTSTRAP_SALTSTACK_OPTS=${BOOTSTRAP_SALTSTACK_OPTS:- -dX stable 2016.3 }
 SALT_SOURCE=${SALT_SOURCE:-pkg}
 SALT_VERSION=${SALT_VERSION:-latest}

 # environment
 if [ "$FORMULAS_SOURCE" == "git" ]; then
   SALT_ENV=${SALT_ENV:-dev}
 elif [ "$FORMULAS_SOURCE" == "pkg" ]; then
   SALT_ENV=${SALT_ENV:-prd}
 fi
 eval $(cat /etc/*release 2> /dev/null)
 PLATFORM_FAMILY=$(echo ${ID_LIKE// */} | tr A-Z a-z)
 case $PLATFORM_FAMILY in
   debian )
       PKGTOOL="$SUDO apt-get"
       test ${VERSION_ID//\.*/} -ge 16 && {
         SVCTOOL=service
       } || { SVCTOOL=service
       }
     ;;
   rhel )
       PKGTOOL="$SUDO yum"
       test ${VERSION_ID//\.*/} -ge 7 && {
         SVCTOOL=systemctl
       } || { SVCTOOL=service
       }
     ;;
 esac

 export PLATFORM_FAMILY
 export PKGTOOL
 export SVCTOOL

 ##########################################
 # FUNCTIONS

 log_info() {
     echo -e "${YELLOW}[INFO] $* ${NC}"
 }

 log_warn() {
     echo -e "${MAGENTA}[WARN] $* ${NC}"
 }

 log_err() {
     echo -e "${RED}[ERROR] $* ${NC}" >&2
 }

 configure_pkg_repo()
 {

     case $PLATFORM_FAMILY in
       debian)
           if [ -n "$APT_REPOSITORY_PPA" ]; then
             which add-apt-repository || $SUDO apt-get install -y software-properties-common
             $SUDO add-apt-repository -y ppa:${APT_REPOSITORY_PPA}
           else
             echo -e  "$APT_REPOSITORY " | $SUDO tee /etc/apt/sources.list.d/bootstrap.list >/dev/null
             curl -sL $APT_REPOSITORY_GPG | $SUDO apt-key add -
           fi
           $SUDO apt-get clean
           $SUDO apt-get update
         ;;
       rhel)
           $SUDO yum install -y https://repo.saltstack.com/yum/redhat/salt-repo-latest-1.el${VERSION_ID}.noarch.rpm
           $SUDO yum clean all
         ;;
     esac

 }

 _atexit() {
     RETVAL=$?
     trap true INT TERM EXIT

     if [ $RETVAL -ne 0 ]; then
         log_err "Execution failed"
     else
         log_info "Execution successful"
     fi
     return $RETVAL
 }

 retry() {
     local tries
     if [[ $1 =~ ^[0-9]+$ ]]; then
         tries=$1; shift
     else
         tries=3
     fi
     for i in $(seq 1 $tries); do
         "$@" && return 0 || sleep $i
     done
     return 1
 }

 function clone_reclass() {
   if [ ! -d ${RECLASS_ROOT} ]; then
     # No reclass at all, clone from given address
     ssh-keyscan -H github.com >> ~/.ssh/known_hosts || true
     if echo ${RECLASS_BRANCH:-master} | egrep -q "^refs"; then
         git clone ${RECLASS_ADDRESS} ${RECLASS_ROOT}
         cd ${RECLASS_ROOT}
         git fetch ${RECLASS_ADDRESS} ${RECLASS_BRANCH:-master} && git checkout FETCH_HEAD
         cd -
     else
         git clone -b ${RECLASS_BRANCH:-master} ${RECLASS_ADDRESS} ${RECLASS_ROOT};
     fi;
   fi;
 }


 ##########################################
 # Main calls

 system_config_minion() {
     log_info "System configuration salt minion"
 }

 system_config_master() {
     log_info "System configuration salt master"

     # salt-formulas custom modules dependencies, etc:
     $SUDO $PKGTOOL install -y iproute2 curl sudo apt-transport-https python-psutil python-apt python-m2crypto python-oauth python-pip &>/dev/null

     $SUDO mkdir -p $RECLASS_ROOT/classes/service
     $SUDO mkdir -p /root/.ssh
     ssh-keyscan -H github.com >> ~/.ssh/known_hosts || true
     echo -e "Host *\n\tStrictHostKeyChecking no\n" | $SUDO tee ~/.ssh/config >/dev/null
     echo -e "Host *\n\tStrictHostKeyChecking no\n" | $SUDO tee /root/.ssh/config >/dev/null
     echo "127.0.1.2  salt" | $SUDO tee -a /etc/hosts >/dev/null

     which reclass || $SUDO $PKGTOOL install -y reclass

     which reclass-salt || {
       test -e /usr/share/reclass/reclass-salt && {
         ln -fs /usr/share/reclass/reclass-salt /usr/bin
       }
     }
 }

 configure_salt_master()
 {

   echo "Configuring salt-master ..."

   [ ! -d /etc/salt/master.d ] && mkdir -p /etc/salt/master.d
   cat <<-EOF > /etc/salt/master.d/master.conf
 	file_roots:
 	  base:
 	  - /usr/share/salt-formulas/env
 	  prd:
 	  - /srv/salt/env/prd
 	  dev:
 	  - /srv/salt/env/dev
 	pillar_opts: False
 	open_mode: True
 	reclass: &reclass
 	  storage_type: yaml_fs
 	  inventory_base_uri: ${RECLASS_ROOT}
 	ext_pillar:
 	  - reclass: *reclass
 	master_tops:
 	  reclass: *reclass
 EOF

   echo "Configuring reclass ..."

   [ ! -d /etc/reclass ] && mkdir /etc/reclass
   cat <<-EOF > /etc/reclass/reclass-config.yml
 	storage_type: yaml_fs
 	pretty_print: True
 	output: yaml
 	inventory_base_uri: ${RECLASS_ROOT}
 EOF

   clone_reclass
   # override some envs from cluster level *.env, use with care
   source_local_envs

   cd ${RECLASS_ROOT}
   if [ ! -d ${RECLASS_ROOT}/classes/system/linux ]; then
     # Possibly subrepo checkout needed
     git submodule update --init --recursive
   fi

   #sed -ie "s#\(reclass_data_revision.\).*#\1 $RECLASS_BRANCH#" $(find nodes -name ${MASTER_HOSTNAME}.yml|tail -n1)

   mkdir -vp ${RECLASS_ROOT}/nodes
   CONFIG=$(find ${RECLASS_ROOT}/nodes -name ${MINION_ID}.yml| grep yml | tail -n1)
   CONFIG=${CONFIG:-${RECLASS_ROOT}/nodes/${MINION_ID}.yml}
   if [[ $SALT_MASTER_BOOTSTRAP_MINIMIZED =~ ^(True|true|1|yes)$ || ! -f "${CONFIG}" ]]; then
   cat <<-EOF > ${CONFIG}
 	classes:
 	- service.git.client
 	- system.linux.system.single
 	- system.salt.master.single
 	- system.salt.master.$FORMULAS_SOURCE
 	- system.reclass.storage.salt
 	parameters:
 	  _param:
 	    reclass_data_repository: "$RECLASS_ADDRESS"
 	    reclass_data_revision: ${RECLASS_BRANCH:-master}
 	    salt_formula_branch: ${SALT_FORMULAS_BRANCH:-master}
 	    reclass_config_master: $SALT_MASTER
 	    single_address: $SALT_MASTER
 	    salt_master_host: $SALT_MASTER
 	    salt_master_base_environment: $SALT_ENV
 	    linux_system_codename: $DISTRIB_CODENAME
 	  linux:
 	    system:
 	      name: $MINION_ID
 	      domain: $DOMAIN
 	# ########
 EOF

     if [ "$SALT_VERSION" == "latest" ]; then
       VERSION=""
     else
       VERSION="version: $SALT_VERSION"
     fi

     cat <<-EOF >> ${CONFIG}
 		  salt:
 		    master:
 		      accept_policy: open_mode
 		      source:
 		        engine: $SALT_SOURCE
 		        $VERSION
 		    minion:
 		      source:
 		        engine: $SALT_SOURCE
 		        $VERSION
 		# ########
 		# vim: ft=yaml sw=2 ts=2 sts=2
 EOF
   fi
 }

 configure_salt_minion()
 {
   [ ! -d /etc/salt/minion.d ] && mkdir -p /etc/salt/minion.d
   cat <<-EOF > /etc/salt/minion.d/minion.conf
 	master: $SALT_MASTER
 	id: $MINION_ID
 	EOF
 }


 install_salt_master_pkg()
 {
     echo -e "\nPreparing base OS repository ...\n"

     configure_pkg_repo

     echo -e "\nInstalling salt master ...\n"

     case $PLATFORM_FAMILY in
       debian)
           $SUDO apt-get install -y git
 	  which reclass || $SUDO apt install -qqq -y reclass
           curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
         ;;
       rhel)
           yum install -y git
           which reclass || $SUDO yum install -y reclass
           curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
         ;;
     esac

     which reclass-salt || {
       test -e /usr/share/reclass/reclass-salt && {
         ln -fs /usr/share/reclass/reclass-salt /usr/bin
       }
     }

     configure_salt_master

     echo -e "\nRestarting services ...\n"
     [ -f /etc/salt/pki/minion/minion_master.pub ] && rm -f /etc/salt/pki/minion/minion_master.pub
     $SVCTOOL salt-master restart
 }

 install_salt_master_pip()
 {
     echo -e "\nPreparing base OS repository ...\n"

     case $PLATFORM_FAMILY in
       debian)
           $SUDO apt-get install -y python-pip python-dev zlib1g-dev git
 	  which reclass || $SUDO apt-get install -y reclass
         ;;
       rhel)
 	  $SUDO yum install -y git
 	  which reclass || $SUDO yum install -y reclass
         ;;
     esac

     echo -e "\nInstalling salt master ...\n"
     # TODO: replace with saltstack bootstrap script

     if [ "$SALT_VERSION" == "latest" ]; then
       pip install salt
     else
       pip install salt==$SALT_VERSION
     fi

     curl -Lo /etc/init.d/salt-master https://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-master.init && chmod 755 /etc/init.d/salt-master
     ln -s /usr/local/bin/salt-master /usr/bin/salt-master

     which reclass-salt || {
       test -e /usr/share/reclass/reclass-salt && {
         ln -fs /usr/share/reclass/reclass-salt /usr/bin
       }
     }

     configure_salt_master

     echo -e "\nRestarting services ...\n"
     [ -f /etc/salt/pki/minion/minion_master.pub ] && rm -f /etc/salt/pki/minion/minion_master.pub
     $SVCTOOL salt-master restart
 }


 install_salt_minion_pkg()
 {

     configure_pkg_repo

     echo -e "\nInstalling salt minion ...\n"

     case $PLATFORM_FAMILY in
       debian)
           curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
       ;;
       rhel)
           curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
       ;;
     esac


     configure_salt_minion

     $SVCTOOL salt-minion restart
 }

 install_salt_minion_pip()
 {
     echo -e "\nInstalling salt minion ...\n"

     curl -Lo /etc/init.d/salt-minion https://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-minion.init && chmod 755 /etc/init.d/salt-minion
     ln -s /usr/local/bin/salt-minion /usr/bin/salt-minion

     configure_salt_minion
     $SVCTOOL salt-minion restart
 }


 install_salt_formula_pkg()
 {
     configure_pkg_repo

     case $PLATFORM_FAMILY in
       debian)
           echo "Configuring necessary formulas ..."

           [ ! -d ${RECLASS_ROOT}/classes/service ] && mkdir -p ${RECLASS_ROOT}/classes/service
           # Set essentials if FORMULAS_SALT_MASTER is not defined at all
           [ -z ${FORMULAS_SALT_MASTER+x} ] && declare -a FORMULAS_SALT_MASTER=("linux" "reclass" "salt" "memcached")
           for formula_service in "${FORMULAS_SALT_MASTER[@]}"; do
               echo -e "\nConfiguring salt formula ${formula_service} ...\n"
               [ ! -d "${FORMULAS_PATH}/env/${formula_service}" ] && \
                   if ! $SUDO apt-get install -y salt-formula-${formula_service}; then
                     echo -e "\nInstall salt-formula-${formula_service} failed.\n"
                     exit 1
                   fi
               [ ! -L "${RECLASS_ROOT}/classes/service/${formula_service}" ] && \
                   ln -sf ${FORMULAS_PATH}/reclass/service/${formula_service} ${RECLASS_ROOT}/classes/service/${formula_service}
           done
         ;;
       rhel)
         # TODO
       ;;
     esac

     [ ! -d /srv/salt/env ] && mkdir -p /srv/salt/env || echo ""
     [ ! -L /srv/salt/env/prd ] && ln -s ${FORMULAS_PATH}/env /srv/salt/env/prd || echo ""
 }

 install_salt_formula_git()
 {
     echo "Configuring necessary formulas ..."

     [ ! -d ${RECLASS_ROOT}/classes/service ] && mkdir -p ${RECLASS_ROOT}/classes/service
     # Set essentials if FORMULAS_SALT_MASTER is not defined at all
     [ -z ${FORMULAS_SALT_MASTER+x} ] && declare -a FORMULAS_SALT_MASTER=("linux" "reclass" "salt" "memcached")
     for formula_service in "${FORMULAS_SALT_MASTER[@]}"; do
         echo -e "\nConfiguring salt formula ${formula_service} ...\n"
         _BRANCH=${FORMULAS_BRANCH}
         [ ! -d "${FORMULAS_PATH}/env/_formulas/${formula_service}" ] && {
             if ! git ls-remote --exit-code --heads ${FORMULAS_BASE}/salt-formula-${formula_service}.git ${_BRANCH}; then
               # Fallback to the master branch if the branch doesn't exist for this repository
               _BRANCH=master
             fi
             if ! git clone ${FORMULAS_BASE}/salt-formula-${formula_service}.git ${FORMULAS_PATH}/env/_formulas/${formula_service} -b ${_BRANCH}; then
               echo -e "\nCloning of ${FORMULAS_BASE}/salt-formula-${formula_service}.git failed.\n"
               exit 1
             fi
           } || {
             cd ${FORMULAS_PATH}/env/_formulas/${formula_service};
             git fetch origin/${_BRANCH} || git fetch --all
             git checkout ${_BRANCH} && git pull || git pull;
             cd -
         }
         [ ! -L "/usr/share/salt-formulas/env/${formula_service}" ] && \
             ln -sf ${FORMULAS_PATH}/env/_formulas/${formula_service}/${formula_service} /usr/share/salt-formulas/env/${formula_service}
         [ ! -L "${RECLASS_ROOT}/classes/service/${formula_service}" ] && \
             ln -sf ${FORMULAS_PATH}/env/_formulas/${formula_service}/metadata/service ${RECLASS_ROOT}/classes/service/${formula_service}
     done

     [ ! -d /srv/salt/env ] && mkdir -p /srv/salt/env || echo ""
     [ ! -L /srv/salt/env/dev ] && ln -s /usr/share/salt-formulas/env /srv/salt/env/dev || echo ""
 }


 saltmaster_bootstrap() {

     log_info "Salt master setup"
     test -n "$MASTER_HOSTNAME" || exit 1

     clone_reclass
     # override some envs from cluster level *.env, use with care
     source_local_envs

     pgrep salt-master | sed /$$/d | xargs --no-run-if-empty -i{} $SUDO kill -9 {}
     pkill -9 salt-minion
     test -e ${SCRIPTS}/.salt-master-setup.sh.passed || {
         export SALT_MASTER=localhost
         export MINION_ID=${MASTER_HOSTNAME}
         if ! [[ $DEBUG =~ ^(True|true|1|yes)$ ]]; then
           SALT_MASTER_SETUP_OUTPUT='/dev/stdout'
         fi
         # call local "setup() master"
         #if ! $SUDO ${SCRIPTS}/salt-master-setup.sh master &> ${SALT_MASTER_SETUP_OUTPUT:-/tmp/salt-master-setup.log}; then
         if ! setup master; then
           #cat /tmp/salt-master-setup.log
           log_err "salt master setup() failed."
           exit 1
         else
           $SUDO touch ${SCRIPTS}/.salt-master-setup.sh.passed
         fi
     }

     log_info "Clean up generated"
     cd $RECLASS_ROOT
     $SUDO rm -rf $RECLASS_ROOT/nodes/_generated/*

     log_info "Re/starting salt services"
     pgrep salt-master | sed /$$/d | xargs --no-run-if-empty -i{} $SUDO kill -9 {}
     pkill -9 salt-minion
     sleep 1
     $SUDO service salt-master restart
     $SUDO service salt-minion restart
     sleep 10
 }

 # Init salt master
 saltmaster_init() {

     log_info "Runing saltmaster states"
     test -n "$MASTER_HOSTNAME" || exit 1

     set -e
     $SUDO salt-call saltutil.sync_all >/dev/null

     # TODO: Placeholder update saltmaster spec (nodes/FQDN.yml) to be able to bootstrap with minimal configuration
     # (ie: with linux, git, salt formulas)

     #log_info "Verify SaltMaster, before salt-master is fully initialized"
     #if ! $SUDO reclass-salt -p ${MASTER_HOSTNAME} &> /tmp/${MASTER_HOSTNAME}.pillar;then
     #   log_warn "Node verification before initialization failed."; cat /tmp/${MASTER_HOSTNAME}.pillar;
     #fi

     log_info "State: salt.master.env"
     if ! $SUDO salt-call ${SALT_OPTS} -linfo state.apply salt.master.env; then
       log_err "State salt.master.env failed, keep your eyes wide open."
     fi

     log_info "State: salt.master.pillar"
     retry ${SALT_STATE_RETRY} $SUDO salt-call ${SALT_OPTS} state.apply salt.master.pillar pillar='{"reclass":{"storage":{"data_source":{"engine":"local"}}}}'
     # Note: sikp reclass data dir states
     #       in order to avoid pull from configured repo/branch

     # Revert temporary SaltMaster minimal configuration, if any
     pushd $RECLASS_ROOT
     if [ $(git diff --name-only nodes | sort | uniq | wc -l) -ge 1 ]; then
       git status || true
       log_warn "Locally modified $RECLASS_ROOT/nodes found. (Possibly salt-master minimized setup from salt-master-setup.sh call)"
       log_info "Checkout HEAD state of $RECLASS_ROOT/nodes/*."
       git checkout -- $RECLASS_ROOT/nodes || true
       log_info "Re-Run states: salt.master.env and salt.master.pillar according the HEAD state."
       log_info "State: salt.master.env"
       if ! $SUDO salt-call ${SALT_OPTS} -linfo state.apply salt.master.env; then
         log_err "State salt.master.env failed, keep your eyes wide open."
       fi
       log_info "State: salt.master.pillar"
       retry ${SALT_STATE_RETRY} $SUDO salt-call ${SALT_OPTS} state.apply salt.master.pillar pillar='{"reclass":{"storage":{"data_source":{"engine":"local"}}}}'
     fi
     popd

     log_info "State: salt.master.storage.node"
     set +e
     $SUDO salt-call ${SALT_OPTS} state.apply reclass.storage.node
     ret=$?
     set -e

     if [[ $ret -eq 2 ]]; then
         log_err "State reclass.storage.node failed with exit code 2 but continuing."
     elif [[ $ret -ne 0 ]]; then
         log_err "State reclass.storage.node failed with exit code $ret"
         exit 1
     fi

     log_info "Re/starting salt services"
     $SUDO sed -i 's/^master:.*/master: localhost/' /etc/salt/minion.d/minion.conf
     $SUDO service salt-minion restart >/dev/null
     $SUDO salt-call ${SALT_OPTS} saltutil.sync_all >/dev/null

     verify_salt_master
     set +e

 }


 function verify_salt_master() {
     set -e

     log_info "Verify Salt master"
     test -n "$MASTER_HOSTNAME" || exit 1

     if [[ $VERIFY_SALT_CALL =~ ^(True|true|1|yes)$ ]]; then
       $SUDO salt-call ${SALT_OPTS} --id=${MASTER_HOSTNAME} grains.item roles > /tmp/${MASTER_HOSTNAME}.grains.item.roles
       $SUDO salt-call ${SALT_OPTS} --id=${MASTER_HOSTNAME} state.show_lowstate > /tmp/${MASTER_HOSTNAME}.state.show_state
       $SUDO salt-call --no-color grains.items
       $SUDO salt-call --no-color pillar.data
     fi
     if ! $SUDO reclass --nodeinfo ${MASTER_HOSTNAME} > /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo; then
         log_err "For more details see full log /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo"
         exit 1
     fi
 }

 function verify_salt_minion() {
   set -e
   node=$1
   log_info "Verifying ${node}"
   if [[ $VERIFY_SALT_CALL =~ ^(True|true|1|yes)$ ]]; then
     $SUDO salt-call ${SALT_OPTS} --id=${node} grains.item roles > /tmp/${node}.grains.item.roles
     $SUDO salt-call ${SALT_OPTS} --id=${node} state.show_lowstate > /tmp/${node}.state.show_lowstate
   fi
   if ! $SUDO reclass --nodeinfo ${node} > /tmp/${node}.reclass.nodeinfo; then
       log_err "For more details see full log /tmp/${node}.reclass.nodeinfo"
       if [[ ${BREAK_ON_VERIFICATION_ERROR:-yes} =~ ^(True|true|1|yes)$ ]]; then
         exit 1
       fi
   fi
 }

 function verify_salt_minions() {
     #set -e
     NODES=$(find $RECLASS_ROOT/nodes/ -name "*.yml" | grep -v "cfg")
     log_info "Verifying minions: $(echo ${NODES}|xargs)"

     # Parallel
     #echo $NODES | parallel --no-notice -j 2 --halt 2 "verify_salt_minion \$(basename {} .yml) > {}.pillar_verify"
     #ls -lrta *.pillar_verify | tail -n 1 | xargs -n1 tail -n30

     function filterFails() {
         grep -v '/grains' | tee -a $1 | tail -n20
     }

     log_info "Verify nodes"
     passed=0
     for node in ${NODES}; do
         node=$(basename $node .yml)

         # filter first in cluster.. ctl-01, mon-01, etc..
         if [[ "${node//.*}" =~ 01 || "${node//.*}" =~ 02  ]] ;then
             verify_salt_minion ${node} || continue
         else
             echo Skipped $node.
         fi
         passed=$(($passed+1))
     done
     # fail on failures
     total=$(echo $NODES | xargs --no-run-if-empty -n1 echo |wc -l)
     test ! $passed -lt $total || log_err "Results: $passed of $total passed."
     test ! $passed -lt $total || {
       tail -n50 /tmp/*.pillar_verify
       return 1
     }
 }


 ##########################################
 # To install salt master/minon

 function install() {
   setup $@
 }

 function setup() {
   # CLI
   while [ x"$1" != x"" ]; do
     which curl &>/dev/null || $PKGTOOL -y install curl &>/dev/null

     case $1 in
         master )
           install_salt_master_$SALT_SOURCE
           install_salt_minion_$SALT_SOURCE
           install_salt_formula_$FORMULAS_SOURCE
           ;;
         minion )
           install_salt_minion_$SALT_SOURCE
           ;;
     esac
     shift
   done
   echo DONE
 }

 function bootstrap() {
   log_info "Bootstrap & verification of SaltMaster and configured minions."
   trap _atexit INT TERM EXIT

   system_config_master
   saltmaster_bootstrap &&\
   saltmaster_init &&\
   verify_salt_minions
 }

 function default() {
   bootstrap $@
 }


 ##########################################
 [[ "$0" != "$BASH_SOURCE" ]] || {
 # unless file is being sourced
   default $@
 }
	#!/bin/bash -e

	# bootstrap.sh

	# Installs Salt and configure minimal SaltMaster or Minion to be used with:
	# - http://github.com/salt-formulas-scripts
	# - http://github.com/salt-formulas/salt-formula-salt (salt.master sls)

	# TODO:
	# - use PPA repository as formula source
	# - support for spm/yum


	# Source specific env vars.
	# shopt -u dotglob
	export RECLASS_ROOT=${RECLASS_ROOT:-/srv/salt/reclass}
	function source_local_envs() {
	for path in /tmp/kitchen /srv/salt . ${RECLASS_ROOT}/classes/cluster ${RECLASS_ROOT}/classes/cluster/${CLUSTER_NAME}; do
	export $(find $path -maxdepth 1 -name '*.env' 2> /dev/null \| xargs --no-run-if-empty cat ) > /dev/null
	done;
	}
	source_local_envs

	##########################################
	# Set defaults env variables

	if [[ $DEBUG =~ ^(True\|true\|1\|yes)$ ]]; then
	set -x
	SALT_LOG_LEVEL="--state-verbose=true -ldebug"
	fi

	export MAGENTA='\033[0;95m'
	export YELLOW='\033[1;33m'
	export BLUE='\033[0;35m'
	export CYAN='\033[0;96m'
	export RED='\033[0;31m'
	export NC='\033[0m' # No Color'

	export LC_ALL=C
	export SALT_LOG_LEVEL="--state-verbose=false -lerror"
	export SALT_OPTS="${SALT_OPTS:- --timeout=120 --state-output=changes --retcode-passthrough --force-color $SALT_LOG_LEVEL }"
	export SALT_STATE_RETRY=${SALT_STATE_RETRY:-3}
	export BOOTSTRAP_SALTSTACK=${BOOTSTRAP_SALTSTACK:-True}
	export BOOTSTRAP_SALTSTACK_OPTS=${BOOTSTRAP_SALTSTACK_OPTS:- -dX stable 2016.3 }


	# salt apt repository
	test -e /etc/lsb-release && eval $(cat /etc/lsb-release)
	which lsb_release && DISTRIB_CODENAME=${DISTRIB_CODENAME:-$(lsb_release -cs)}
	#
	export APT_REPOSITORY="deb [arch=amd64] http://apt-mk.mirantis.com/${DISTRIB_CODENAME} ${DISTRIB_REVISION:-stable} salt"
	export APT_REPOSITORY_GPG=${APT_REPOSITORY_GPG:-http://apt-mk.mirantis.com/public.gpg}

	# reclass
	export RECLASS_ADDRESS=${RECLASS_ADDRESS:-https://github.com/salt-formulas/openstack-salt.git} # https/git

	# formula
	export FORMULAS_BASE=${FORMULAS_BASE:-https://github.com/salt-formulas}
	export FORMULAS_PATH=${FORMULAS_PATH:-/usr/share/salt-formulas}
	export FORMULAS_BRANCH=${FORMULAS_BRANCH:-master}
	export FORMULAS_SOURCE=${FORMULAS_SOURCE:-pkg} # pkg/git
	# essential set of formulas (known to by used on cfg01 node for most setups)
	FORMULAS_SALT_MASTER=${FORMULAS_SALT_MASTER:- $EXTRA_FORMULAS memcached openssh ntp nginx collectd sensu heka sphinx mysql grafana libvirt rsyslog glusterfs postfix xtrabackup freeipa prometheus telegraf elasticsearch kibana rundeck devops-portal}
	# minimal set of formulas for salt-master bootstrap
	declare -a FORMULAS_SALT_MASTER=(linux reclass salt git $(echo $FORMULAS_SALT_MASTER))
	export FORMULAS_SALT_MASTER

	# system / host
	export HOSTNAME=${HOSTNAME:-cfg01}
	export DOMAIN=${DOMAIN:-bootstrap.local}

	# salt
	export SALT_MASTER=${SALT_MASTER:-127.0.0.1} # ip or fqdn
	export MINION_ID=${MINION_ID:-${HOSTNAME}.${DOMAIN}}

	# saltstack
	BOOTSTRAP_SALTSTACK=${BOOTSTRAP_SALTSTACK:-True}
	BOOTSTRAP_SALTSTACK_OPTS=${BOOTSTRAP_SALTSTACK_OPTS:- -dX stable 2016.3 }
	SALT_SOURCE=${SALT_SOURCE:-pkg}
	SALT_VERSION=${SALT_VERSION:-latest}

	# environment
	if [ "$FORMULAS_SOURCE" == "git" ]; then
	SALT_ENV=${SALT_ENV:-dev}
	elif [ "$FORMULAS_SOURCE" == "pkg" ]; then
	SALT_ENV=${SALT_ENV:-prd}
	fi
	eval $(cat /etc/*release 2> /dev/null)
	PLATFORM_FAMILY=$(echo ${ID_LIKE// */} \| tr A-Z a-z)
	case $PLATFORM_FAMILY in
	debian )
	PKGTOOL="$SUDO apt-get"
	test ${VERSION_ID//\.*/} -ge 16 && {
	SVCTOOL=service
	} \|\| { SVCTOOL=service
	}
	;;
	rhel )
	PKGTOOL="$SUDO yum"
	test ${VERSION_ID//\.*/} -ge 7 && {
	SVCTOOL=systemctl
	} \|\| { SVCTOOL=service
	}
	;;
	esac

	export PLATFORM_FAMILY
	export PKGTOOL
	export SVCTOOL

	##########################################
	# FUNCTIONS

	log_info() {
	echo -e "${YELLOW}[INFO] $* ${NC}"
	}

	log_warn() {
	echo -e "${MAGENTA}[WARN] $* ${NC}"
	}

	log_err() {
	echo -e "${RED}[ERROR] $* ${NC}" >&2
	}

	configure_pkg_repo()
	{

	case $PLATFORM_FAMILY in
	debian)
	if [ -n "$APT_REPOSITORY_PPA" ]; then
	which add-apt-repository \|\| $SUDO apt-get install -y software-properties-common
	$SUDO add-apt-repository -y ppa:${APT_REPOSITORY_PPA}
	else
	echo -e "$APT_REPOSITORY " \| $SUDO tee /etc/apt/sources.list.d/bootstrap.list >/dev/null
	curl -sL $APT_REPOSITORY_GPG \| $SUDO apt-key add -
	fi
	$SUDO apt-get clean
	$SUDO apt-get update
	;;
	rhel)
	$SUDO yum install -y https://repo.saltstack.com/yum/redhat/salt-repo-latest-1.el${VERSION_ID}.noarch.rpm
	$SUDO yum clean all
	;;
	esac

	}

	_atexit() {
	RETVAL=$?
	trap true INT TERM EXIT

	if [ $RETVAL -ne 0 ]; then
	log_err "Execution failed"
	else
	log_info "Execution successful"
	fi
	return $RETVAL
	}

	retry() {
	local tries
	if [[ $1 =~ ^[0-9]+$ ]]; then
	tries=$1; shift
	else
	tries=3
	fi
	for i in $(seq 1 $tries); do
	"$@" && return 0 \|\| sleep $i
	done
	return 1
	}

	function clone_reclass() {
	if [ ! -d ${RECLASS_ROOT} ]; then
	# No reclass at all, clone from given address
	ssh-keyscan -H github.com >> ~/.ssh/known_hosts \|\| true
	if echo ${RECLASS_BRANCH:-master} \| egrep -q "^refs"; then
	git clone ${RECLASS_ADDRESS} ${RECLASS_ROOT}
	cd ${RECLASS_ROOT}
	git fetch ${RECLASS_ADDRESS} ${RECLASS_BRANCH:-master} && git checkout FETCH_HEAD
	cd -
	else
	git clone -b ${RECLASS_BRANCH:-master} ${RECLASS_ADDRESS} ${RECLASS_ROOT};
	fi;
	fi;
	}


	##########################################
	# Main calls

	system_config_minion() {
	log_info "System configuration salt minion"
	}

	system_config_master() {
	log_info "System configuration salt master"

	# salt-formulas custom modules dependencies, etc:
	$SUDO $PKGTOOL install -y iproute2 curl sudo apt-transport-https python-psutil python-apt python-m2crypto python-oauth python-pip &>/dev/null

	$SUDO mkdir -p $RECLASS_ROOT/classes/service
	$SUDO mkdir -p /root/.ssh
	ssh-keyscan -H github.com >> ~/.ssh/known_hosts \|\| true
	echo -e "Host *\n\tStrictHostKeyChecking no\n" \| $SUDO tee ~/.ssh/config >/dev/null
	echo -e "Host *\n\tStrictHostKeyChecking no\n" \| $SUDO tee /root/.ssh/config >/dev/null
	echo "127.0.1.2 salt" \| $SUDO tee -a /etc/hosts >/dev/null

	which reclass \|\| $SUDO $PKGTOOL install -y reclass

	which reclass-salt \|\| {
	test -e /usr/share/reclass/reclass-salt && {
	ln -fs /usr/share/reclass/reclass-salt /usr/bin
	}
	}
	}

	configure_salt_master()
	{

	echo "Configuring salt-master ..."

	[ ! -d /etc/salt/master.d ] && mkdir -p /etc/salt/master.d
	cat <<-EOF > /etc/salt/master.d/master.conf
	file_roots:
	base:
	- /usr/share/salt-formulas/env
	prd:
	- /srv/salt/env/prd
	dev:
	- /srv/salt/env/dev
	pillar_opts: False
	open_mode: True
	reclass: &reclass
	storage_type: yaml_fs
	inventory_base_uri: ${RECLASS_ROOT}
	ext_pillar:
	- reclass: *reclass
	master_tops:
	reclass: *reclass
	EOF

	echo "Configuring reclass ..."

	[ ! -d /etc/reclass ] && mkdir /etc/reclass
	cat <<-EOF > /etc/reclass/reclass-config.yml
	storage_type: yaml_fs
	pretty_print: True
	output: yaml
	inventory_base_uri: ${RECLASS_ROOT}
	EOF

	clone_reclass
	# override some envs from cluster level *.env, use with care
	source_local_envs

	cd ${RECLASS_ROOT}
	if [ ! -d ${RECLASS_ROOT}/classes/system/linux ]; then
	# Possibly subrepo checkout needed
	git submodule update --init --recursive
	fi

	#sed -ie "s#\(reclass_data_revision.\).*#\1 $RECLASS_BRANCH#" $(find nodes -name ${MASTER_HOSTNAME}.yml\|tail -n1)

	mkdir -vp ${RECLASS_ROOT}/nodes
	CONFIG=$(find ${RECLASS_ROOT}/nodes -name ${MINION_ID}.yml\| grep yml \| tail -n1)
	CONFIG=${CONFIG:-${RECLASS_ROOT}/nodes/${MINION_ID}.yml}
	if [[ $SALT_MASTER_BOOTSTRAP_MINIMIZED =~ ^(True\|true\|1\|yes)$ \|\| ! -f "${CONFIG}" ]]; then
	cat <<-EOF > ${CONFIG}
	classes:
	- service.git.client
	- system.linux.system.single
	- system.salt.master.single
	- system.salt.master.$FORMULAS_SOURCE
	- system.reclass.storage.salt
	parameters:
	_param:
	reclass_data_repository: "$RECLASS_ADDRESS"
	reclass_data_revision: ${RECLASS_BRANCH:-master}
	salt_formula_branch: ${SALT_FORMULAS_BRANCH:-master}
	reclass_config_master: $SALT_MASTER
	single_address: $SALT_MASTER
	salt_master_host: $SALT_MASTER
	salt_master_base_environment: $SALT_ENV
	linux_system_codename: $DISTRIB_CODENAME
	linux:
	system:
	name: $MINION_ID
	domain: $DOMAIN
	# ########
	EOF

	if [ "$SALT_VERSION" == "latest" ]; then
	VERSION=""
	else
	VERSION="version: $SALT_VERSION"
	fi

	cat <<-EOF >> ${CONFIG}
	salt:
	master:
	accept_policy: open_mode
	source:
	engine: $SALT_SOURCE
	$VERSION
	minion:
	source:
	engine: $SALT_SOURCE
	$VERSION
	# ########
	# vim: ft=yaml sw=2 ts=2 sts=2
	EOF
	fi
	}

	configure_salt_minion()
	{
	[ ! -d /etc/salt/minion.d ] && mkdir -p /etc/salt/minion.d
	cat <<-EOF > /etc/salt/minion.d/minion.conf
	master: $SALT_MASTER
	id: $MINION_ID
	EOF
	}


	install_salt_master_pkg()
	{
	echo -e "\nPreparing base OS repository ...\n"

	configure_pkg_repo

	echo -e "\nInstalling salt master ...\n"

	case $PLATFORM_FAMILY in
	debian)
	$SUDO apt-get install -y git
	which reclass \|\| $SUDO apt install -qqq -y reclass
	curl -L https://bootstrap.saltstack.com \| $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null \|\| true
	;;
	rhel)
	yum install -y git
	which reclass \|\| $SUDO yum install -y reclass
	curl -L https://bootstrap.saltstack.com \| $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null \|\| true
	;;
	esac

	which reclass-salt \|\| {
	test -e /usr/share/reclass/reclass-salt && {
	ln -fs /usr/share/reclass/reclass-salt /usr/bin
	}
	}

	configure_salt_master

	echo -e "\nRestarting services ...\n"
	[ -f /etc/salt/pki/minion/minion_master.pub ] && rm -f /etc/salt/pki/minion/minion_master.pub
	$SVCTOOL salt-master restart
	}

	install_salt_master_pip()
	{
	echo -e "\nPreparing base OS repository ...\n"

	case $PLATFORM_FAMILY in
	debian)
	$SUDO apt-get install -y python-pip python-dev zlib1g-dev git
	which reclass \|\| $SUDO apt-get install -y reclass
	;;
	rhel)
	$SUDO yum install -y git
	which reclass \|\| $SUDO yum install -y reclass
	;;
	esac

	echo -e "\nInstalling salt master ...\n"
	# TODO: replace with saltstack bootstrap script

	if [ "$SALT_VERSION" == "latest" ]; then
	pip install salt
	else
	pip install salt==$SALT_VERSION
	fi

	curl -Lo /etc/init.d/salt-master https://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-master.init && chmod 755 /etc/init.d/salt-master
	ln -s /usr/local/bin/salt-master /usr/bin/salt-master

	which reclass-salt \|\| {
	test -e /usr/share/reclass/reclass-salt && {
	ln -fs /usr/share/reclass/reclass-salt /usr/bin
	}
	}

	configure_salt_master

	echo -e "\nRestarting services ...\n"
	[ -f /etc/salt/pki/minion/minion_master.pub ] && rm -f /etc/salt/pki/minion/minion_master.pub
	$SVCTOOL salt-master restart
	}



	install_salt_minion_pkg()
	{

	configure_pkg_repo

	echo -e "\nInstalling salt minion ...\n"

	case $PLATFORM_FAMILY in
	debian)
	curl -L https://bootstrap.saltstack.com \| $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null \|\| true
	;;
	rhel)
	curl -L https://bootstrap.saltstack.com \| $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null \|\| true
	;;
	esac


	configure_salt_minion

	$SVCTOOL salt-minion restart
	}

	install_salt_minion_pip()
	{
	echo -e "\nInstalling salt minion ...\n"

	curl -Lo /etc/init.d/salt-minion https://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-minion.init && chmod 755 /etc/init.d/salt-minion
	ln -s /usr/local/bin/salt-minion /usr/bin/salt-minion

	configure_salt_minion
	$SVCTOOL salt-minion restart
	}


	install_salt_formula_pkg()
	{
	configure_pkg_repo

	case $PLATFORM_FAMILY in
	debian)
	echo "Configuring necessary formulas ..."

	[ ! -d ${RECLASS_ROOT}/classes/service ] && mkdir -p ${RECLASS_ROOT}/classes/service
	# Set essentials if FORMULAS_SALT_MASTER is not defined at all
	[ -z ${FORMULAS_SALT_MASTER+x} ] && declare -a FORMULAS_SALT_MASTER=("linux" "reclass" "salt" "memcached")
	for formula_service in "${FORMULAS_SALT_MASTER[@]}"; do
	echo -e "\nConfiguring salt formula ${formula_service} ...\n"
	[ ! -d "${FORMULAS_PATH}/env/${formula_service}" ] && \
	if ! $SUDO apt-get install -y salt-formula-${formula_service}; then
	echo -e "\nInstall salt-formula-${formula_service} failed.\n"
	exit 1
	fi
	[ ! -L "${RECLASS_ROOT}/classes/service/${formula_service}" ] && \
	ln -sf ${FORMULAS_PATH}/reclass/service/${formula_service} ${RECLASS_ROOT}/classes/service/${formula_service}
	done
	;;
	rhel)
	# TODO
	;;
	esac

	[ ! -d /srv/salt/env ] && mkdir -p /srv/salt/env \|\| echo ""
	[ ! -L /srv/salt/env/prd ] && ln -s ${FORMULAS_PATH}/env /srv/salt/env/prd \|\| echo ""
	}

	install_salt_formula_git()
	{
	echo "Configuring necessary formulas ..."

	[ ! -d ${RECLASS_ROOT}/classes/service ] && mkdir -p ${RECLASS_ROOT}/classes/service
	# Set essentials if FORMULAS_SALT_MASTER is not defined at all
	[ -z ${FORMULAS_SALT_MASTER+x} ] && declare -a FORMULAS_SALT_MASTER=("linux" "reclass" "salt" "memcached")
	for formula_service in "${FORMULAS_SALT_MASTER[@]}"; do
	echo -e "\nConfiguring salt formula ${formula_service} ...\n"
	_BRANCH=${FORMULAS_BRANCH}
	[ ! -d "${FORMULAS_PATH}/env/_formulas/${formula_service}" ] && {
	if ! git ls-remote --exit-code --heads ${FORMULAS_BASE}/salt-formula-${formula_service}.git ${_BRANCH}; then
	# Fallback to the master branch if the branch doesn't exist for this repository
	_BRANCH=master
	fi
	if ! git clone ${FORMULAS_BASE}/salt-formula-${formula_service}.git ${FORMULAS_PATH}/env/_formulas/${formula_service} -b ${_BRANCH}; then
	echo -e "\nCloning of ${FORMULAS_BASE}/salt-formula-${formula_service}.git failed.\n"
	exit 1
	fi
	} \|\| {
	cd ${FORMULAS_PATH}/env/_formulas/${formula_service};
	git fetch origin/${_BRANCH} \|\| git fetch --all
	git checkout ${_BRANCH} && git pull \|\| git pull;
	cd -
	}
	[ ! -L "/usr/share/salt-formulas/env/${formula_service}" ] && \
	ln -sf ${FORMULAS_PATH}/env/_formulas/${formula_service}/${formula_service} /usr/share/salt-formulas/env/${formula_service}
	[ ! -L "${RECLASS_ROOT}/classes/service/${formula_service}" ] && \
	ln -sf ${FORMULAS_PATH}/env/_formulas/${formula_service}/metadata/service ${RECLASS_ROOT}/classes/service/${formula_service}
	done

	[ ! -d /srv/salt/env ] && mkdir -p /srv/salt/env \|\| echo ""
	[ ! -L /srv/salt/env/dev ] && ln -s /usr/share/salt-formulas/env /srv/salt/env/dev \|\| echo ""
	}


	saltmaster_bootstrap() {

	log_info "Salt master setup"
	test -n "$MASTER_HOSTNAME" \|\| exit 1

	clone_reclass
	# override some envs from cluster level *.env, use with care
	source_local_envs

	pgrep salt-master \| sed /$$/d \| xargs --no-run-if-empty -i{} $SUDO kill -9 {}
	pkill -9 salt-minion
	test -e ${SCRIPTS}/.salt-master-setup.sh.passed \|\| {
	export SALT_MASTER=localhost
	export MINION_ID=${MASTER_HOSTNAME}
	if ! [[ $DEBUG =~ ^(True\|true\|1\|yes)$ ]]; then
	SALT_MASTER_SETUP_OUTPUT='/dev/stdout'
	fi
	# call local "setup() master"
	#if ! $SUDO ${SCRIPTS}/salt-master-setup.sh master &> ${SALT_MASTER_SETUP_OUTPUT:-/tmp/salt-master-setup.log}; then
	if ! setup master; then
	#cat /tmp/salt-master-setup.log
	log_err "salt master setup() failed."
	exit 1
	else
	$SUDO touch ${SCRIPTS}/.salt-master-setup.sh.passed
	fi
	}

	log_info "Clean up generated"
	cd $RECLASS_ROOT
	$SUDO rm -rf $RECLASS_ROOT/nodes/_generated/*

	log_info "Re/starting salt services"
	pgrep salt-master \| sed /$$/d \| xargs --no-run-if-empty -i{} $SUDO kill -9 {}
	pkill -9 salt-minion
	sleep 1
	$SUDO service salt-master restart
	$SUDO service salt-minion restart
	sleep 10
	}

	# Init salt master
	saltmaster_init() {

	log_info "Runing saltmaster states"
	test -n "$MASTER_HOSTNAME" \|\| exit 1

	set -e
	$SUDO salt-call saltutil.sync_all >/dev/null

	# TODO: Placeholder update saltmaster spec (nodes/FQDN.yml) to be able to bootstrap with minimal configuration
	# (ie: with linux, git, salt formulas)

	#log_info "Verify SaltMaster, before salt-master is fully initialized"
	#if ! $SUDO reclass-salt -p ${MASTER_HOSTNAME} &> /tmp/${MASTER_HOSTNAME}.pillar;then
	# log_warn "Node verification before initialization failed."; cat /tmp/${MASTER_HOSTNAME}.pillar;
	#fi

	log_info "State: salt.master.env"
	if ! $SUDO salt-call ${SALT_OPTS} -linfo state.apply salt.master.env; then
	log_err "State salt.master.env failed, keep your eyes wide open."
	fi

	log_info "State: salt.master.pillar"
	retry ${SALT_STATE_RETRY} $SUDO salt-call ${SALT_OPTS} state.apply salt.master.pillar pillar='{"reclass":{"storage":{"data_source":{"engine":"local"}}}}'
	# Note: sikp reclass data dir states
	# in order to avoid pull from configured repo/branch

	# Revert temporary SaltMaster minimal configuration, if any
	pushd $RECLASS_ROOT
	if [ $(git diff --name-only nodes \| sort \| uniq \| wc -l) -ge 1 ]; then
	git status \|\| true
	log_warn "Locally modified $RECLASS_ROOT/nodes found. (Possibly salt-master minimized setup from salt-master-setup.sh call)"
	log_info "Checkout HEAD state of $RECLASS_ROOT/nodes/*."
	git checkout -- $RECLASS_ROOT/nodes \|\| true
	log_info "Re-Run states: salt.master.env and salt.master.pillar according the HEAD state."
	log_info "State: salt.master.env"
	if ! $SUDO salt-call ${SALT_OPTS} -linfo state.apply salt.master.env; then
	log_err "State salt.master.env failed, keep your eyes wide open."
	fi
	log_info "State: salt.master.pillar"
	retry ${SALT_STATE_RETRY} $SUDO salt-call ${SALT_OPTS} state.apply salt.master.pillar pillar='{"reclass":{"storage":{"data_source":{"engine":"local"}}}}'
	fi
	popd

	log_info "State: salt.master.storage.node"
	set +e
	$SUDO salt-call ${SALT_OPTS} state.apply reclass.storage.node
	ret=$?
	set -e

	if [[ $ret -eq 2 ]]; then
	log_err "State reclass.storage.node failed with exit code 2 but continuing."
	elif [[ $ret -ne 0 ]]; then
	log_err "State reclass.storage.node failed with exit code $ret"
	exit 1
	fi

	log_info "Re/starting salt services"
	$SUDO sed -i 's/^master:.*/master: localhost/' /etc/salt/minion.d/minion.conf
	$SUDO service salt-minion restart >/dev/null
	$SUDO salt-call ${SALT_OPTS} saltutil.sync_all >/dev/null

	verify_salt_master
	set +e

	}


	function verify_salt_master() {
	set -e

	log_info "Verify Salt master"
	test -n "$MASTER_HOSTNAME" \|\| exit 1

	if [[ $VERIFY_SALT_CALL =~ ^(True\|true\|1\|yes)$ ]]; then
	$SUDO salt-call ${SALT_OPTS} --id=${MASTER_HOSTNAME} grains.item roles > /tmp/${MASTER_HOSTNAME}.grains.item.roles
	$SUDO salt-call ${SALT_OPTS} --id=${MASTER_HOSTNAME} state.show_lowstate > /tmp/${MASTER_HOSTNAME}.state.show_state
	$SUDO salt-call --no-color grains.items
	$SUDO salt-call --no-color pillar.data
	fi
	if ! $SUDO reclass --nodeinfo ${MASTER_HOSTNAME} > /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo; then
	log_err "For more details see full log /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo"
	exit 1
	fi
	}

	function verify_salt_minion() {
	set -e
	node=$1
	log_info "Verifying ${node}"
	if [[ $VERIFY_SALT_CALL =~ ^(True\|true\|1\|yes)$ ]]; then
	$SUDO salt-call ${SALT_OPTS} --id=${node} grains.item roles > /tmp/${node}.grains.item.roles
	$SUDO salt-call ${SALT_OPTS} --id=${node} state.show_lowstate > /tmp/${node}.state.show_lowstate
	fi
	if ! $SUDO reclass --nodeinfo ${node} > /tmp/${node}.reclass.nodeinfo; then
	log_err "For more details see full log /tmp/${node}.reclass.nodeinfo"
	if [[ ${BREAK_ON_VERIFICATION_ERROR:-yes} =~ ^(True\|true\|1\|yes)$ ]]; then
	exit 1
	fi
	fi
	}

	function verify_salt_minions() {
	#set -e
	NODES=$(find $RECLASS_ROOT/nodes/ -name "*.yml" \| grep -v "cfg")
	log_info "Verifying minions: $(echo ${NODES}\|xargs)"

	# Parallel
	#echo $NODES \| parallel --no-notice -j 2 --halt 2 "verify_salt_minion \$(basename {} .yml) > {}.pillar_verify"
	#ls -lrta *.pillar_verify \| tail -n 1 \| xargs -n1 tail -n30

	function filterFails() {
	grep -v '/grains' \| tee -a $1 \| tail -n20
	}

	log_info "Verify nodes"
	passed=0
	for node in ${NODES}; do
	node=$(basename $node .yml)

	# filter first in cluster.. ctl-01, mon-01, etc..
	if [[ "${node//.}" =~ 01 \|\| "${node//.}" =~ 02 ]] ;then
	verify_salt_minion ${node} \|\| continue
	else
	echo Skipped $node.
	fi
	passed=$(($passed+1))
	done
	# fail on failures
	total=$(echo $NODES \| xargs --no-run-if-empty -n1 echo \|wc -l)
	test ! $passed -lt $total \|\| log_err "Results: $passed of $total passed."
	test ! $passed -lt $total \|\| {
	tail -n50 /tmp/*.pillar_verify
	return 1
	}
	}



	##########################################
	# To install salt master/minon

	function install() {
	setup $@
	}

	function setup() {
	# CLI
	while [ x"$1" != x"" ]; do
	which curl &>/dev/null \|\| $PKGTOOL -y install curl &>/dev/null

	case $1 in
	master )
	install_salt_master_$SALT_SOURCE
	install_salt_minion_$SALT_SOURCE
	install_salt_formula_$FORMULAS_SOURCE
	;;
	minion )
	install_salt_minion_$SALT_SOURCE
	;;
	esac
	shift
	done
	echo DONE
	}

	function bootstrap() {
	log_info "Bootstrap & verification of SaltMaster and configured minions."
	trap _atexit INT TERM EXIT

	system_config_master
	saltmaster_bootstrap &&\
	saltmaster_init &&\
	verify_salt_minions
	}

	function default() {
	bootstrap $@
	}


	##########################################
	[[ "$0" != "$BASH_SOURCE" ]] \|\| {
	# unless file is being sourced
	default $@
	}