Merge pull request #31 from ruzickap/bug/add_dependencies
Fix dependencies
diff --git a/bootstrap.sh b/bootstrap.sh
index 56144be..a4e008b 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -6,10 +6,9 @@
# - http://github.com/salt-formulas-scripts
# - http://github.com/salt-formulas/salt-formula-salt (salt.master sls)
-# TODO:
-# - use PPA repository as formula source
-# - support for spm/yum
-
+# NOTE: This script is collection of shared functions to automate bootstrap of "salted" CI environments.
+# Its not intended to be used in PRODUCTION unless you know what you are doing.
+# You have been warned!
# Source specific env vars.
# shopt -u dotglob
@@ -44,14 +43,12 @@
export SALT_OPTS="${SALT_OPTS:- --timeout=120 --state-output=changes --retcode-passthrough --force-color $SALT_LOG_LEVEL }"
export SALT_STATE_RETRY=${SALT_STATE_RETRY:-3}
-
# salt apt repository
test -e /etc/lsb-release && eval $(cat /etc/lsb-release)
which lsb_release && DISTRIB_CODENAME=${DISTRIB_CODENAME:-$(lsb_release -cs)}
#
export APT_REPOSITORY=${APT_REPOSITORY:- deb [arch=amd64] http://apt.mirantis.com/${DISTRIB_CODENAME} ${DISTRIB_REVISION:-stable} salt}
export APT_REPOSITORY_GPG=${APT_REPOSITORY_GPG:-http://apt.mirantis.com/public.gpg}
-
# reclass
export RECLASS_ADDRESS=${RECLASS_ADDRESS:-https://github.com/salt-formulas/openstack-salt.git} # https/git
export RECLASS_BRANCH=${RECLASS_BRANCH:-master}
@@ -80,11 +77,17 @@
# saltstack
BOOTSTRAP_SALTSTACK=${BOOTSTRAP_SALTSTACK:-True}
+BOOTSTRAP_SALTSTACK_COM=${BOOTSTRAP_SALTSTACK_COM:-"https://bootstrap.saltstack.com"}
BOOTSTRAP_SALTSTACK_VERSION=${BOOTSTRAP_SALTSTACK_VERSION:- stable 2016.3 }
+BOOTSTRAP_SALTSTACK_VERSION=${BOOTSTRAP_SALTSTACK_VERSION//latest*/stable}
+# TODO add 'r' option by default
+# Currently, without 'r' option, upstream saltstack repos will be used. Otherwise
+# local one should be used, from http://apt.mirantis.com/ or whatever.
BOOTSTRAP_SALTSTACK_OPTS=${BOOTSTRAP_SALTSTACK_OPTS:- -dX $BOOTSTRAP_SALTSTACK_VERSION }
SALT_SOURCE=${SALT_SOURCE:-pkg}
-# the version below is used salt pillar data
-SALT_VERSION=${SALT_VERSION:-latest}
+
+# SECURITY
+SSH_STRICTHOSTKEYCHECKING=no
# environment
if [ "$FORMULAS_SOURCE" == "git" ]; then
@@ -187,7 +190,11 @@
function clone_reclass() {
if [ ! -d ${RECLASS_ROOT}/classes ]; then
# No reclass at all, clone from given address
- ssh-keyscan -H github.com >> ~/.ssh/known_hosts || true
+ # In case, non-github repo
+ _tmp_host=$(echo ${RECLASS_ADDRESS} | awk -F/ '{print $3}')
+ ssh-keyscan -T 1 -H ${_tmp_host} >> ~/.ssh/known_hosts || true
+ # But, awk is not perfect solution, so lets make double-check for github
+ ssh-keyscan -T 1 -H github.com >> ~/.ssh/known_hosts || true
if echo ${RECLASS_BRANCH} | egrep -q "^refs"; then
git clone ${RECLASS_ADDRESS} ${RECLASS_ROOT}
cd ${RECLASS_ROOT}
@@ -211,17 +218,33 @@
##########################################
# Main calls
-system_config_ssh_conf() {
- for conf in ~/.ssh/config /root/.ssh/config; do
- $SUDO mkdir -p $(dirname $conf)
- if ! grep StrictHostKeyChecking $conf; then
- # this should be used only in CI environment
- echo -e "Host *\n\tStrictHostKeyChecking no\n" | $SUDO tee $conf >/dev/null
- fi
- done
+ci_config_ssh() {
+
+ # for CI current user
+ conf=~/.ssh/config
+ mkdir -p $(dirname $conf)
+ touch $conf
+ echo -e "Host *\n\tStrictHostKeyChecking $SSH_STRICTHOSTKEYCHECKING\n" | tee -a $conf >/dev/null
+ touch ~/.ssh/known_hosts
+
if ! grep github.com ~/.ssh/known_hosts; then
ssh-keyscan -H github.com >> ~/.ssh/known_hosts || true
+ ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts || true
fi
+
+ # for root
+ #conf=/root/.ssh/config
+ #$SUDO mkdir -p $(dirname $conf)
+ #$SUDO touch $conf
+ #if ! $SSH_STRICTHOSTKEYCHECKING; then
+ # echo -e "Host *\n\tStrictHostKeyChecking no\n" | $SUDO tee -a $conf >/dev/null
+ #fi
+}
+
+# DEPRECATED
+system_config_ssh_conf() {
+ # for backward compatibility
+ ci_config_ssh
}
system_config_salt_modules_prereq() {
@@ -237,19 +260,11 @@
log_info "System configuration salt master"
system_config_salt_modules_prereq
- system_config_ssh_conf
+ ci_config_ssh
if ! grep '127.0.1.2.*salt' /etc/hosts; then
echo "127.0.1.2 salt" | $SUDO tee -a /etc/hosts >/dev/null
fi
-
- which reclass || $SUDO $PKGTOOL install -y reclass
-
- which reclass-salt || {
- test -e /usr/share/reclass/reclass-salt && {
- ln -fs /usr/share/reclass/reclass-salt /usr/bin
- }
- }
}
configure_salt_master()
@@ -343,23 +358,15 @@
# ########
EOF
- if [ "$SALT_VERSION" == "latest" ]; then
- VERSION=""
- else
- VERSION="version: $SALT_VERSION"
- fi
-
cat <<-EOF >> ${CONFIG}
salt:
master:
accept_policy: open_mode
source:
engine: $SALT_SOURCE
- $VERSION
minion:
source:
engine: $SALT_SOURCE
- $VERSION
# ########
# vim: ft=yaml sw=2 ts=2 sts=2
EOF
@@ -385,6 +392,11 @@
case ${VERSION} in
pkg|package)
which reclass || $SUDO $PKGTOOL install -y reclass
+ which reclass-salt || {
+ if [ -e /usr/share/reclass/reclass-salt ]; then
+ ln -fs /usr/share/reclass/reclass-salt /usr/bin
+ fi
+ }
;;
*)
log_warn "Install development version of reclass"
@@ -411,19 +423,13 @@
case $PLATFORM_FAMILY in
debian)
$SUDO apt-get install -y git
- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
+ curl -L ${BOOTSTRAP_SALTSTACK_COM} | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
;;
rhel)
yum install -y git
- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
+ curl -L ${BOOTSTRAP_SALTSTACK_COM} | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
;;
esac
-
- which reclass-salt || {
- test -e /usr/share/reclass/reclass-salt && {
- ln -fs /usr/share/reclass/reclass-salt /usr/bin
- }
- }
configure_salt_master
@@ -447,22 +453,16 @@
echo -e "\nInstalling salt master ...\n"
# TODO: replace with saltstack bootstrap script
-
- if [ "$SALT_VERSION" == "latest" ]; then
+
+ if [ -z ${PIP_SALT_VERSION} ] || [ "$PIP_SALT_VERSION" == "latest" ]; then
pip install salt
else
- pip install salt==$SALT_VERSION
+ pip install salt==${PIP_SALT_VERSION}
fi
curl -Lo /etc/init.d/salt-master https://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-master.init && chmod 755 /etc/init.d/salt-master
ln -s /usr/local/bin/salt-master /usr/bin/salt-master
- which reclass-salt || {
- test -e /usr/share/reclass/reclass-salt && {
- ln -fs /usr/share/reclass/reclass-salt /usr/bin
- }
- }
-
configure_salt_master
echo -e "\nRestarting services ...\n"
@@ -479,10 +479,10 @@
case $PLATFORM_FAMILY in
debian)
- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
+ curl -L ${BOOTSTRAP_SALTSTACK_COM} | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
;;
rhel)
- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
+ curl -L ${BOOTSTRAP_SALTSTACK_COM} | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true
;;
esac
@@ -521,6 +521,8 @@
echo -e "\nInstall salt-formula-${formula_service} failed.\n"
exit 1
fi
+ #Since some salt formula names contain "-" and in symlinks they should contain "_" adding replacement
+ formula_service=${formula_service//-/$'_'}
[ ! -L "${RECLASS_ROOT}/classes/service/${formula_service}" ] && \
ln -sf ${FORMULAS_PATH}/reclass/service/${formula_service} ${RECLASS_ROOT}/classes/service/${formula_service}
done
@@ -569,20 +571,41 @@
[ ! -L /srv/salt/env/dev ] && ln -s /usr/share/salt-formulas/env /srv/salt/env/dev || echo ""
}
+saltservice_stop() {
+ set +e
+ $SUDO service salt-minion stop
+ $SUDO service salt-master stop
+ sleep ${SALT_STOPSTART_WAIT:-30}
+ ${SUDO} pkill -9 salt-master && ${SUDO} rm -rf /var/run/salt/master/* || true
+ ${SUDO} pkill -9 salt-minion
+}
+saltservice_start() {
+ set +e
+ $SUDO service salt-master start
+ $SUDO service salt-minion start
+ sleep ${SALT_STOPSTART_WAIT:-30}
+}
+
+saltservice_restart() {
+ set +e
+ saltservice_stop
+ saltservice_start
+}
saltmaster_bootstrap() {
log_info "Salt master setup"
test -n "$MASTER_HOSTNAME" || exit 1
- clone_reclass
# override some envs from cluster level *.env, use with care
source_local_envs
- pgrep salt-master | sed /$$/d | xargs --no-run-if-empty -i{} $SUDO kill -9 {} || true
- pkill -9 salt-minion
+ saltservice_stop
+
+ clone_reclass
+
SCRIPTS=$(dirname $0)
-
+
test -e ${SCRIPTS}/.salt-master-setup.sh.passed || {
export MASTER_IP=${MASTER_IP:-127.0.0.1}
export MINION_ID=${MASTER_HOSTNAME}
@@ -604,14 +627,7 @@
install_reclass ${RECLASS_VERSION/dev*/develop}
log_info "Re/starting salt services"
- $SUDO service salt-minion stop
- $SUDO service salt-master stop
- sleep 10
- pgrep salt-master | sed /$$/d | xargs --no-run-if-empty -i{} $SUDO kill -9 {} || true
- pkill -9 salt-minion
- $SUDO service salt-master start
- $SUDO service salt-minion start
- sleep 15
+ saltservice_restart
}
# Init salt master
@@ -652,7 +668,10 @@
# finally re-configure salt master conf, ie: may remove ignore_class_notfound option
log_info "State: salt.master.service"
+ # ensure salt master is started
+ saltservice_start
retry ${SALT_STATE_RETRY} $SUDO salt-call ${SALT_OPTS} state.apply salt.master.service || true
+ saltservice_start
log_info "State: salt.master.storage.node"
set +e
@@ -660,8 +679,8 @@
# FIXME: PLACEHOLDER TO TRIGGER NODE GENERATION THROUG SALT REACT.
retry ${SALT_STATE_RETRY} $SUDO salt-call ${SALT_OPTS} state.apply reclass.storage.node
ret=$?
- set -e
+ set -e
if [[ $ret -eq 2 ]]; then
log_err "State reclass.storage.node failed with exit code 2 but continuing."
elif [[ $ret -ne 0 ]]; then
@@ -669,22 +688,97 @@
exit 1
fi
- log_info "Re/starting salt services"
+ set +e
+ log_info "Updating minion.conf -> master: localhost"
$SUDO sed -i 's/^master:.*/master: localhost/' /etc/salt/minion.d/minion.conf
- $SUDO service salt-minion stop
- $SUDO service salt-master stop
- sleep 10
- pgrep salt-master | sed /$$/d | xargs --no-run-if-empty -i{} $SUDO kill -9 {} || true
- $SUDO service salt-master start
- $SUDO service salt-minion start
- sleep 15
+
+ log_info "Re/starting salt services" # in order to load new deployed configuration from model
+ saltservice_restart
+
+ log_info "Salt Util sync all"
$SUDO salt-call ${SALT_OPTS} saltutil.sync_all >/dev/null
verify_salt_master
- set +e
}
+## CI Workarounds
+
+function mockup_node_registration() {
+
+ # for dynamic nodes registrations (require github.com/salt-formulas/salt-formula-reclass)
+
+ source /etc/os-release
+ os_codename=$VERSION_CODENAME
+ domain=$(hostname -d)
+ master_ip=$(hostname -I | awk '{print $1}')
+ fake_ip_preffix=$(echo $master_ip | awk -F. '{print $1"."$2}')
+ fake_ip_base=10
+
+ # SHOULD BE ALREADY RUN AS A PART OF BOOTSTRAP
+ # sync, in order to load custom modules
+ #salt-call saltutil.sync_all
+
+ # SHOULD BE ALREADY RUN AS A PART OF BOOTSTRAP
+ #PILLAR='{"reclass":{"storage":{"data_source":{"engine":"local"}}} }'
+ #salt-call state.apply reclass.storage.node pillar="$PILLAR" > /dev/null 2>/dev/null || true
+
+ # rotate over dynamic hosts
+ for host_idx in {01..02};do
+ for host in `salt-call pillar.items reclass:storage |grep '<<node_hostname>>' | awk -F_ '{print $NF}' | sed 's/[0-9]*//g' | egrep -v cfg | sort -u`; do
+ hostname=${host}${host_idx}
+ fake_ip_base=$(($fake_ip_base + 1))
+
+ node_network01_ip="$fake_ip_preffix.11.$fake_ip_base"
+ node_network02_ip="$fake_ip_preffix.12.$fake_ip_base"
+ node_network03_ip="$fake_ip_preffix.13.$fake_ip_base"
+ node_network04_ip="$fake_ip_preffix.14.$fake_ip_base"
+ node_network05_ip="$fake_ip_preffix.15.$fake_ip_base"
+ node_network01_iface=$(ls /sys/class/net/ | grep -v lo | sort | head -n1)
+ node_network02_iface="eth1"
+ node_network03_iface="eth2"
+ node_network04_iface="eth3"
+ node_network05_iface="eth4"
+
+ declare -A vars
+ vars=(
+ ["node_master_ip"]=
+ ["node_os"]=${os_codename}
+ ["node_deploy_ip"]=${node_network01_ip}
+ ["node_deploy_iface"]=${node_network01_iface}
+ ["node_control_ip"]=${node_network02_ip}
+ ["node_control_iface"]=${node_network02_iface}
+ ["node_tenant_ip"]=${node_network03_ip}
+ ["node_tenant_iface"]=${node_network03_iface}
+ ["node_external_ip"]=${node_network04_ip}
+ ["node_external_iface"]=${node_network04_iface}
+ ["node_baremetal_ip"]=${node_network05_ip}
+ ["node_baremetal_iface"]=${node_network05_iface}
+ ["node_domain"]=$domain
+ ["node_cluster"]=$(hostname -d |awk -F. '{print $1}')
+ ["node_hostname"]=$hostname
+ )
+
+ data=""; i=0
+ for key in "${!vars[@]}"; do
+ data+="\"${key}\": \"${vars[${key}]}\""
+ i=$(($i+1))
+ if [ $i -lt ${#vars[@]} ]; then
+ data+=", "
+ fi
+ done
+ echo "Classifying node $hostname"
+ NODECR='"node_name": "'${hostname}.${domain}'", "node_data": {'$data'}'
+ PILLAR='{'${NODECR}', "reclass":{"storage":{"data_source":{"engine":"local"}}} }'
+ salt-call state.apply reclass.reactor_sls.node_register pillar="$PILLAR" #-l info
+ #salt-call event.send "reclass/minion/classify" "{$data}"
+ done
+ done
+
+}
+
+## VERIFY
+
function verify_salt_master() {
set -e
@@ -701,7 +795,7 @@
$SUDO salt-call --no-color pillar.data
fi
# TODO: REMOVE reclass --nodeinfo section / run only on debug - as the only required is reclass.validate_*
- if ! $SUDO reclass --nodeinfo ${MASTER_HOSTNAME} > /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo; then
+ if ! $SUDO python -m reclass.cli --nodeinfo ${MASTER_HOSTNAME} > /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo; then
log_err "For more details see full log /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo"
exit 1
fi
@@ -716,7 +810,7 @@
$SUDO salt-call ${SALT_OPTS} --id=${node} grains.item roles > /tmp/${node}.grains.item.roles
$SUDO salt-call ${SALT_OPTS} --id=${node} state.show_lowstate > /tmp/${node}.state.show_lowstate
fi
- if ! $SUDO reclass --nodeinfo ${node} > /tmp/${node}.reclass.nodeinfo; then
+ if ! $SUDO python -m reclass.cli --nodeinfo ${node} > /tmp/${node}.reclass.nodeinfo; then
log_err "For more details see full log /tmp/${node}.reclass.nodeinfo"
if [[ ${BREAK_ON_VERIFICATION_ERROR:-yes} =~ ^(True|true|1|yes)$ ]]; then
exit 1
diff --git a/formula-fetch.sh b/formula-fetch.sh
index 03153d3..f103fcd 100755
--- a/formula-fetch.sh
+++ b/formula-fetch.sh
@@ -3,13 +3,34 @@
# Usage:
# ./formula-fetch.sh <Formula URL> <Name> <Branch>
#
-# Example:
-# GIT_FORMULAS_PATH=.vendor/formulas ./formula-fetch.sh https://github.com/salt-formulas/salt-formula-salt
+# Example usage:
+# FORMULA_SOURCES=https://github.com/epcim/my-salt-formulas https://github.com/salt-formulas https://github.com/saltstack-formulas
+# SALT_ENV_PATH=.vendor/formulas
# --
-# GIT_FORMULAS_PATH=/usr/share/salt-formulas/env/_formulas
+# ./formula-fetch.sh
# xargs -n1 ./formula-fetch.sh < dependencies.txt
+## DEFAULTS
+#
+# default sources
+FORMULA_SOURCES="${SALT_FORMULA_SOURCES:-https://github.com/salt-formulas https://github.com/saltstack-formulas}"
+FORMULA_VERSION="${SALT_FORMULA_VERSION:-master}"
+# where to fetch formulas
+FORMULAS_BASE=${SALT_FORMULAS_BASE:-/srv/salt/formula}
+# For better stability, skip formula repos without recognized CI
+FORMULA_WITHOUT_CI=${SALT_FORMULA_WITHOUT_CI:-false}
+# salt env/root, where formulas are found
+SALT_ENV_PATH=${SALT_ENV_PATH:-/srv/salt/env/prd}
+#SALT_ENV_PATH=${SALT_ENV_PATH:-.vendor/formulas}
+#SALT_ENV_PATH=${SALT_ENV_PATH:/usr/share/salt-formulas/env/_formulas}
+# reclass related
+RECLASS_BASE=${RECLASS_BASE:-/srv/salt/reclass}
+# env
+LC_ALL=en_US.UTF-8
+LANG=en_US.UTF-8
+
+
# Parse git dependencies from metadata.yml
# $1 - path to <formula>/metadata.yml
# sample to output:
@@ -17,46 +38,149 @@
# https://github.com/salt-formulas/salt-formula-salt salt
function fetchDependencies() {
METADATA="$1";
- grep -E "^dependencies:" "$METADATA" >/dev/null || return 0
- (python - "$METADATA" | while read dep; do fetchGitFormula "$dep"; done) <<-DEPS
+ grep -E "^dependencies:" "$METADATA" &>/dev/null || return 0
+ (python3 - "$METADATA" | while read dep; do fetchGitFormula $dep; done) <<-DEPS
import sys,yaml
- for dep in yaml.load(open(sys.argv[1], "ro"))["dependencies"]:
- print("{source} {name}").format(**dep)
+ try:
+ for dep in yaml.load(open(sys.argv[1], "r"))["dependencies"]:
+ if len(set(('name', 'source')) & set(dep.keys())) == 2:
+ print("{source} {name}".format(**dep))
+ except Exception as e:
+ print("[W] {}".format(e.__doc__))
+ print("[W] {}".format(e.message))
+ pass
DEPS
}
+
+# Read formula name from meetadata.yml
+# $1 - path to <formula>/metadata.yml
+function getFormulaName() {
+ python3 - "$1" <<-READ_NAME
+ try:
+ import sys,yaml;print(yaml.load(open(sys.argv[1], "r"))["name"]);
+ except Exception as e:
+ print("[W] {}".format(e.__doc__))
+ print("[W] {}".format(e.message))
+ pass
+ READ_NAME
+}
+
+
# Fetch formula from git repo
# $1 - formula git repo url
# $2 - formula name (optional)
# $3 - branch (optional)
function fetchGitFormula() {
test -n "${FETCHED}" || declare -a FETCHED=()
- export GIT_FORMULAS_PATH=${GIT_FORMULAS_PATH:-/usr/share/salt-formulas/env/_formulas}
- mkdir -p "$GIT_FORMULAS_PATH"
+ mkdir -p "$SALT_ENV_PATH" "$FORMULAS_BASE"
+
if [ -n "$1" ]; then
- source="$1"
- name="$2"
- test -n "$name" || name="${source//*salt-formula-}"
- test -z "$3" && branch=master || branch=$3
+
+ # set origin uri
+ # FIXME, TEMP fix for not yet up to date gh:salt-formulas -> s/tcpcloud/salt-formulas/
+ origin="${1/tcpcloud/salt-formulas}"
+ # set gh repo https://salt-formulas/salt-formula-salt -> $FORMULAS_BASE/salt-formulas/salt-formula-salt
+ repo=$(echo $origin | awk -F'/' '{ print substr($0, index($0,$4)) }')
+ # set normula name
+ test -n "$2" && name=$2 || name="$(echo ${origin//*\/} | sed -e 's/-formula$//' -e 's/^salt-formula-//' -e 's/^formula-//')"
+ # set branch
+ test -n "$3" && branch=$3 || branch=${FORMULA_VERSION}
+
+ # DEBUG
+ #echo '--- ------------------------------'
+ #echo origin, $origin
+ #echo repo, $repo
+ #echo fetched ${FETCHED[@]}
+ #echo -e name, $name
+ #echo '---'
+ #return
+
if ! [[ "${FETCHED[*]}" =~ $name ]]; then # dependency not yet fetched
- echo "Fetching: $name"
- if test -e "$GIT_FORMULAS_PATH/$name"; then
- pushd "$GIT_FORMULAS_PATH/$name" &>/dev/null
- test ! -e .git || git pull -r
+ echo -e "[I] Fetching: $origin -> $FORMULAS_BASE/$repo"
+ if [ -e "$FORMULAS_BASE/$repo" ]; then
+ pushd "$FORMULAS_BASE/$repo" &>/dev/null
+ git pull -r; git checkout $branch;
popd &>/dev/null
else
- echo "git clone $source $GIT_FORMULAS_PATH/$name -b $branch"
- git clone "$source" "$GIT_FORMULAS_PATH/$name" -b "$branch"
+ echo -e "[I] git clone $origin $FORMULAS_BASE/$repo -b $branch"
+ if ! git ls-remote --exit-code --heads $origin $branch; then
+ # Fallback to the master branch if the branch doesn't exist for this repository
+ branch=master
+ fi
+ if ! git clone "$origin" "$FORMULAS_BASE/$repo" -b "$branch"; then
+ echo -e "[E] Fetching formula from $origin failed."
+ return ${FAIL_ON_ERRORS:-0}
+ fi
fi
- # install dependencies
- FETCHED+=($name)
- fetchDependencies "$GIT_FORMULAS_PATH/$name/metadata.yml"
+
+ # A metadata.yml is github.com/salt-formulas specific
+ if [ ! -n "$2" -a -e "$FORMULAS_BASE/$repo/metadata.yml" ]; then
+ # try to update name as in formula metadata
+ name=$(getFormulaName "$FORMULAS_BASE/$repo/metadata.yml")
+ fi
+
+ # FIXME, better formula recognition/name fixup for saltstack formulas
+ # Recognize the repo is formula
+ if [ ! -e $FORMULAS_BASE/$repo/$name ]; then
+ echo -e "[E] The repository $FORMULAS_BASE/$repo was not recognized as formula repository."
+ rm -rf "$FORMULAS_BASE/$repo"
+ return ${FAIL_ON_ERRORS:-0}
+ fi
+
+ # Avoid checkout formulas/repos without CI
+ if ! $FORMULA_WITHOUT_CI; then
+ CI=false
+ for p in .circleci .travis.yml .kitchen.yml invoke.yml tasks.py tox.ini test tests; do
+ if [ -e "$FORMULAS_BASE/$repo/$p" ]; then
+ CI=true; break;
+ fi
+ done
+ if ! $CI; then
+ mv "$FORMULAS_BASE/$repo" "$FORMULAS_BASE/${repo}.deprecated-no-ci";
+ return ${FAIL_ON_ERRORS:-0}
+ fi
+ fi
+
+ # SET FORMULA IN SALT ENV
+ if [ ! -e "$SALT_ENV_PATH/$name" ]; then
+
+ # link formula
+ ln -svf $FORMULAS_BASE/$repo/$name $SALT_ENV_PATH/$name
+
+ # copy custom _states, _modules, _etc ...
+ for c in $(/bin/ls $FORMULAS_BASE/$repo | grep '^_' | xargs -n1 --no-run-if-empty); do
+ test -e $SALT_ENV_PATH/$c || mkdir -p $SALT_ENV_PATH/$c
+ ln -svf $FORMULAS_BASE/$repo/$c/* $SALT_ENV_PATH/$c
+ done
+
+ # install optional dependencies (python/pip related as of now only)
+ if [ -e $FORMULAS_BASE/$repo/requirements.txt ]; then
+ pip install -r $FORMULAS_BASE/$repo/requirements.txt
+ fi
+
+ # NOTE: github.com/salt-formulas specific steps
+ # link formula service pillars
+ if [ -n "$RECLASS_BASE" -a -e "$FORMULAS_BASE/$repo/metadata/service" ]; then
+ test -e $RECLASS_BASE/classes/service || mkdir -p $RECLASS_BASE/classes/service
+ ln -svf $FORMULAS_BASE/$repo/metadata/service $RECLASS_BASE/classes/service/$name
+ fi
+ # install dependencies
+ FETCHED+=($name)
+ if [ -e "$FORMULAS_BASE/$repo/metadata.yml" ]; then
+ fetchDependencies "$FORMULAS_BASE/$repo/metadata.yml"
+ fi
+ else
+ echo -e "[I] Formula "$name" already fetched."
+ fi
fi
else
- echo Usage: fetchGitFormula "<git repo>" "[local formula directory name]" "[branch]"
+ echo -e '[I] Usage: fetchGitFormula git_repo_uri [branch] [local formula directory name]'
fi
}
+# DEPRECATED, kept for backward compatibility
+# for github.com/salt-formulas (linking "service" pillar metadata from formula to reclas classes)
function linkFormulas() {
# OPTIONAL: Link formulas from git/pkg
@@ -70,12 +194,76 @@
# form pkgs
find "$SALT_ENV" -maxdepth 1 -mindepth 1 -path "*_formulas*" -prune -o -name "*" -type d -print0| xargs -0 -n1 --no-run-if-empty basename | xargs -I{} --no-run-if-empty \
ln -fs "$SALT_ENV"/{} "$SALT_ROOT"/{};
+}
+
+function setupPyEnv() {
+ MODULES="pygithub pyyaml"
+ pip3 install --upgrade $MODULES || {
+ which pipenv || {
+ pip install --upgrade pipenv
+ }
+ pipenv --three
+ pipenv install $MODULES
+ }
+}
+
+function listRepos_github_com() {
+ #export python=$(pipenv --py || (setupPyEnv &>/dev/null; pipenv --py))
+ if [ -e Pipfile.lock ]; then python=$(pipenv --py); else python=python3; fi
+ $python - "$1" <<-LIST_REPOS
+ import sys
+ import github
+
+ def make_github_agent(user=None, password=None):
+ """ Create github agent to auth """
+ if not user:
+ return github.Github()
+ else:
+ return github.Github(user, password)
+
+ def get_org_repos(gh, org_name):
+ org = gh.get_organization(org_name)
+ for repo in org.get_repos():
+ yield repo.name
+
+ try:
+ print(*get_org_repos(make_github_agent(), str(sys.argv[1])), sep="\n")
+ except Exception as e:
+ print("[E] {}".format(e.__doc__))
+ print("[E] {}".format(e.message))
+ sys.exit(1)
+ LIST_REPOS
+}
+
+function fetchAll() {
+ # iterate over all defined sources
+ for source in $(echo ${FORMULA_SOURCES} | xargs -n1 --no-run-if-empty); do
+ hosting=$(echo ${source//\./_} | awk -F'/' '{print $3}')
+ orgname=$(echo ${source//\./_} | awk -F'/' '{print $4}')
+
+ # Get repos. To protect builds on master we likely fail on errors/none while getting repos from $source
+ set -o pipefail
+ repos="$(listRepos_$hosting "$orgname" | xargs -n1 --no-run-if-empty| sort)"
+ set +o pipefail
+ if [ ! -n "$repos" ]; then
+ echo "[E] Error caught or no repositories found at $source. Exiting.";
+ exit 1;
+ fi
+
+ # fetch all repos that looks like formula
+ for repo in $(echo ${repos} | xargs -n1 --no-run-if-empty); do
+ # TODO, avoid a hardcoded pattern to filter formula repos
+ if [[ $repo =~ ^(.*formula.*)$ ]]; then
+ fetchGitFormula "$source/$repo";
+ fi
+ done;
+
+ done;
}
# detect if file is being sourced
[[ "$0" != "$BASH_SOURCE" ]] || {
- # if executed, run implicit function
- fetchGitFormula "${@}"
+ # if executed, fetch specific formula
+ fetchGitFormula ${@}
}
-
diff --git a/teams/team.sh b/teams/team.sh
index 2b55e9e..cb58587 100755
--- a/teams/team.sh
+++ b/teams/team.sh
@@ -5,5 +5,5 @@
awk -F';' '\
{ fname=$1;gsub(/\s+/,_,$1); sname=tolower(substr($1,1,1)substr($1,index(fname," "))); split($4, key ," "); }
- { print "FNAME=\\\""fname"\\\"", "SNAME=\\\""sname"\\\"", "KEY=\\\""key[1]" "key[2]"\\\"","EMAIL="$2" envsubst < team.template > "tolower($1)".yml" }'\
- < $1 | xargs -n11 echo
+ { print "FNAME=\""fname"\"", "SNAME=\""sname"\"", "KEY=\""key[1]" "key[2]"\"","EMAIL="$2" envsubst < team.template > "tolower($1)".yml" }'\
+ < $1
diff --git a/teams/team.template b/teams/team.template
index 496c817..ae1dfc2 100644
--- a/teams/team.template
+++ b/teams/team.template
@@ -5,13 +5,12 @@
${SNAME}:
enabled: true
name: ${SNAME}
- sudo: true
+ sudo: ${_param:linux_system_user_sudo}
full_name: ${FNAME}
home: /home/${SNAME}
email: ${EMAIL}
openssh:
server:
- enabled: true
user:
${SNAME}:
enabled: true