Merge pull request #35 from salt-formulas/develop
Don't install reclass pkg out-of-the-box
diff --git a/bootstrap.sh b/bootstrap.sh
index f12fc1c..5ec7067 100755
--- a/bootstrap.sh
+++ b/bootstrap.sh
@@ -6,10 +6,9 @@
# - http://github.com/salt-formulas-scripts
# - http://github.com/salt-formulas/salt-formula-salt (salt.master sls)
-# TODO:
-# - use PPA repository as formula source
-# - support for spm/yum
-
+# NOTE: This script is collection of shared functions to automate bootstrap of "salted" CI environments.
+# Its not intended to be used in PRODUCTION unless you know what you are doing.
+# You have been warned!
# Source specific env vars.
# shopt -u dotglob
@@ -89,6 +88,9 @@
# the version below is used salt pillar data
SALT_VERSION=${SALT_VERSION:-latest}
+# SECURITY
+SSH_STRICTHOSTKEYCHECKING=no
+
# environment
if [ "$FORMULAS_SOURCE" == "git" ]; then
SALT_ENV=${SALT_ENV:-dev}
@@ -218,17 +220,33 @@
##########################################
# Main calls
-system_config_ssh_conf() {
- for conf in ~/.ssh/config /root/.ssh/config; do
- $SUDO mkdir -p $(dirname $conf)
- if ! grep StrictHostKeyChecking $conf; then
- # this should be used only in CI environment
- echo -e "Host *\n\tStrictHostKeyChecking no\n" | $SUDO tee $conf >/dev/null
- fi
- done
+ci_config_ssh() {
+
+ # for CI current user
+ conf=~/.ssh/config
+ mkdir -p $(dirname $conf)
+ touch $conf
+ echo -e "Host *\n\tStrictHostKeyChecking $SSH_STRICTHOSTKEYCHECKING\n" | tee -a $conf >/dev/null
+ touch ~/.ssh/known_hosts
+
if ! grep github.com ~/.ssh/known_hosts; then
ssh-keyscan -H github.com >> ~/.ssh/known_hosts || true
+ ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts || true
fi
+
+ # for root
+ #conf=/root/.ssh/config
+ #$SUDO mkdir -p $(dirname $conf)
+ #$SUDO touch $conf
+ #if ! $SSH_STRICTHOSTKEYCHECKING; then
+ # echo -e "Host *\n\tStrictHostKeyChecking no\n" | $SUDO tee -a $conf >/dev/null
+ #fi
+}
+
+# DEPRECATED
+system_config_ssh_conf() {
+ # for backward compatibility
+ ci_config_ssh
}
system_config_salt_modules_prereq() {
@@ -244,19 +262,11 @@
log_info "System configuration salt master"
system_config_salt_modules_prereq
- system_config_ssh_conf
+ ci_config_ssh
if ! grep '127.0.1.2.*salt' /etc/hosts; then
echo "127.0.1.2 salt" | $SUDO tee -a /etc/hosts >/dev/null
fi
-
- which reclass || $SUDO $PKGTOOL install -y reclass
-
- which reclass-salt || {
- test -e /usr/share/reclass/reclass-salt && {
- ln -fs /usr/share/reclass/reclass-salt /usr/bin
- }
- }
}
configure_salt_master()
@@ -392,6 +402,11 @@
case ${VERSION} in
pkg|package)
which reclass || $SUDO $PKGTOOL install -y reclass
+ which reclass-salt || {
+ if [ -e /usr/share/reclass/reclass-salt ]; then
+ ln -fs /usr/share/reclass/reclass-salt /usr/bin
+ fi
+ }
;;
*)
log_warn "Install development version of reclass"
@@ -426,12 +441,6 @@
;;
esac
- which reclass-salt || {
- test -e /usr/share/reclass/reclass-salt && {
- ln -fs /usr/share/reclass/reclass-salt /usr/bin
- }
- }
-
configure_salt_master
echo -e "\nRestarting services ...\n"
@@ -464,12 +473,6 @@
curl -Lo /etc/init.d/salt-master https://anonscm.debian.org/cgit/pkg-salt/salt.git/plain/debian/salt-master.init && chmod 755 /etc/init.d/salt-master
ln -s /usr/local/bin/salt-master /usr/bin/salt-master
- which reclass-salt || {
- test -e /usr/share/reclass/reclass-salt && {
- ln -fs /usr/share/reclass/reclass-salt /usr/bin
- }
- }
-
configure_salt_master
echo -e "\nRestarting services ...\n"
@@ -709,6 +712,83 @@
}
+## CI Workarounds
+
+function mockup_node_registration() {
+
+ # for dynamic nodes registrations (require github.com/salt-formulas/salt-formula-reclass)
+
+ source /etc/os-release
+ os_codename=$VERSION_CODENAME
+ domain=$(hostname -d)
+ master_ip=$(hostname -I | awk '{print $1}')
+ fake_ip_preffix=$(echo $master_ip | awk -F. '{print $1"."$2}')
+ fake_ip_base=10
+
+ # SHOULD BE ALREADY RUN AS A PART OF BOOTSTRAP
+ # sync, in order to load custom modules
+ #salt-call saltutil.sync_all
+
+ # SHOULD BE ALREADY RUN AS A PART OF BOOTSTRAP
+ #PILLAR='{"reclass":{"storage":{"data_source":{"engine":"local"}}} }'
+ #salt-call state.apply reclass.storage.node pillar="$PILLAR" > /dev/null 2>/dev/null || true
+
+ # rotate over dynamic hosts
+ for host_idx in {01..02};do
+ for host in `salt-call pillar.items reclass:storage |grep '<<node_hostname>>' | awk -F_ '{print $NF}' | sed 's/[0-9]*//g' | egrep -v cfg | sort -u`; do
+ hostname=${host}${host_idx}
+ fake_ip_base=$(($fake_ip_base + 1))
+
+ node_network01_ip="$fake_ip_preffix.11.$fake_ip_base"
+ node_network02_ip="$fake_ip_preffix.12.$fake_ip_base"
+ node_network03_ip="$fake_ip_preffix.13.$fake_ip_base"
+ node_network04_ip="$fake_ip_preffix.14.$fake_ip_base"
+ node_network05_ip="$fake_ip_preffix.15.$fake_ip_base"
+ node_network01_iface=$(ls /sys/class/net/ | grep -v lo | sort | head -n1)
+ node_network02_iface="eth1"
+ node_network03_iface="eth2"
+ node_network04_iface="eth3"
+ node_network05_iface="eth4"
+
+ declare -A vars
+ vars=(
+ ["node_master_ip"]=
+ ["node_os"]=${os_codename}
+ ["node_deploy_ip"]=${node_network01_ip}
+ ["node_deploy_iface"]=${node_network01_iface}
+ ["node_control_ip"]=${node_network02_ip}
+ ["node_control_iface"]=${node_network02_iface}
+ ["node_tenant_ip"]=${node_network03_ip}
+ ["node_tenant_iface"]=${node_network03_iface}
+ ["node_external_ip"]=${node_network04_ip}
+ ["node_external_iface"]=${node_network04_iface}
+ ["node_baremetal_ip"]=${node_network05_ip}
+ ["node_baremetal_iface"]=${node_network05_iface}
+ ["node_domain"]=$domain
+ ["node_cluster"]=$(hostname -d |awk -F. '{print $1}')
+ ["node_hostname"]=$hostname
+ )
+
+ data=""; i=0
+ for key in "${!vars[@]}"; do
+ data+="\"${key}\": \"${vars[${key}]}\""
+ i=$(($i+1))
+ if [ $i -lt ${#vars[@]} ]; then
+ data+=", "
+ fi
+ done
+ echo "Classifying node $hostname"
+ NODECR='"node_name": "'${hostname}.${domain}'", "node_data": {'$data'}'
+ PILLAR='{'${NODECR}', "reclass":{"storage":{"data_source":{"engine":"local"}}} }'
+ salt-call state.apply reclass.reactor_sls.node_register pillar="$PILLAR" #-l info
+ #salt-call event.send "reclass/minion/classify" "{$data}"
+ done
+ done
+
+}
+
+## VERIFY
+
function verify_salt_master() {
set -e
@@ -725,7 +805,7 @@
$SUDO salt-call --no-color pillar.data
fi
# TODO: REMOVE reclass --nodeinfo section / run only on debug - as the only required is reclass.validate_*
- if ! $SUDO reclass --nodeinfo ${MASTER_HOSTNAME} > /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo; then
+ if ! $SUDO python -m reclass.cli --nodeinfo ${MASTER_HOSTNAME} > /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo; then
log_err "For more details see full log /tmp/${MASTER_HOSTNAME}.reclass.nodeinfo"
exit 1
fi
@@ -740,7 +820,7 @@
$SUDO salt-call ${SALT_OPTS} --id=${node} grains.item roles > /tmp/${node}.grains.item.roles
$SUDO salt-call ${SALT_OPTS} --id=${node} state.show_lowstate > /tmp/${node}.state.show_lowstate
fi
- if ! $SUDO reclass --nodeinfo ${node} > /tmp/${node}.reclass.nodeinfo; then
+ if ! $SUDO python -m reclass.cli --nodeinfo ${node} > /tmp/${node}.reclass.nodeinfo; then
log_err "For more details see full log /tmp/${node}.reclass.nodeinfo"
if [[ ${BREAK_ON_VERIFICATION_ERROR:-yes} =~ ^(True|true|1|yes)$ ]]; then
exit 1
diff --git a/formula-fetch.sh b/formula-fetch.sh
index 03153d3..f103fcd 100755
--- a/formula-fetch.sh
+++ b/formula-fetch.sh
@@ -3,13 +3,34 @@
# Usage:
# ./formula-fetch.sh <Formula URL> <Name> <Branch>
#
-# Example:
-# GIT_FORMULAS_PATH=.vendor/formulas ./formula-fetch.sh https://github.com/salt-formulas/salt-formula-salt
+# Example usage:
+# FORMULA_SOURCES=https://github.com/epcim/my-salt-formulas https://github.com/salt-formulas https://github.com/saltstack-formulas
+# SALT_ENV_PATH=.vendor/formulas
# --
-# GIT_FORMULAS_PATH=/usr/share/salt-formulas/env/_formulas
+# ./formula-fetch.sh
# xargs -n1 ./formula-fetch.sh < dependencies.txt
+## DEFAULTS
+#
+# default sources
+FORMULA_SOURCES="${SALT_FORMULA_SOURCES:-https://github.com/salt-formulas https://github.com/saltstack-formulas}"
+FORMULA_VERSION="${SALT_FORMULA_VERSION:-master}"
+# where to fetch formulas
+FORMULAS_BASE=${SALT_FORMULAS_BASE:-/srv/salt/formula}
+# For better stability, skip formula repos without recognized CI
+FORMULA_WITHOUT_CI=${SALT_FORMULA_WITHOUT_CI:-false}
+# salt env/root, where formulas are found
+SALT_ENV_PATH=${SALT_ENV_PATH:-/srv/salt/env/prd}
+#SALT_ENV_PATH=${SALT_ENV_PATH:-.vendor/formulas}
+#SALT_ENV_PATH=${SALT_ENV_PATH:/usr/share/salt-formulas/env/_formulas}
+# reclass related
+RECLASS_BASE=${RECLASS_BASE:-/srv/salt/reclass}
+# env
+LC_ALL=en_US.UTF-8
+LANG=en_US.UTF-8
+
+
# Parse git dependencies from metadata.yml
# $1 - path to <formula>/metadata.yml
# sample to output:
@@ -17,46 +38,149 @@
# https://github.com/salt-formulas/salt-formula-salt salt
function fetchDependencies() {
METADATA="$1";
- grep -E "^dependencies:" "$METADATA" >/dev/null || return 0
- (python - "$METADATA" | while read dep; do fetchGitFormula "$dep"; done) <<-DEPS
+ grep -E "^dependencies:" "$METADATA" &>/dev/null || return 0
+ (python3 - "$METADATA" | while read dep; do fetchGitFormula $dep; done) <<-DEPS
import sys,yaml
- for dep in yaml.load(open(sys.argv[1], "ro"))["dependencies"]:
- print("{source} {name}").format(**dep)
+ try:
+ for dep in yaml.load(open(sys.argv[1], "r"))["dependencies"]:
+ if len(set(('name', 'source')) & set(dep.keys())) == 2:
+ print("{source} {name}".format(**dep))
+ except Exception as e:
+ print("[W] {}".format(e.__doc__))
+ print("[W] {}".format(e.message))
+ pass
DEPS
}
+
+# Read formula name from meetadata.yml
+# $1 - path to <formula>/metadata.yml
+function getFormulaName() {
+ python3 - "$1" <<-READ_NAME
+ try:
+ import sys,yaml;print(yaml.load(open(sys.argv[1], "r"))["name"]);
+ except Exception as e:
+ print("[W] {}".format(e.__doc__))
+ print("[W] {}".format(e.message))
+ pass
+ READ_NAME
+}
+
+
# Fetch formula from git repo
# $1 - formula git repo url
# $2 - formula name (optional)
# $3 - branch (optional)
function fetchGitFormula() {
test -n "${FETCHED}" || declare -a FETCHED=()
- export GIT_FORMULAS_PATH=${GIT_FORMULAS_PATH:-/usr/share/salt-formulas/env/_formulas}
- mkdir -p "$GIT_FORMULAS_PATH"
+ mkdir -p "$SALT_ENV_PATH" "$FORMULAS_BASE"
+
if [ -n "$1" ]; then
- source="$1"
- name="$2"
- test -n "$name" || name="${source//*salt-formula-}"
- test -z "$3" && branch=master || branch=$3
+
+ # set origin uri
+ # FIXME, TEMP fix for not yet up to date gh:salt-formulas -> s/tcpcloud/salt-formulas/
+ origin="${1/tcpcloud/salt-formulas}"
+ # set gh repo https://salt-formulas/salt-formula-salt -> $FORMULAS_BASE/salt-formulas/salt-formula-salt
+ repo=$(echo $origin | awk -F'/' '{ print substr($0, index($0,$4)) }')
+ # set normula name
+ test -n "$2" && name=$2 || name="$(echo ${origin//*\/} | sed -e 's/-formula$//' -e 's/^salt-formula-//' -e 's/^formula-//')"
+ # set branch
+ test -n "$3" && branch=$3 || branch=${FORMULA_VERSION}
+
+ # DEBUG
+ #echo '--- ------------------------------'
+ #echo origin, $origin
+ #echo repo, $repo
+ #echo fetched ${FETCHED[@]}
+ #echo -e name, $name
+ #echo '---'
+ #return
+
if ! [[ "${FETCHED[*]}" =~ $name ]]; then # dependency not yet fetched
- echo "Fetching: $name"
- if test -e "$GIT_FORMULAS_PATH/$name"; then
- pushd "$GIT_FORMULAS_PATH/$name" &>/dev/null
- test ! -e .git || git pull -r
+ echo -e "[I] Fetching: $origin -> $FORMULAS_BASE/$repo"
+ if [ -e "$FORMULAS_BASE/$repo" ]; then
+ pushd "$FORMULAS_BASE/$repo" &>/dev/null
+ git pull -r; git checkout $branch;
popd &>/dev/null
else
- echo "git clone $source $GIT_FORMULAS_PATH/$name -b $branch"
- git clone "$source" "$GIT_FORMULAS_PATH/$name" -b "$branch"
+ echo -e "[I] git clone $origin $FORMULAS_BASE/$repo -b $branch"
+ if ! git ls-remote --exit-code --heads $origin $branch; then
+ # Fallback to the master branch if the branch doesn't exist for this repository
+ branch=master
+ fi
+ if ! git clone "$origin" "$FORMULAS_BASE/$repo" -b "$branch"; then
+ echo -e "[E] Fetching formula from $origin failed."
+ return ${FAIL_ON_ERRORS:-0}
+ fi
fi
- # install dependencies
- FETCHED+=($name)
- fetchDependencies "$GIT_FORMULAS_PATH/$name/metadata.yml"
+
+ # A metadata.yml is github.com/salt-formulas specific
+ if [ ! -n "$2" -a -e "$FORMULAS_BASE/$repo/metadata.yml" ]; then
+ # try to update name as in formula metadata
+ name=$(getFormulaName "$FORMULAS_BASE/$repo/metadata.yml")
+ fi
+
+ # FIXME, better formula recognition/name fixup for saltstack formulas
+ # Recognize the repo is formula
+ if [ ! -e $FORMULAS_BASE/$repo/$name ]; then
+ echo -e "[E] The repository $FORMULAS_BASE/$repo was not recognized as formula repository."
+ rm -rf "$FORMULAS_BASE/$repo"
+ return ${FAIL_ON_ERRORS:-0}
+ fi
+
+ # Avoid checkout formulas/repos without CI
+ if ! $FORMULA_WITHOUT_CI; then
+ CI=false
+ for p in .circleci .travis.yml .kitchen.yml invoke.yml tasks.py tox.ini test tests; do
+ if [ -e "$FORMULAS_BASE/$repo/$p" ]; then
+ CI=true; break;
+ fi
+ done
+ if ! $CI; then
+ mv "$FORMULAS_BASE/$repo" "$FORMULAS_BASE/${repo}.deprecated-no-ci";
+ return ${FAIL_ON_ERRORS:-0}
+ fi
+ fi
+
+ # SET FORMULA IN SALT ENV
+ if [ ! -e "$SALT_ENV_PATH/$name" ]; then
+
+ # link formula
+ ln -svf $FORMULAS_BASE/$repo/$name $SALT_ENV_PATH/$name
+
+ # copy custom _states, _modules, _etc ...
+ for c in $(/bin/ls $FORMULAS_BASE/$repo | grep '^_' | xargs -n1 --no-run-if-empty); do
+ test -e $SALT_ENV_PATH/$c || mkdir -p $SALT_ENV_PATH/$c
+ ln -svf $FORMULAS_BASE/$repo/$c/* $SALT_ENV_PATH/$c
+ done
+
+ # install optional dependencies (python/pip related as of now only)
+ if [ -e $FORMULAS_BASE/$repo/requirements.txt ]; then
+ pip install -r $FORMULAS_BASE/$repo/requirements.txt
+ fi
+
+ # NOTE: github.com/salt-formulas specific steps
+ # link formula service pillars
+ if [ -n "$RECLASS_BASE" -a -e "$FORMULAS_BASE/$repo/metadata/service" ]; then
+ test -e $RECLASS_BASE/classes/service || mkdir -p $RECLASS_BASE/classes/service
+ ln -svf $FORMULAS_BASE/$repo/metadata/service $RECLASS_BASE/classes/service/$name
+ fi
+ # install dependencies
+ FETCHED+=($name)
+ if [ -e "$FORMULAS_BASE/$repo/metadata.yml" ]; then
+ fetchDependencies "$FORMULAS_BASE/$repo/metadata.yml"
+ fi
+ else
+ echo -e "[I] Formula "$name" already fetched."
+ fi
fi
else
- echo Usage: fetchGitFormula "<git repo>" "[local formula directory name]" "[branch]"
+ echo -e '[I] Usage: fetchGitFormula git_repo_uri [branch] [local formula directory name]'
fi
}
+# DEPRECATED, kept for backward compatibility
+# for github.com/salt-formulas (linking "service" pillar metadata from formula to reclas classes)
function linkFormulas() {
# OPTIONAL: Link formulas from git/pkg
@@ -70,12 +194,76 @@
# form pkgs
find "$SALT_ENV" -maxdepth 1 -mindepth 1 -path "*_formulas*" -prune -o -name "*" -type d -print0| xargs -0 -n1 --no-run-if-empty basename | xargs -I{} --no-run-if-empty \
ln -fs "$SALT_ENV"/{} "$SALT_ROOT"/{};
+}
+
+function setupPyEnv() {
+ MODULES="pygithub pyyaml"
+ pip3 install --upgrade $MODULES || {
+ which pipenv || {
+ pip install --upgrade pipenv
+ }
+ pipenv --three
+ pipenv install $MODULES
+ }
+}
+
+function listRepos_github_com() {
+ #export python=$(pipenv --py || (setupPyEnv &>/dev/null; pipenv --py))
+ if [ -e Pipfile.lock ]; then python=$(pipenv --py); else python=python3; fi
+ $python - "$1" <<-LIST_REPOS
+ import sys
+ import github
+
+ def make_github_agent(user=None, password=None):
+ """ Create github agent to auth """
+ if not user:
+ return github.Github()
+ else:
+ return github.Github(user, password)
+
+ def get_org_repos(gh, org_name):
+ org = gh.get_organization(org_name)
+ for repo in org.get_repos():
+ yield repo.name
+
+ try:
+ print(*get_org_repos(make_github_agent(), str(sys.argv[1])), sep="\n")
+ except Exception as e:
+ print("[E] {}".format(e.__doc__))
+ print("[E] {}".format(e.message))
+ sys.exit(1)
+ LIST_REPOS
+}
+
+function fetchAll() {
+ # iterate over all defined sources
+ for source in $(echo ${FORMULA_SOURCES} | xargs -n1 --no-run-if-empty); do
+ hosting=$(echo ${source//\./_} | awk -F'/' '{print $3}')
+ orgname=$(echo ${source//\./_} | awk -F'/' '{print $4}')
+
+ # Get repos. To protect builds on master we likely fail on errors/none while getting repos from $source
+ set -o pipefail
+ repos="$(listRepos_$hosting "$orgname" | xargs -n1 --no-run-if-empty| sort)"
+ set +o pipefail
+ if [ ! -n "$repos" ]; then
+ echo "[E] Error caught or no repositories found at $source. Exiting.";
+ exit 1;
+ fi
+
+ # fetch all repos that looks like formula
+ for repo in $(echo ${repos} | xargs -n1 --no-run-if-empty); do
+ # TODO, avoid a hardcoded pattern to filter formula repos
+ if [[ $repo =~ ^(.*formula.*)$ ]]; then
+ fetchGitFormula "$source/$repo";
+ fi
+ done;
+
+ done;
}
# detect if file is being sourced
[[ "$0" != "$BASH_SOURCE" ]] || {
- # if executed, run implicit function
- fetchGitFormula "${@}"
+ # if executed, fetch specific formula
+ fetchGitFormula ${@}
}
-