Upload OSLO templates for SF to work with Queens
* Adds some templates from oslo/keystoneauth.
* Adds rendering tests.
* Small cleanup after initial commit.
Change-Id: I8a1e37d132e6befd66d03da1809a95359f836365
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index a348912..96cb9ad 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -1,35 +1,25 @@
#!/usr/bin/env bash
-###
-# Script requirments:
-#apt-get install -y python-yaml virtualenv git
-
set -e
[ -n "$DEBUG" ] && set -x
CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
METADATA=${CURDIR}/../metadata.yml
FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
-FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
## Overrideable parameters
PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
BUILDDIR=${BUILDDIR:-${CURDIR}/build}
VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
-MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
DEPSDIR=${BUILDDIR}/deps
-SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"}
SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
-SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods}
SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
-IGNORE_MODELVALIDATE_MASK=${IGNORE_MODELVALIDATE_MASK:-"novalidate"}
-
if [ "x${SALT_VERSION}" != "x" ]; then
PIP_SALT_VERSION="==${SALT_VERSION}"
fi
@@ -45,31 +35,48 @@
setup_virtualenv() {
log_info "Setting up Python virtualenv"
- dependency_check virtualenv
virtualenv $VENV_DIR
source ${VENV_DIR}/bin/activate
python -m pip install salt${PIP_SALT_VERSION}
- if [[ -f ${CURDIR}/test-requirements.txt ]]; then
- python -m pip install -r ${CURDIR}/test-requirements.txt
- fi
}
-setup_mock_bin() {
- # If some state requires a binary, a lightweight replacement for
- # such binary can be put into MOCK_BIN_DIR for test purposes
- if [ -d "${MOCK_BIN_DIR}" ]; then
- PATH="${MOCK_BIN_DIR}:$PATH"
- export PATH
- fi
+setup_test_state() {
+ local template_path=$1
+ local state_name=$2
+
+ [ ! -d ${BUILDDIR}/tstates/ ] && mkdir ${BUILDDIR}/tstates/
+ [ ! -d ${BUILDDIR}/rfiles/ ] && mkdir ${BUILDDIR}/rfiles/
+
+ cat << EOF > ${BUILDDIR}/tstates/${state_name}.sls
+
+test_${state_name}_rendering:
+ file.managed:
+ - name: ${BUILDDIR}/rfiles/${state_name}.conf
+ - template: jinja
+ - source: ${template_path}
+ - context:
+ _data: {{ pillar.get("_data", {}) }}
+EOF
}
setup_pillar() {
[ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
echo "base:" > ${SALT_PILLAR_DIR}/top.sls
- for pillar in ${PILLARDIR}/*; do
- grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
- state_name=$(basename ${pillar%.sls})
- echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
+ local sdir
+ local state_name
+ local template_name
+ local pillar_name
+
+ pushd ${PILLARDIR}/
+ for spath in $(find ./ -type f -name '*.sls'); do
+ pillar_name=$(basename $spath | sed -e 's/.sls$//')
+ sdir=$(dirname $spath | sed -e 's/^.\///g')
+ template_name=$(echo $pillar_name | cut -d '-' -f 1)
+ state_name=$(echo ${sdir}_${pillar_name} | sed -e 's/\//_/g')
+ if ! echo $pillar_name |grep '-'; then
+ setup_test_state "salt://oslo_templates/files/$sdir/$template_name.conf" "$state_name"
+ fi
+ echo -e " ${state_name}:\n - ${sdir}/${pillar_name}" >> ${SALT_PILLAR_DIR}/top.sls
done
}
@@ -77,7 +84,6 @@
[ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
[ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
[ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
- [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR}
echo "base:" > ${SALT_FILE_DIR}/top.sls
for pillar in ${PILLARDIR}/*.sls; do
@@ -89,7 +95,6 @@
cat << EOF > ${SALT_CONFIG_DIR}/minion
file_client: local
cachedir: ${SALT_CACHE_DIR}
-extension_modules: ${SALT_CACHE_EXTMODS_DIR}
verify_env: False
minion_id_caching: False
@@ -97,6 +102,8 @@
base:
- ${SALT_FILE_DIR}
- ${CURDIR}/..
+ - ${BUILDDIR}/tstates/
+ - /usr/share/salt-formulas/env
pillar_roots:
base:
@@ -106,14 +113,13 @@
}
fetch_dependency() {
- # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux"
dep_name="$(echo $1|cut -d : -f 1)"
dep_source="$(echo $1|cut -d : -f 2-)"
dep_root="${DEPSDIR}/$(basename $dep_source .git)"
dep_metadata="${dep_root}/metadata.yml"
- dependency_check git
- [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; }
+ [ -d /usr/share/salt-formulas/env/${dep_name} ] && log_info "Dependency $dep_name already present in system-wide salt env" && return 0
+ [ -d $dep_root ] && log_info "Dependency $dep_name already fetched" && return 0
log_info "Fetching dependency $dep_name"
[ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
@@ -123,19 +129,6 @@
METADATA="${dep_metadata}" install_dependencies
}
-link_modules(){
- # Link modules *.py files to temporary salt-root
- local SALT_ROOT=${1:-$SALT_FILE_DIR}
- local SALT_ENV=${2:-$DEPSDIR}
-
- mkdir -p "${SALT_ROOT}/_modules/"
- # from git, development versions
- find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do
- ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ;
- done
- salt_run saltutil.sync_all
-}
-
install_dependencies() {
grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
(python - | while read dep; do fetch_dependency "$dep"; done) << EOF
@@ -152,96 +145,36 @@
salt_run() {
[ -e ${VENV_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
- python $(which salt-call) ${SALT_OPTS} $*
+ local cmd=''
+ cmd="python $(which salt-call) ${SALT_OPTS} $*"
+ log_info "$cmd"
+ $cmd
}
prepare() {
- if [[ -f ${BUILDDIR}/.prepare_done ]]; then
- log_info "${BUILDDIR}/.prepare_done exist, not rebuilding BUILDDIR"
- return
- fi
- [[ -d ${BUILDDIR} ]] && mkdir -p ${BUILDDIR}
+ [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
- [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
- setup_mock_bin
+ which salt-call || setup_virtualenv
setup_pillar
setup_salt
install_dependencies
- link_modules
- touch ${BUILDDIR}/.prepare_done
-}
-
-lint_releasenotes() {
- [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
- source ${VENV_DIR}/bin/activate
- reno lint ${CURDIR}/../
-}
-
-lint() {
-# lint_releasenotes
- log_err "TODO: lint_releasenotes"
}
run() {
- for pillar in ${PILLARDIR}/*.sls; do
- grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
- state_name=$(basename ${pillar%.sls})
- salt_run grains.set 'noservices' False force=True
-
- echo "Checking state ${FORMULA_NAME}.${state_name} ..."
- salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
-
- # Check that all files in 'meta' folder can be rendered using any valid pillar
- for meta in `find ${FORMULA_META_DIR} -type f`; do
- meta_name=$(basename ${meta})
- echo "Checking meta ${meta_name} ..."
- salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
- || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; }
- cat ${SALT_CACHE_DIR}/${meta_name}
- done
+ pushd ${PILLARDIR}/
+ local sdir
+ local sname
+ local state_name
+ local pillar_name
+ for spath in $(find ./ -type f -name '*.sls'); do
+ sname=$(basename $spath | basename $spath | sed -e 's/.sls$//')
+ sdir=$(dirname $spath | sed -e 's/^.\///')
+ state_name=$(echo ${sdir}_${sname} | sed -e 's/\//_/g' | cut -d '-' -f 1)
+ pillar_name=$(echo ${sdir}_${sname} | sed -e 's/\//_/g')
+ salt_run --id=${pillar_name} state.show_sls ${state_name} || (log_err "Show state ${state_name} with pillar ${pillar_name} failed"; exit 1)
+ salt_run --id=${pillar_name} state.sls ${state_name} || (log_err "Execution of ${state_name} with_pillar ${pillar_name} failed"; exit 1)
done
-}
-
-real_run() {
- for pillar in ${PILLARDIR}/*.sls; do
- state_name=$(basename ${pillar%.sls})
- salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; }
- done
-}
-
-run_model_validate(){
- # Run modelschema.model_validate validation.
- # TEST iterateble, run for `each formula ROLE against each ROLE_PILLARNAME`
- # Pillars should be named in conviend ROLE_XXX.sls or ROLE.sls
- # Example:
- # client.sls client_auth.sls server.sls server_auth.sls
- if [ -d ${SCHEMARDIR} ]; then
- # model validator require py modules
- fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt"
- link_modules
- salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all;
- for role in ${SCHEMARDIR}/*.yaml; do
- role_name=$(basename "${role%*.yaml}")
- for pillar in $(ls pillar/${role_name}*.sls | grep -v ${IGNORE_MODELVALIDATE_MASK} ); do
- pillar_name=$(basename "${pillar%*.sls}")
- local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${pillar_name}"
- log_info "model_validate ${_message}"
- # Rendered Example:
- # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster
- salt_run -m ${DEPSDIR}/salt-formula-salt --id=${pillar_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
- done
- done
- else
- log_info "${SCHEMARDIR} not found!";
- fi
-}
-
-dependency_check() {
- local DEPENDENCY_COMMANDS=$*
-
- for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do
- which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; )
- done
+ popd
}
_atexit() {
@@ -257,32 +190,22 @@
}
## Main
-trap _atexit INT TERM EXIT
+[[ "$0" != "$BASH_SOURCE" ]] || {
+ trap _atexit INT TERM EXIT
-case $1 in
- clean)
- clean
- ;;
- prepare)
- prepare
- ;;
- lint)
- lint
- ;;
- run)
- run
- ;;
- real-run)
- real_run
- ;;
- model-validate)
- prepare
- run_model_validate
- ;;
- *)
- prepare
-# lint
- run
- run_model_validate
- ;;
-esac
+ case $1 in
+ clean)
+ clean
+ ;;
+ prepare)
+ prepare
+ ;;
+ run)
+ run
+ ;;
+ *)
+ prepare
+ run
+ ;;
+ esac
+}