blob: 4665d343bcfa6848f6af73f799e2b52127c8983e [file] [log] [blame]
Filip Pytloun235949e2016-01-25 18:19:39 +01001#!/usr/bin/env bash
2
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +01003###
4# Script source: https://gerrit.mcp.mirantis.com/#/admin/projects/salt-formulas/cookiecutter-salt-formula
5# Script requirments:
6#apt-get install -y python-yaml virtualenv git
7
8__ScriptVersion="2019.01.07"
9__ScriptName="run_tests.sh"
10__ScriptFullName="$0"
11__ScriptArgs="$*"
12
Filip Pytloun235949e2016-01-25 18:19:39 +010013set -e
14[ -n "$DEBUG" ] && set -x
15
16CURDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
17METADATA=${CURDIR}/../metadata.yml
18FORMULA_NAME=$(cat $METADATA | python -c "import sys,yaml; print yaml.load(sys.stdin)['name']")
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +030019FORMULA_META_DIR=${CURDIR}/../${FORMULA_NAME}/meta
Filip Pytloun235949e2016-01-25 18:19:39 +010020
21## Overrideable parameters
22PILLARDIR=${PILLARDIR:-${CURDIR}/pillar}
23BUILDDIR=${BUILDDIR:-${CURDIR}/build}
24VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +030025MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
Filip Pytloun235949e2016-01-25 18:19:39 +010026DEPSDIR=${BUILDDIR}/deps
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010027SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"}
Filip Pytloun235949e2016-01-25 18:19:39 +010028
29SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
30SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
31SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
32SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010033SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods}
Filip Pytloun235949e2016-01-25 18:19:39 +010034
Simak, Jan87340f82016-06-23 13:52:43 +020035SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
Filip Pytloun235949e2016-01-25 18:19:39 +010036
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010037IGNORE_MODELVALIDATE_MASK=${IGNORE_MODELVALIDATE_MASK:-"novalidate"}
38
Filip Pytloun235949e2016-01-25 18:19:39 +010039if [ "x${SALT_VERSION}" != "x" ]; then
40 PIP_SALT_VERSION="==${SALT_VERSION}"
41fi
42
43## Functions
44log_info() {
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010045 echo -e "[INFO] $*"
Filip Pytloun235949e2016-01-25 18:19:39 +010046}
47
48log_err() {
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010049 echo -e "[ERROR] $*" >&2
Filip Pytloun235949e2016-01-25 18:19:39 +010050}
51
52setup_virtualenv() {
53 log_info "Setting up Python virtualenv"
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010054 dependency_check virtualenv
Filip Pytloun235949e2016-01-25 18:19:39 +010055 virtualenv $VENV_DIR
56 source ${VENV_DIR}/bin/activate
Jakub Joseffbbbbfa2018-02-08 13:22:18 +010057 python -m pip install salt${PIP_SALT_VERSION}
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010058 if [[ -f ${CURDIR}/test-requirements.txt ]]; then
59 python -m pip install -r ${CURDIR}/test-requirements.txt
60 fi
Filip Pytloun235949e2016-01-25 18:19:39 +010061}
62
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +030063setup_mock_bin() {
64 # If some state requires a binary, a lightweight replacement for
65 # such binary can be put into MOCK_BIN_DIR for test purposes
66 if [ -d "${MOCK_BIN_DIR}" ]; then
67 PATH="${MOCK_BIN_DIR}:$PATH"
68 export PATH
69 fi
70}
71
Filip Pytloun235949e2016-01-25 18:19:39 +010072setup_pillar() {
73 [ ! -d ${SALT_PILLAR_DIR} ] && mkdir -p ${SALT_PILLAR_DIR}
74 echo "base:" > ${SALT_PILLAR_DIR}/top.sls
75 for pillar in ${PILLARDIR}/*; do
Petr Michalec5f41e6e2017-04-10 10:18:12 +020076 grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
Filip Pytloun235949e2016-01-25 18:19:39 +010077 state_name=$(basename ${pillar%.sls})
78 echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
79 done
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010080 for pillar in $(find $PILLARDIR -mindepth 2 -type f -iname *.sls); do
81 state_name=$(basename "${pillar%*.sls}")
82 os_release=$(echo $pillar | rev | cut -d'/' -f2 | rev)
83 grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
84 echo -e " ${os_release}_${state_name}:\n - ${os_release}.${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
85 done
Filip Pytloun235949e2016-01-25 18:19:39 +010086}
87
88setup_salt() {
89 [ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
90 [ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
91 [ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +010092 [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR}
Filip Pytloun235949e2016-01-25 18:19:39 +010093
94 echo "base:" > ${SALT_FILE_DIR}/top.sls
95 for pillar in ${PILLARDIR}/*.sls; do
Petr Michalec5f41e6e2017-04-10 10:18:12 +020096 grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
Filip Pytloun235949e2016-01-25 18:19:39 +010097 state_name=$(basename ${pillar%.sls})
98 echo -e " ${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
99 done
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100100 for pillar in $(find $PILLARDIR -mindepth 2 -type f -iname *.sls); do
101 state_name=$(basename "${pillar%*.sls}")
102 os_release=$(echo $pillar | rev | cut -d'/' -f2 | rev)
103 grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
104 echo -e " ${os_release}_${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
105 done
Filip Pytloun235949e2016-01-25 18:19:39 +0100106
107 cat << EOF > ${SALT_CONFIG_DIR}/minion
108file_client: local
109cachedir: ${SALT_CACHE_DIR}
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100110extension_modules: ${SALT_CACHE_EXTMODS_DIR}
Filip Pytloun235949e2016-01-25 18:19:39 +0100111verify_env: False
Ondřej Novýae1ffe12016-04-23 17:11:07 +0200112minion_id_caching: False
Filip Pytloun235949e2016-01-25 18:19:39 +0100113
114file_roots:
115 base:
116 - ${SALT_FILE_DIR}
117 - ${CURDIR}/..
118
119pillar_roots:
120 base:
121 - ${SALT_PILLAR_DIR}
122 - ${PILLARDIR}
123EOF
124}
125
126fetch_dependency() {
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100127 # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux"
Filip Pytloun3ef8a312016-04-14 11:51:09 +0200128 dep_name="$(echo $1|cut -d : -f 1)"
129 dep_source="$(echo $1|cut -d : -f 2-)"
130 dep_root="${DEPSDIR}/$(basename $dep_source .git)"
Filip Pytloun235949e2016-01-25 18:19:39 +0100131 dep_metadata="${dep_root}/metadata.yml"
132
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100133 dependency_check git
134 [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; }
Filip Pytloun235949e2016-01-25 18:19:39 +0100135
Filip Pytloun3ef8a312016-04-14 11:51:09 +0200136 log_info "Fetching dependency $dep_name"
Filip Pytloun235949e2016-01-25 18:19:39 +0100137 [ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
Filip Pytloun3ef8a312016-04-14 11:51:09 +0200138 git clone $dep_source ${DEPSDIR}/$(basename $dep_source .git)
Filip Pytloun235949e2016-01-25 18:19:39 +0100139 ln -s ${dep_root}/${dep_name} ${SALT_FILE_DIR}/${dep_name}
140
141 METADATA="${dep_metadata}" install_dependencies
142}
143
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100144link_modules(){
145 # Link modules *.py files to temporary salt-root
146 local SALT_ROOT=${1:-$SALT_FILE_DIR}
147 local SALT_ENV=${2:-$DEPSDIR}
148
149 mkdir -p "${SALT_ROOT}/_modules/"
150 # from git, development versions
151 find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do
152 ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ;
153 done
154 salt_run saltutil.sync_all
155}
156
Filip Pytloun235949e2016-01-25 18:19:39 +0100157install_dependencies() {
158 grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
159 (python - | while read dep; do fetch_dependency "$dep"; done) << EOF
160import sys,yaml
161for dep in yaml.load(open('${METADATA}', 'ro'))['dependencies']:
Filip Pytloun3ef8a312016-04-14 11:51:09 +0200162 print '%s:%s' % (dep["name"], dep["source"])
Filip Pytloun235949e2016-01-25 18:19:39 +0100163EOF
164}
165
166clean() {
167 log_info "Cleaning up ${BUILDDIR}"
168 [ -d ${BUILDDIR} ] && rm -rf ${BUILDDIR} || exit 0
169}
170
171salt_run() {
Jakub Josefd28b6782017-12-15 16:58:30 +0100172 [ -e ${VENV_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
Jakub Joseffbbbbfa2018-02-08 13:22:18 +0100173 python $(which salt-call) ${SALT_OPTS} $*
Filip Pytloun235949e2016-01-25 18:19:39 +0100174}
175
176prepare() {
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100177 if [[ -f ${BUILDDIR}/.prepare_done ]]; then
178 log_info "${BUILDDIR}/.prepare_done exist, not rebuilding BUILDDIR"
179 return
180 fi
181 [[ -d ${BUILDDIR} ]] && mkdir -p ${BUILDDIR}
Filip Pytloun235949e2016-01-25 18:19:39 +0100182
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100183 [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +0300184 setup_mock_bin
Filip Pytloun235949e2016-01-25 18:19:39 +0100185 setup_pillar
186 setup_salt
187 install_dependencies
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100188 link_modules
189 touch ${BUILDDIR}/.prepare_done
190}
191
192lint_releasenotes() {
193 [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
194 source ${VENV_DIR}/bin/activate
195 reno lint ${CURDIR}/../
196}
197
198lint() {
199# lint_releasenotes
200 log_err "TODO: lint_releasenotes"
Filip Pytloun235949e2016-01-25 18:19:39 +0100201}
202
203run() {
204 for pillar in ${PILLARDIR}/*.sls; do
Petr Michalec5f41e6e2017-04-10 10:18:12 +0200205 grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
Filip Pytloun235949e2016-01-25 18:19:39 +0100206 state_name=$(basename ${pillar%.sls})
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +0300207 salt_run grains.set 'noservices' False force=True
208
209 echo "Checking state ${FORMULA_NAME}.${state_name} ..."
Filip Pytloun235949e2016-01-25 18:19:39 +0100210 salt_run --id=${state_name} state.show_sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +0300211
212 # Check that all files in 'meta' folder can be rendered using any valid pillar
213 for meta in `find ${FORMULA_META_DIR} -type f`; do
214 meta_name=$(basename ${meta})
215 echo "Checking meta ${meta_name} ..."
216 salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100217 || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; }
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +0300218 cat ${SALT_CACHE_DIR}/${meta_name}
219 done
220 done
221}
222
223real_run() {
224 for pillar in ${PILLARDIR}/*.sls; do
225 state_name=$(basename ${pillar%.sls})
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100226 salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; }
Filip Pytloun235949e2016-01-25 18:19:39 +0100227 done
228}
229
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100230run_model_validate(){
231 # Run modelschema.model_validate validation.
232 # TEST iterateble, run for `each formula ROLE against each ROLE_PILLARNAME`
233 # Pillars should be named in conviend ROLE_XXX.sls or ROLE.sls
234 # Example:
235 # client.sls client_auth.sls server.sls server_auth.sls
236 if [ -d ${SCHEMARDIR} ]; then
237 # model validator require py modules
238 fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt"
239 link_modules
240 salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all;
241 for role in $(find $SCHEMARDIR/* -maxdepth 0 -type f -iname *.yaml); do
242 role_name=$(basename "${role%*.yaml}")
243 for pillar in $(ls pillar/${role_name}*.sls | grep -v ${IGNORE_MODELVALIDATE_MASK} ); do
244 pillar_name=$(basename "${pillar%*.sls}")
245 local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${pillar_name}"
246 log_info "model_validate ${_message}"
247 # Rendered Example:
248 # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster
249 salt_run -m ${DEPSDIR}/salt-formula-salt --id=${pillar_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
250 done
251 done
252 for schema in $(find $SCHEMARDIR -mindepth 2 -type f -iname *.yaml); do
253 role_name=$(basename "${schema%*.yaml}")
254 os_release=$(echo $schema | rev | cut -d'/' -f2 | rev)
255 local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${role_name}"
256 log_info "model_validate ${_message}"
257 salt_run -m ${DEPSDIR}/salt-formula-salt --id=${os_release}_${role_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} ${os_release} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
258 done
259 else
260 log_info "${SCHEMARDIR} not found!";
261 fi
262}
263
264dependency_check() {
265 local DEPENDENCY_COMMANDS=$*
266
267 for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do
268 which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; )
269 done
270}
271
Filip Pytloun235949e2016-01-25 18:19:39 +0100272_atexit() {
273 RETVAL=$?
274 trap true INT TERM EXIT
275
276 if [ $RETVAL -ne 0 ]; then
277 log_err "Execution failed"
278 else
279 log_info "Execution successful"
280 fi
281 return $RETVAL
282}
283
284## Main
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100285
286log_info "Running version: ${__ScriptVersion}"
287log_info "Command line: '${__ScriptFullName} ${__ScriptArgs}'"
288
Filip Pytloun235949e2016-01-25 18:19:39 +0100289trap _atexit INT TERM EXIT
290
291case $1 in
292 clean)
293 clean
294 ;;
295 prepare)
296 prepare
297 ;;
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100298 lint)
299 lint
300 ;;
Filip Pytloun235949e2016-01-25 18:19:39 +0100301 run)
302 run
303 ;;
Oleksandr Shyshkof16baae2018-07-31 18:01:12 +0300304 real-run)
305 real_run
306 ;;
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100307 model-validate)
308 prepare
309 run_model_validate
310 ;;
Filip Pytloun235949e2016-01-25 18:19:39 +0100311 *)
312 prepare
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100313# lint
Filip Pytloun235949e2016-01-25 18:19:39 +0100314 run
Dzmitry Stremkouski8babb812020-02-16 17:02:44 +0100315 run_model_validate
Filip Pytloun235949e2016-01-25 18:19:39 +0100316 ;;
317esac