Change some default values for Rabbit driver
It was observed that for the rabbitmq-server version 3.8.2 default
values during failover of one of the rabbit nodes cause rapid connection
recreation. In some cases it leads to the creation of broken exchanges
and hangs of OpenStack operations. Changing rabbit_retry_interval to 5,
rabbit_retry_backoff to 10 and kombu_reconnect_delay to 5.0 fix an
issue.
This change is Pike-only, Queens is implemented in oslo-templates
formula.
Related-Issue: PROD-34332
Change-Id: I590b9c06999768b2c55646fff26649a762e2ba81
diff --git a/barbican/files/pike/barbican.conf.Debian b/barbican/files/pike/barbican.conf.Debian
index 330b83b..93d304b 100644
--- a/barbican/files/pike/barbican.conf.Debian
+++ b/barbican/files/pike/barbican.conf.Debian
@@ -191,6 +191,24 @@
# DO NOT USE THIS, due to '# FIXME(markmc): support multiple hosts' in oslo/messaging/_drivers/amqpdriver.py
# transport_url = rabbit://guest@localhost:5672/
+# NOTE(pas-ha) default values of below options are problematic with RMQ 3.8,
+# see PROD-34322
+# recreating queues on a secondary broker immediately after primary broker
+# has gone down leads to these queues being non-functional.
+
+# How long to wait before reconnecting in response to an AMQP consumer
+# cancel notification. (floating point value)
+#kombu_reconnect_delay = 1.0
+kombu_reconnect_delay = 5.0
+
+# How frequently to retry connecting with RabbitMQ. (integer value)
+#rabbit_retry_interval = 1
+rabbit_retry_interval = 5
+
+# How long to backoff for between retries when connecting to RabbitMQ.
+# (integer value)
+#rabbit_retry_backoff = 2
+rabbit_retry_backoff = 10
[oslo_messaging_notifications]
# oslo notification driver for sending audit events via audit middleware.
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 29fb975..4665d34 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -1,5 +1,15 @@
#!/usr/bin/env bash
+###
+# Script source: https://gerrit.mcp.mirantis.com/#/admin/projects/salt-formulas/cookiecutter-salt-formula
+# Script requirments:
+#apt-get install -y python-yaml virtualenv git
+
+__ScriptVersion="2019.01.07"
+__ScriptName="run_tests.sh"
+__ScriptFullName="$0"
+__ScriptArgs="$*"
+
set -e
[ -n "$DEBUG" ] && set -x
@@ -14,32 +24,40 @@
VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
DEPSDIR=${BUILDDIR}/deps
+SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"}
SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
+SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods}
SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
+IGNORE_MODELVALIDATE_MASK=${IGNORE_MODELVALIDATE_MASK:-"novalidate"}
+
if [ "x${SALT_VERSION}" != "x" ]; then
PIP_SALT_VERSION="==${SALT_VERSION}"
fi
## Functions
log_info() {
- echo "[INFO] $*"
+ echo -e "[INFO] $*"
}
log_err() {
- echo "[ERROR] $*" >&2
+ echo -e "[ERROR] $*" >&2
}
setup_virtualenv() {
log_info "Setting up Python virtualenv"
+ dependency_check virtualenv
virtualenv $VENV_DIR
source ${VENV_DIR}/bin/activate
- pip install salt${PIP_SALT_VERSION}
+ python -m pip install salt${PIP_SALT_VERSION}
+ if [[ -f ${CURDIR}/test-requirements.txt ]]; then
+ python -m pip install -r ${CURDIR}/test-requirements.txt
+ fi
}
setup_mock_bin() {
@@ -59,12 +77,19 @@
state_name=$(basename ${pillar%.sls})
echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
done
+ for pillar in $(find $PILLARDIR -mindepth 2 -type f -iname *.sls); do
+ state_name=$(basename "${pillar%*.sls}")
+ os_release=$(echo $pillar | rev | cut -d'/' -f2 | rev)
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+ echo -e " ${os_release}_${state_name}:\n - ${os_release}.${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
+ done
}
setup_salt() {
[ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
[ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
[ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
+ [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR}
echo "base:" > ${SALT_FILE_DIR}/top.sls
for pillar in ${PILLARDIR}/*.sls; do
@@ -72,10 +97,17 @@
state_name=$(basename ${pillar%.sls})
echo -e " ${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
done
+ for pillar in $(find $PILLARDIR -mindepth 2 -type f -iname *.sls); do
+ state_name=$(basename "${pillar%*.sls}")
+ os_release=$(echo $pillar | rev | cut -d'/' -f2 | rev)
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+ echo -e " ${os_release}_${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
+ done
cat << EOF > ${SALT_CONFIG_DIR}/minion
file_client: local
cachedir: ${SALT_CACHE_DIR}
+extension_modules: ${SALT_CACHE_EXTMODS_DIR}
verify_env: False
minion_id_caching: False
@@ -83,7 +115,6 @@
base:
- ${SALT_FILE_DIR}
- ${CURDIR}/..
- - /usr/share/salt-formulas/env
pillar_roots:
base:
@@ -93,13 +124,14 @@
}
fetch_dependency() {
+ # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux"
dep_name="$(echo $1|cut -d : -f 1)"
dep_source="$(echo $1|cut -d : -f 2-)"
dep_root="${DEPSDIR}/$(basename $dep_source .git)"
dep_metadata="${dep_root}/metadata.yml"
- [ -d /usr/share/salt-formulas/env/${dep_name} ] && log_info "Dependency $dep_name already present in system-wide salt env" && return 0
- [ -d $dep_root ] && log_info "Dependency $dep_name already fetched" && return 0
+ dependency_check git
+ [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; }
log_info "Fetching dependency $dep_name"
[ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
@@ -109,6 +141,19 @@
METADATA="${dep_metadata}" install_dependencies
}
+link_modules(){
+ # Link modules *.py files to temporary salt-root
+ local SALT_ROOT=${1:-$SALT_FILE_DIR}
+ local SALT_ENV=${2:-$DEPSDIR}
+
+ mkdir -p "${SALT_ROOT}/_modules/"
+ # from git, development versions
+ find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do
+ ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ;
+ done
+ salt_run saltutil.sync_all
+}
+
install_dependencies() {
grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
(python - | while read dep; do fetch_dependency "$dep"; done) << EOF
@@ -124,18 +169,35 @@
}
salt_run() {
- [ -e ${VEN_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
- salt-call ${SALT_OPTS} $*
+ [ -e ${VENV_DIR}/bin/activate ] && source ${VENV_DIR}/bin/activate
+ python $(which salt-call) ${SALT_OPTS} $*
}
prepare() {
- [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
+ if [[ -f ${BUILDDIR}/.prepare_done ]]; then
+ log_info "${BUILDDIR}/.prepare_done exist, not rebuilding BUILDDIR"
+ return
+ fi
+ [[ -d ${BUILDDIR} ]] && mkdir -p ${BUILDDIR}
- which salt-call || setup_virtualenv
+ [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
setup_mock_bin
setup_pillar
setup_salt
install_dependencies
+ link_modules
+ touch ${BUILDDIR}/.prepare_done
+}
+
+lint_releasenotes() {
+ [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+ source ${VENV_DIR}/bin/activate
+ reno lint ${CURDIR}/../
+}
+
+lint() {
+# lint_releasenotes
+ log_err "TODO: lint_releasenotes"
}
run() {
@@ -152,7 +214,7 @@
meta_name=$(basename ${meta})
echo "Checking meta ${meta_name} ..."
salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
- || (log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1)
+ || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; }
cat ${SALT_CACHE_DIR}/${meta_name}
done
done
@@ -161,10 +223,52 @@
real_run() {
for pillar in ${PILLARDIR}/*.sls; do
state_name=$(basename ${pillar%.sls})
- salt_run --id=${state_name} state.sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+ salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; }
done
}
+run_model_validate(){
+ # Run modelschema.model_validate validation.
+ # TEST iterateble, run for `each formula ROLE against each ROLE_PILLARNAME`
+ # Pillars should be named in conviend ROLE_XXX.sls or ROLE.sls
+ # Example:
+ # client.sls client_auth.sls server.sls server_auth.sls
+ if [ -d ${SCHEMARDIR} ]; then
+ # model validator require py modules
+ fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt"
+ link_modules
+ salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all;
+ for role in $(find $SCHEMARDIR/* -maxdepth 0 -type f -iname *.yaml); do
+ role_name=$(basename "${role%*.yaml}")
+ for pillar in $(ls pillar/${role_name}*.sls | grep -v ${IGNORE_MODELVALIDATE_MASK} ); do
+ pillar_name=$(basename "${pillar%*.sls}")
+ local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${pillar_name}"
+ log_info "model_validate ${_message}"
+ # Rendered Example:
+ # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster
+ salt_run -m ${DEPSDIR}/salt-formula-salt --id=${pillar_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
+ done
+ done
+ for schema in $(find $SCHEMARDIR -mindepth 2 -type f -iname *.yaml); do
+ role_name=$(basename "${schema%*.yaml}")
+ os_release=$(echo $schema | rev | cut -d'/' -f2 | rev)
+ local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${role_name}"
+ log_info "model_validate ${_message}"
+ salt_run -m ${DEPSDIR}/salt-formula-salt --id=${os_release}_${role_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} ${os_release} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
+ done
+ else
+ log_info "${SCHEMARDIR} not found!";
+ fi
+}
+
+dependency_check() {
+ local DEPENDENCY_COMMANDS=$*
+
+ for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do
+ which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; )
+ done
+}
+
_atexit() {
RETVAL=$?
trap true INT TERM EXIT
@@ -178,6 +282,10 @@
}
## Main
+
+log_info "Running version: ${__ScriptVersion}"
+log_info "Command line: '${__ScriptFullName} ${__ScriptArgs}'"
+
trap _atexit INT TERM EXIT
case $1 in
@@ -187,14 +295,23 @@
prepare)
prepare
;;
+ lint)
+ lint
+ ;;
run)
run
;;
real-run)
real_run
;;
+ model-validate)
+ prepare
+ run_model_validate
+ ;;
*)
prepare
+# lint
run
+ run_model_validate
;;
esac
diff --git a/tests/test-requirements.txt b/tests/test-requirements.txt
new file mode 100644
index 0000000..bc7dc7f
--- /dev/null
+++ b/tests/test-requirements.txt
@@ -0,0 +1,4 @@
+jsonschema
+reno
+setuptools<45.0.0
+msgpack<1.0.0