Change some default values for Rabbit driver
It was observed that for the rabbitmq-server version 3.8.2 default
values during failover of one of the rabbit nodes cause rapid connection
recreation. In some cases it leads to the creation of broken exchanges
and hangs of OpenStack operations. Changing rabbit_retry_interval to 5,
rabbit_retry_backoff to 10 and kombu_reconnect_delay to 5.0 fix an
issue.
This change is Pike-only, Queens is implemented in oslo-templates
formula.
Related-Issue: PROD-34332
Change-Id: I3ba619d9e7cde94405992acb4be4913674b6fb9e
diff --git a/manila/files/pike/_oslo_messaging_rabbit.conf b/manila/files/pike/_oslo_messaging_rabbit.conf
index 8bd8a56..2fd5329 100644
--- a/manila/files/pike/_oslo_messaging_rabbit.conf
+++ b/manila/files/pike/_oslo_messaging_rabbit.conf
@@ -13,3 +13,24 @@
kombu_ssl_ca_certs = {{ _message_queue.ssl.cacert_file }}
{%- endif %}
{%- endif %}
+
+# NOTE(pas-ha) default values of below options are problematic with RMQ 3.8,
+# see PROD-34322
+# recreating queues on a secondary broker immediately after primary broker
+# has gone down leads to these queues being non-functional.
+
+# How long to wait before reconnecting in response to an AMQP consumer cancel
+# notification. (floating point value)
+# Deprecated group/name - [DEFAULT]/kombu_reconnect_delay
+#kombu_reconnect_delay = 1.0
+kombu_reconnect_delay = 5.0
+
+# How frequently to retry connecting with RabbitMQ. (integer value)
+#rabbit_retry_interval = 1
+rabbit_retry_interval = 5
+
+# How long to backoff for between retries when connecting to RabbitMQ. (integer
+# value)
+# Deprecated group/name - [DEFAULT]/rabbit_retry_backoff
+#rabbit_retry_backoff = 2
+rabbit_retry_backoff = 10
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 9761585..4665d34 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -1,5 +1,15 @@
#!/usr/bin/env bash
+###
+# Script source: https://gerrit.mcp.mirantis.com/#/admin/projects/salt-formulas/cookiecutter-salt-formula
+# Script requirments:
+#apt-get install -y python-yaml virtualenv git
+
+__ScriptVersion="2019.01.07"
+__ScriptName="run_tests.sh"
+__ScriptFullName="$0"
+__ScriptArgs="$*"
+
set -e
[ -n "$DEBUG" ] && set -x
@@ -14,32 +24,40 @@
VENV_DIR=${VENV_DIR:-${BUILDDIR}/virtualenv}
MOCK_BIN_DIR=${MOCK_BIN_DIR:-${CURDIR}/mock_bin}
DEPSDIR=${BUILDDIR}/deps
+SCHEMARDIR=${SCHEMARDIR:-"${CURDIR}/../${FORMULA_NAME}/schemas/"}
SALT_FILE_DIR=${SALT_FILE_DIR:-${BUILDDIR}/file_root}
SALT_PILLAR_DIR=${SALT_PILLAR_DIR:-${BUILDDIR}/pillar_root}
SALT_CONFIG_DIR=${SALT_CONFIG_DIR:-${BUILDDIR}/salt}
SALT_CACHE_DIR=${SALT_CACHE_DIR:-${SALT_CONFIG_DIR}/cache}
+SALT_CACHE_EXTMODS_DIR=${SALT_CACHE_EXTMODS_DIR:-${SALT_CONFIG_DIR}/cache_master_extmods}
SALT_OPTS="${SALT_OPTS} --retcode-passthrough --local -c ${SALT_CONFIG_DIR} --log-file=/dev/null"
+IGNORE_MODELVALIDATE_MASK=${IGNORE_MODELVALIDATE_MASK:-"novalidate"}
+
if [ "x${SALT_VERSION}" != "x" ]; then
PIP_SALT_VERSION="==${SALT_VERSION}"
fi
## Functions
log_info() {
- echo "[INFO] $*"
+ echo -e "[INFO] $*"
}
log_err() {
- echo "[ERROR] $*" >&2
+ echo -e "[ERROR] $*" >&2
}
setup_virtualenv() {
log_info "Setting up Python virtualenv"
+ dependency_check virtualenv
virtualenv $VENV_DIR
source ${VENV_DIR}/bin/activate
python -m pip install salt${PIP_SALT_VERSION}
+ if [[ -f ${CURDIR}/test-requirements.txt ]]; then
+ python -m pip install -r ${CURDIR}/test-requirements.txt
+ fi
}
setup_mock_bin() {
@@ -59,12 +77,19 @@
state_name=$(basename ${pillar%.sls})
echo -e " ${state_name}:\n - ${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
done
+ for pillar in $(find $PILLARDIR -mindepth 2 -type f -iname *.sls); do
+ state_name=$(basename "${pillar%*.sls}")
+ os_release=$(echo $pillar | rev | cut -d'/' -f2 | rev)
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+ echo -e " ${os_release}_${state_name}:\n - ${os_release}.${state_name}" >> ${SALT_PILLAR_DIR}/top.sls
+ done
}
setup_salt() {
[ ! -d ${SALT_FILE_DIR} ] && mkdir -p ${SALT_FILE_DIR}
[ ! -d ${SALT_CONFIG_DIR} ] && mkdir -p ${SALT_CONFIG_DIR}
[ ! -d ${SALT_CACHE_DIR} ] && mkdir -p ${SALT_CACHE_DIR}
+ [ ! -d ${SALT_CACHE_EXTMODS_DIR} ] && mkdir -p ${SALT_CACHE_EXTMODS_DIR}
echo "base:" > ${SALT_FILE_DIR}/top.sls
for pillar in ${PILLARDIR}/*.sls; do
@@ -72,10 +97,17 @@
state_name=$(basename ${pillar%.sls})
echo -e " ${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
done
+ for pillar in $(find $PILLARDIR -mindepth 2 -type f -iname *.sls); do
+ state_name=$(basename "${pillar%*.sls}")
+ os_release=$(echo $pillar | rev | cut -d'/' -f2 | rev)
+ grep ${FORMULA_NAME}: ${pillar} &>/dev/null || continue
+ echo -e " ${os_release}_${state_name}:\n - ${FORMULA_NAME}" >> ${SALT_FILE_DIR}/top.sls
+ done
cat << EOF > ${SALT_CONFIG_DIR}/minion
file_client: local
cachedir: ${SALT_CACHE_DIR}
+extension_modules: ${SALT_CACHE_EXTMODS_DIR}
verify_env: False
minion_id_caching: False
@@ -83,7 +115,6 @@
base:
- ${SALT_FILE_DIR}
- ${CURDIR}/..
- - /usr/share/salt-formulas/env
pillar_roots:
base:
@@ -93,13 +124,14 @@
}
fetch_dependency() {
+ # example: fetch_dependency "linux:https://github.com/salt-formulas/salt-formula-linux"
dep_name="$(echo $1|cut -d : -f 1)"
dep_source="$(echo $1|cut -d : -f 2-)"
dep_root="${DEPSDIR}/$(basename $dep_source .git)"
dep_metadata="${dep_root}/metadata.yml"
- [ -d /usr/share/salt-formulas/env/${dep_name} ] && log_info "Dependency $dep_name already present in system-wide salt env" && return 0
- [ -d $dep_root ] && log_info "Dependency $dep_name already fetched" && return 0
+ dependency_check git
+ [ -d $dep_root ] && { log_info "Dependency $dep_name already fetched"; return 0; }
log_info "Fetching dependency $dep_name"
[ ! -d ${DEPSDIR} ] && mkdir -p ${DEPSDIR}
@@ -109,6 +141,19 @@
METADATA="${dep_metadata}" install_dependencies
}
+link_modules(){
+ # Link modules *.py files to temporary salt-root
+ local SALT_ROOT=${1:-$SALT_FILE_DIR}
+ local SALT_ENV=${2:-$DEPSDIR}
+
+ mkdir -p "${SALT_ROOT}/_modules/"
+ # from git, development versions
+ find ${SALT_ENV} -maxdepth 3 -mindepth 3 -path '*_modules*' -iname "*.py" -type f -print0 | while read -d $'\0' file; do
+ ln -fs $(readlink -e ${file}) "$SALT_ROOT"/_modules/$(basename ${file}) ;
+ done
+ salt_run saltutil.sync_all
+}
+
install_dependencies() {
grep -E "^dependencies:" ${METADATA} >/dev/null || return 0
(python - | while read dep; do fetch_dependency "$dep"; done) << EOF
@@ -129,13 +174,30 @@
}
prepare() {
- [ -d ${BUILDDIR} ] && mkdir -p ${BUILDDIR}
+ if [[ -f ${BUILDDIR}/.prepare_done ]]; then
+ log_info "${BUILDDIR}/.prepare_done exist, not rebuilding BUILDDIR"
+ return
+ fi
+ [[ -d ${BUILDDIR} ]] && mkdir -p ${BUILDDIR}
- which salt-call || setup_virtualenv
+ [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
setup_mock_bin
setup_pillar
setup_salt
install_dependencies
+ link_modules
+ touch ${BUILDDIR}/.prepare_done
+}
+
+lint_releasenotes() {
+ [[ ! -f "${VENV_DIR}/bin/activate" ]] && setup_virtualenv
+ source ${VENV_DIR}/bin/activate
+ reno lint ${CURDIR}/../
+}
+
+lint() {
+# lint_releasenotes
+ log_err "TODO: lint_releasenotes"
}
run() {
@@ -152,7 +214,7 @@
meta_name=$(basename ${meta})
echo "Checking meta ${meta_name} ..."
salt_run --out=quiet --id=${state_name} cp.get_template ${meta} ${SALT_CACHE_DIR}/${meta_name} \
- || (log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1)
+ || { log_err "Failed to render meta ${meta} using pillar ${FORMULA_NAME}.${state_name}"; exit 1; }
cat ${SALT_CACHE_DIR}/${meta_name}
done
done
@@ -161,10 +223,52 @@
real_run() {
for pillar in ${PILLARDIR}/*.sls; do
state_name=$(basename ${pillar%.sls})
- salt_run --id=${state_name} state.sls ${FORMULA_NAME} || (log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1)
+ salt_run --id=${state_name} state.sls ${FORMULA_NAME} || { log_err "Execution of ${FORMULA_NAME}.${state_name} failed"; exit 1; }
done
}
+run_model_validate(){
+ # Run modelschema.model_validate validation.
+ # TEST iterateble, run for `each formula ROLE against each ROLE_PILLARNAME`
+ # Pillars should be named in conviend ROLE_XXX.sls or ROLE.sls
+ # Example:
+ # client.sls client_auth.sls server.sls server_auth.sls
+ if [ -d ${SCHEMARDIR} ]; then
+ # model validator require py modules
+ fetch_dependency "salt:https://github.com/salt-formulas/salt-formula-salt"
+ link_modules
+ salt_run saltutil.clear_cache; salt_run saltutil.refresh_pillar; salt_run saltutil.sync_all;
+ for role in $(find $SCHEMARDIR/* -maxdepth 0 -type f -iname *.yaml); do
+ role_name=$(basename "${role%*.yaml}")
+ for pillar in $(ls pillar/${role_name}*.sls | grep -v ${IGNORE_MODELVALIDATE_MASK} ); do
+ pillar_name=$(basename "${pillar%*.sls}")
+ local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${pillar_name}"
+ log_info "model_validate ${_message}"
+ # Rendered Example:
+ # python $(which salt-call) --local -c /test1/maas/tests/build/salt --id=maas_cluster modelschema.model_validate maas cluster
+ salt_run -m ${DEPSDIR}/salt-formula-salt --id=${pillar_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
+ done
+ done
+ for schema in $(find $SCHEMARDIR -mindepth 2 -type f -iname *.yaml); do
+ role_name=$(basename "${schema%*.yaml}")
+ os_release=$(echo $schema | rev | cut -d'/' -f2 | rev)
+ local _message="FORMULA:${FORMULA_NAME} ROLE:${role_name} against PILLAR:${role_name}"
+ log_info "model_validate ${_message}"
+ salt_run -m ${DEPSDIR}/salt-formula-salt --id=${os_release}_${role_name} modelschema.model_validate ${FORMULA_NAME} ${role_name} ${os_release} || { log_err "Execution of model_validate ${_message} failed"; exit 1 ; }
+ done
+ else
+ log_info "${SCHEMARDIR} not found!";
+ fi
+}
+
+dependency_check() {
+ local DEPENDENCY_COMMANDS=$*
+
+ for DEPENDENCY_COMMAND in $DEPENDENCY_COMMANDS; do
+ which $DEPENDENCY_COMMAND > /dev/null || ( log_err "Command \"$DEPENDENCY_COMMAND\" can not be found in default path."; exit 1; )
+ done
+}
+
_atexit() {
RETVAL=$?
trap true INT TERM EXIT
@@ -178,6 +282,10 @@
}
## Main
+
+log_info "Running version: ${__ScriptVersion}"
+log_info "Command line: '${__ScriptFullName} ${__ScriptArgs}'"
+
trap _atexit INT TERM EXIT
case $1 in
@@ -187,14 +295,23 @@
prepare)
prepare
;;
+ lint)
+ lint
+ ;;
run)
run
;;
real-run)
real_run
;;
+ model-validate)
+ prepare
+ run_model_validate
+ ;;
*)
prepare
+# lint
run
+ run_model_validate
;;
esac
diff --git a/tests/test-requirements.txt b/tests/test-requirements.txt
new file mode 100644
index 0000000..bc7dc7f
--- /dev/null
+++ b/tests/test-requirements.txt
@@ -0,0 +1,4 @@
+jsonschema
+reno
+setuptools<45.0.0
+msgpack<1.0.0