[RabbitMQ] add possibility to randomise reconnection intervals
The patch adds a simple pillar for OpenStack services:
message_queue:
rabbit_timeouts_random: True
This parameter makes OS services to reconnect to RabbitMQ in a more
wide time span, reducing the peak of reconnections at the same moment,
that in turn increases the probability of successful cluster self-healing.
Also, while here, unhardcoded the kombu_reconnect_delay parameter.
Closes-Bug: PROD-36441
Closes-Bug: PROD-36458
Change-Id: Ic8ca646d359b683865af3f271731fbcd4ec144c4
diff --git a/oslo_templates/files/queens/oslo/messaging/_rabbit.conf b/oslo_templates/files/queens/oslo/messaging/_rabbit.conf
index f04ff9d..80df145 100644
--- a/oslo_templates/files/queens/oslo/messaging/_rabbit.conf
+++ b/oslo_templates/files/queens/oslo/messaging/_rabbit.conf
@@ -56,8 +56,13 @@
# NOTE(pas-ha) default value is problematic with RMQ 3.8, see PROD-34322
# recreating queues on a secondary broker immediately after primary broker
# has gone down leads to these queues being non-functional.
+{%- if _data.kombu_reconnect_delay is defined %}
+kombu_reconnect_delay = {{ _data.kombu_reconnect_delay }}
+{%- elif _data.rabbit_timeouts_random is sameas true %}
+kombu_reconnect_delay = {{ range(30, 60) | random }}
+{%- else %}
kombu_reconnect_delay = 5.0
-
+{%- endif %}
# EXPERIMENTAL: Possible values are: gzip, bz2. If not set compression
# will not be used. This option may not be available in future
# versions. (string value)
@@ -131,6 +136,8 @@
# has gone down leads to these queues being non-functional.
{%- if _data.rabbit_retry_interval is defined %}
rabbit_retry_interval = {{ _data.rabbit_retry_interval }}
+{%- elif _data.rabbit_timeouts_random is sameas true %}
+rabbit_retry_interval = {{ range(10, 60) | random }}
{%- else %}
rabbit_retry_interval = 5
{%- endif %}
@@ -143,6 +150,8 @@
# has gone down leads to these queues being non-functional.
{%- if _data.rabbit_retry_backoff is defined %}
rabbit_retry_backoff = {{ _data.rabbit_retry_backoff }}
+{%- elif _data.rabbit_timeouts_random is sameas true %}
+rabbit_retry_backoff = {{ range(30, 60) | random }}
{%- else %}
rabbit_retry_backoff = 10
{%- endif %}
@@ -152,6 +161,10 @@
#rabbit_interval_max = 30
{%- if _data.rabbit_interval_max is defined %}
rabbit_interval_max = {{ _data.rabbit_interval_max }}
+{%- elif _data.rabbit_timeouts_random is sameas true %}
+rabbit_interval_max = {{ range(60, 180) | random }}
+{%- else %}
+rabbit_interval_max = 60
{%- endif %}
# DEPRECATED: Maximum number of RabbitMQ connection retries. Default