Merge "Set allow_availability_zone_fallback option"
diff --git a/README.rst b/README.rst
index 6d6eeb0..15c19cd 100644
--- a/README.rst
+++ b/README.rst
@@ -88,6 +88,43 @@
             multihost: true
             multipath: true
             pool: SAS7K2
+            w
+
+Client-side RabbitMQ HA setup for controller
+
+.. code-block:: yaml
+
+    cinder:
+      controller:
+        ....
+        message_queue:
+          engine: rabbitmq
+          members:
+            - host: 10.0.16.1
+            - host: 10.0.16.2
+            - host: 10.0.16.3
+          user: openstack
+          password: pwd
+          virtual_host: '/openstack'
+        ....
+
+Client-side RabbitMQ HA setup for volume component
+
+.. code-block:: yaml
+
+    cinder:
+      volume:
+        ....
+        message_queue:
+          engine: rabbitmq
+          members:
+            - host: 10.0.16.1
+            - host: 10.0.16.2
+            - host: 10.0.16.3
+          user: openstack
+          password: pwd
+          virtual_host: '/openstack'
+        ....
 
 Cinder setup with zeroing deleted volumes
 
diff --git a/cinder/files/kilo/cinder.conf.controller.Debian b/cinder/files/kilo/cinder.conf.controller.Debian
index ce89626..bffbf73 100644
--- a/cinder/files/kilo/cinder.conf.controller.Debian
+++ b/cinder/files/kilo/cinder.conf.controller.Debian
@@ -55,7 +55,6 @@
 rpc_backend=cinder.openstack.common.rpc.impl_kombu
 control_exchange=cinder
 
-
 volume_clear={{ controller.wipe_method }}
 
 {%- if controller.notification %}
@@ -63,7 +62,6 @@
 notification_driver = messagingv2
 {%- endif %}
 
-
 volume_name_template = volume-%s
 
 #volume_group = vg_cinder_volume
@@ -83,12 +81,19 @@
 lock_path=/var/lock/cinder
 
 [oslo_messaging_rabbit]
+{%- if controller.message_queue.members is defined %}
+rabbit_hosts = {% for member in controller.message_queue.members -%}
+                   {{ member.host }}:{{ member.get('port', 5672) }}
+                   {%- if not loop.last -%},{%- endif -%}
+               {%- endfor -%}
+{%- else %}
 rabbit_host = {{ controller.message_queue.host }}
 rabbit_port = {{ controller.message_queue.port }}
+{%- endif %}
+
 rabbit_userid = {{ controller.message_queue.user }}
 rabbit_password = {{ controller.message_queue.password }}
 rabbit_virtual_host = {{ controller.message_queue.virtual_host }}
-rabbit_ha_queues={{ controller.message_queue.ha_queues }}
 kombu_reconnect_delay=5.0
 
 {%- if controller.identity.get('version', 2) == 2 %}
@@ -315,7 +320,6 @@
 
 {%- endif %}
 
-
 {%- if controller.storage.engine == "hp3par" %}
 
 {%- for type in controller.get('types', []) %}
diff --git a/cinder/files/kilo/cinder.conf.volume.Debian b/cinder/files/kilo/cinder.conf.volume.Debian
index 5ec2e6b..1f48741 100644
--- a/cinder/files/kilo/cinder.conf.volume.Debian
+++ b/cinder/files/kilo/cinder.conf.volume.Debian
@@ -90,13 +90,19 @@
 lock_path=/var/lock/cinder
 
 [oslo_messaging_rabbit]
-
+{%- if volume.message_queue.members is defined %}
+rabbit_hosts = {% for member in volume.message_queue.members -%}
+                   {{ member.host }}:{{ member.get('port', 5672) }}
+                   {%- if not loop.last -%},{%- endif -%}
+               {%- endfor -%}
+{%- else %}
 rabbit_host = {{ volume.message_queue.host }}
 rabbit_port = {{ volume.message_queue.port }}
+{%- endif %}
+
 rabbit_userid = {{ volume.message_queue.user }}
 rabbit_password = {{ volume.message_queue.password }}
 rabbit_virtual_host = {{ volume.message_queue.virtual_host }}
-rabbit_ha_queues={{ volume.message_queue.ha_queues }}
 kombu_reconnect_delay=5.0
 
 {%- if volume.identity.get('version', 2) == 2 %}
diff --git a/cinder/files/liberty/cinder.conf.controller.Debian b/cinder/files/liberty/cinder.conf.controller.Debian
index 14c9213..6143552 100644
--- a/cinder/files/liberty/cinder.conf.controller.Debian
+++ b/cinder/files/liberty/cinder.conf.controller.Debian
@@ -83,12 +83,19 @@
 lock_path=/var/lock/cinder
 
 [oslo_messaging_rabbit]
+{%- if controller.message_queue.members is defined %}
+rabbit_hosts = {% for member in controller.message_queue.members -%}
+                   {{ member.host }}:{{ member.get('port', 5672) }}
+                   {%- if not loop.last -%},{%- endif -%}
+               {%- endfor -%}
+{%- else %}
 rabbit_host = {{ controller.message_queue.host }}
 rabbit_port = {{ controller.message_queue.port }}
+{%- endif %}
+
 rabbit_userid = {{ controller.message_queue.user }}
 rabbit_password = {{ controller.message_queue.password }}
 rabbit_virtual_host = {{ controller.message_queue.virtual_host }}
-rabbit_ha_queues={{ controller.message_queue.ha_queues }}
 kombu_reconnect_delay=5.0
 
 {%- if controller.identity.get('version', 2) == 2 %}
diff --git a/cinder/files/liberty/cinder.conf.volume.Debian b/cinder/files/liberty/cinder.conf.volume.Debian
index 5ec2e6b..1f48741 100644
--- a/cinder/files/liberty/cinder.conf.volume.Debian
+++ b/cinder/files/liberty/cinder.conf.volume.Debian
@@ -90,13 +90,19 @@
 lock_path=/var/lock/cinder
 
 [oslo_messaging_rabbit]
-
+{%- if volume.message_queue.members is defined %}
+rabbit_hosts = {% for member in volume.message_queue.members -%}
+                   {{ member.host }}:{{ member.get('port', 5672) }}
+                   {%- if not loop.last -%},{%- endif -%}
+               {%- endfor -%}
+{%- else %}
 rabbit_host = {{ volume.message_queue.host }}
 rabbit_port = {{ volume.message_queue.port }}
+{%- endif %}
+
 rabbit_userid = {{ volume.message_queue.user }}
 rabbit_password = {{ volume.message_queue.password }}
 rabbit_virtual_host = {{ volume.message_queue.virtual_host }}
-rabbit_ha_queues={{ volume.message_queue.ha_queues }}
 kombu_reconnect_delay=5.0
 
 {%- if volume.identity.get('version', 2) == 2 %}
diff --git a/cinder/files/mitaka/cinder.conf.controller.Debian b/cinder/files/mitaka/cinder.conf.controller.Debian
index 7f508fe..04669f9 100644
--- a/cinder/files/mitaka/cinder.conf.controller.Debian
+++ b/cinder/files/mitaka/cinder.conf.controller.Debian
@@ -76,12 +76,19 @@
 lock_path=/var/lock/cinder
 
 [oslo_messaging_rabbit]
+{%- if controller.message_queue.members is defined %}
+rabbit_hosts = {% for member in controller.message_queue.members -%}
+                   {{ member.host }}:{{ member.get('port', 5672) }}
+                   {%- if not loop.last -%},{%- endif -%}
+               {%- endfor -%}
+{%- else %}
 rabbit_host = {{ controller.message_queue.host }}
 rabbit_port = {{ controller.message_queue.port }}
+{%- endif %}
+
 rabbit_userid = {{ controller.message_queue.user }}
 rabbit_password = {{ controller.message_queue.password }}
 rabbit_virtual_host = {{ controller.message_queue.virtual_host }}
-rabbit_ha_queues={{ controller.message_queue.ha_queues }}
 kombu_reconnect_delay=5.0
 
 [keystone_authtoken]
diff --git a/cinder/files/mitaka/cinder.conf.volume.Debian b/cinder/files/mitaka/cinder.conf.volume.Debian
index 91faf84..184a695 100644
--- a/cinder/files/mitaka/cinder.conf.volume.Debian
+++ b/cinder/files/mitaka/cinder.conf.volume.Debian
@@ -74,13 +74,19 @@
 lock_path=/var/lock/cinder
 
 [oslo_messaging_rabbit]
-
+{%- if volume.message_queue.members is defined %}
+rabbit_hosts = {% for member in volume.message_queue.members -%}
+                   {{ member.host }}:{{ member.get('port', 5672) }}
+                   {%- if not loop.last -%},{%- endif -%}
+               {%- endfor -%}
+{%- else %}
 rabbit_host = {{ volume.message_queue.host }}
 rabbit_port = {{ volume.message_queue.port }}
+{%- endif %}
+
 rabbit_userid = {{ volume.message_queue.user }}
 rabbit_password = {{ volume.message_queue.password }}
 rabbit_virtual_host = {{ volume.message_queue.virtual_host }}
-rabbit_ha_queues={{ volume.message_queue.ha_queues }}
 kombu_reconnect_delay=5.0
 
 [keystone_authtoken]
@@ -114,4 +120,4 @@
 
 {%- endfor %}
 
-{%- endif %}
\ No newline at end of file
+{%- endif %}
diff --git a/metadata/service/control/cluster.yml b/metadata/service/control/cluster.yml
index 91a814a..c0c9682 100644
--- a/metadata/service/control/cluster.yml
+++ b/metadata/service/control/cluster.yml
@@ -34,7 +34,6 @@
         user: openstack
         password: ${_param:rabbitmq_openstack_password}
         virtual_host: '/openstack'
-        ha_queues: true
       cache:
         engine: memcached
         members:
@@ -43,4 +42,4 @@
         - host: ${_param:cluster_node02_address}
           port: 11211
         - host: ${_param:cluster_node03_address}
-          port: 11211
\ No newline at end of file
+          port: 11211
diff --git a/metadata/service/control/cluster_control.yml b/metadata/service/control/cluster_control.yml
index 91a814a..c0c9682 100644
--- a/metadata/service/control/cluster_control.yml
+++ b/metadata/service/control/cluster_control.yml
@@ -34,7 +34,6 @@
         user: openstack
         password: ${_param:rabbitmq_openstack_password}
         virtual_host: '/openstack'
-        ha_queues: true
       cache:
         engine: memcached
         members:
@@ -43,4 +42,4 @@
         - host: ${_param:cluster_node02_address}
           port: 11211
         - host: ${_param:cluster_node03_address}
-          port: 11211
\ No newline at end of file
+          port: 11211
diff --git a/metadata/service/control/single.yml b/metadata/service/control/single.yml
index f52b8a6..5b5c9aa 100644
--- a/metadata/service/control/single.yml
+++ b/metadata/service/control/single.yml
@@ -34,4 +34,3 @@
         user: openstack
         password: ${_param:rabbitmq_openstack_password}
         virtual_host: '/openstack'
-        ha_queues: false
diff --git a/metadata/service/volume/single.yml b/metadata/service/volume/single.yml
index 1705867..a452086 100644
--- a/metadata/service/volume/single.yml
+++ b/metadata/service/volume/single.yml
@@ -34,7 +34,6 @@
         user: openstack
         password: ${_param:rabbitmq_openstack_password}
         virtual_host: '/openstack'
-        ha_queues: true
       cache:
         engine: memcached
         members:
@@ -43,4 +42,4 @@
         - host: ${_param:cluster_node02_address}
           port: 11211
         - host: ${_param:cluster_node03_address}
-          port: 11211
\ No newline at end of file
+          port: 11211
diff --git a/tests/pillar/control_cluster.sls b/tests/pillar/control_cluster.sls
index b8c1dfb..b736446 100644
--- a/tests/pillar/control_cluster.sls
+++ b/tests/pillar/control_cluster.sls
@@ -24,12 +24,13 @@
       port: 9292
     message_queue:
       engine: rabbitmq
-      host: 127.0.0.1
-      port: 5672
+      members:
+      - host: 127.0.0.1
+      - host: 127.0.1.1
+      - host: 127.0.2.1
       user: openstack
       password: password
       virtual_host: '/openstack'
-      ha_queues: true
     cache:
       engine: memcached
       members:
@@ -38,4 +39,4 @@
       - host: 127.0.0.1
         port: 11211
       - host: 127.0.0.1
-        port: 11211
\ No newline at end of file
+        port: 11211
diff --git a/tests/pillar/control_single.sls b/tests/pillar/control_single.sls
index d9d6a73..4c3b3bf 100644
--- a/tests/pillar/control_single.sls
+++ b/tests/pillar/control_single.sls
@@ -29,4 +29,3 @@
       user: openstack
       password: password
       virtual_host: '/openstack'
-      ha_queues: false
diff --git a/tests/pillar/volume_single.sls b/tests/pillar/volume_single.sls
index 2c69443..1b66ae2 100644
--- a/tests/pillar/volume_single.sls
+++ b/tests/pillar/volume_single.sls
@@ -28,4 +28,3 @@
       user: openstack
       password: password
       virtual_host: '/openstack'
-      ha_queues: true
\ No newline at end of file
diff --git a/tests/run_tests.sh b/tests/run_tests.sh
index 3f42101..6710a50 100755
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@@ -64,6 +64,7 @@
 file_client: local
 cachedir: ${SALT_CACHE_DIR}
 verify_env: False
+minion_id_caching: False
 
 file_roots:
   base: