Octavia manager services clusterization

Change-Id: I928f57646d1df3b351b11f682288919ea98a9434
Related-PROD: PROD-25309
diff --git a/.kitchen.yml b/.kitchen.yml
index 897fbf3..97f3035 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -73,4 +73,14 @@
             manager:
               version: <%= ENV['OS_VERSION'] || 'ocata' %>
 
+  - name: manager_cluster
+    provisioner:
+      pillars-from-files:
+        octavia.sls: tests/pillar/manager_cluster.sls
+      pillars:
+        release.sls:
+          octavia:
+            manager:
+              version: <%= ENV['OS_VERSION'] || 'queens' %>
+
 # vim: ft=yaml sw=2 ts=2 sts=2 tw=125
diff --git a/.travis.yml b/.travis.yml
index 07bd1d3..a2d28da 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -27,6 +27,7 @@
   - PLATFORM=docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-stable/salt:2018_11_19 OS_VERSION=queens SUITE=api-single
   - PLATFORM=docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-stable/salt:2018_11_19 OS_VERSION=queens SUITE=api-cluster
   - PLATFORM=docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-stable/salt:2018_11_19 OS_VERSION=queens SUITE=manager-single
+  - PLATFORM=docker-dev-local.docker.mirantis.net/epcim/salt/saltstack-ubuntu-xenial-salt-stable/salt:2018_11_19 OS_VERSION=queens SUITE=manager-cluster
 
 before_script:
   - set -o pipefail
diff --git a/metadata/service/manager/cluster.yml b/metadata/service/manager/cluster.yml
new file mode 100644
index 0000000..b4473c9
--- /dev/null
+++ b/metadata/service/manager/cluster.yml
@@ -0,0 +1,57 @@
+applications:
+- octavia
+classes:
+- service.octavia.support
+parameters:
+  _param:
+    keystone_octavia_endpoint_type: internal
+    octavia_loadbalancer_topology: 'SINGLE'
+    octavia_hm_bind_port: 5555
+  octavia:
+    manager:
+      enabled: true
+      version: ${_param:octavia_version}
+      debug: false
+      database:
+        engine: mysql
+        host: ${_param:cluster_vip_address}
+        port: 3306
+        name: octavia
+        user: octavia
+        password: ${_param:mysql_octavia_password}
+      identity:
+        engine: keystone
+        region: RegionOne
+        host: ${_param:cluster_vip_address}
+        port: 35357
+        user: octavia
+        password: ${_param:keystone_octavia_password}
+        tenant: service
+        endpoint_type: ${_param:keystone_octavia_endpoint_type}
+      message_queue:
+        engine: rabbitmq
+        host: ${_param:cluster_vip_address}
+        port: 5672
+        user: openstack
+        password: ${_param:rabbitmq_openstack_password}
+        virtual_host: '/openstack'
+      certificates:
+        ca_private_key: '/etc/octavia/certs/ca.key'
+        ca_certificate: '/etc/octavia/certs/ca_01.pem'
+      controller_worker:
+        amp_flavor_id: ${_param:amp_flavor_id}
+        amp_image_tag: amphora
+        amp_ssh_key_name: octavia_ssh_key
+        loadbalancer_topology: ${_param:octavia_loadbalancer_topology}
+      haproxy_amphora:
+        client_cert: '/etc/octavia/certs/client.pem'
+        client_cert_key: '/etc/octavia/certs/client.key'
+        client_cert_all: '/etc/octavia/certs/client_all.pem'
+        server_ca: '/etc/octavia/certs/ca_01.pem'
+      health_manager:
+        bind_ip: ${_param:octavia_hm_bind_ip}
+        bind_port: ${_param:octavia_hm_bind_port}
+        heartbeat_key: 'insecure'
+        controller_ip_port_list: ${_param:octavia_hm_bind_ip_gtw01}:${_param:octavia_hm_bind_port},${_param:octavia_hm_bind_ip_gtw02}:${_param:octavia_hm_bind_port},${_param:octavia_hm_bind_ip_gtw03}:${_param:octavia_hm_bind_port}
+      house_keeping:
+        spare_amphora_pool_size: 0
diff --git a/metadata/service/manager/single.yml b/metadata/service/manager/single.yml
index 7016966..13713b0 100644
--- a/metadata/service/manager/single.yml
+++ b/metadata/service/manager/single.yml
@@ -6,11 +6,12 @@
   _param:
     keystone_octavia_endpoint_type: internal
     octavia_loadbalancer_topology: 'SINGLE'
+    octavia_hm_bind_port: 5555
   octavia:
     manager:
       enabled: true
       version: ${_param:octavia_version}
-      debug: true
+      debug: false
       database:
         engine: mysql
         host: ${_param:cluster_vip_address}
@@ -35,8 +36,8 @@
         password: ${_param:rabbitmq_openstack_password}
         virtual_host: '/openstack'
       certificates:
-        ca_private_key: '/etc/pki/ca/octavia_ca/ca.key'
-        ca_certificate: '/etc/pki/ca/octavia_ca/ca.crt'
+        ca_private_key: '/etc/octavia/certs/ca.key'
+        ca_certificate: '/etc/octavia/certs/ca_01.pem'
       controller_worker:
         amp_flavor_id: ${_param:amp_flavor_id}
         amp_image_tag: amphora
@@ -49,6 +50,8 @@
         server_ca: '/etc/octavia/certs/ca_01.pem'
       health_manager:
         bind_ip: ${_param:octavia_hm_bind_ip}
+        bind_port: ${_param:octavia_hm_bind_port}
         heartbeat_key: 'insecure'
+        controller_ip_port_list: ${_param:octavia_hm_bind_ip}:${_param:octavia_hm_bind_port}
       house_keeping:
         spare_amphora_pool_size: 0
diff --git a/octavia/files/queens/octavia_manager.conf b/octavia/files/queens/octavia_manager.conf
index 9eebe5f..45380d5 100644
--- a/octavia/files/queens/octavia_manager.conf
+++ b/octavia/files/queens/octavia_manager.conf
@@ -906,7 +906,7 @@
 # Minimum value: 0
 # Maximum value: 65535
 #bind_port = 5555
-bind_port = 5555
+bind_port = {{ manager.health_manager.get('bind_port', 5555) }}
 
 # Number of threads performing amphora failovers. (integer value)
 #failover_threads = 10
@@ -936,7 +936,11 @@
 # List of controller ip and port pairs for the heartbeat receivers. Example
 # 127.0.0.1:5555, 192.168.0.1:5555 (list value)
 #controller_ip_port_list =
-controller_ip_port_list = {{ manager.health_manager.bind_ip }}:5555
+{%- if manager.health_manager.controller_ip_port_list is defined %}
+controller_ip_port_list = {{ manager.health_manager.controller_ip_port_list }}
+{%- else %}
+controller_ip_port_list = {{ manager.health_manager.bind_ip }}:{{ manager.health_manager.get('bind_port', 5555) }}
+{%- endif %}
 
 # Sleep time between sending heartbeats. (integer value)
 #heartbeat_interval = 10
diff --git a/octavia/manager.sls b/octavia/manager.sls
index f0ed6b3..618a85d 100644
--- a/octavia/manager.sls
+++ b/octavia/manager.sls
@@ -69,9 +69,13 @@
 
 {%- if image_mine_data and network_mine_data and secgroup_mine_data and port_mine_data %}
 
-{%- set amp_hm_port_mac = port_mine_data[0]['octavia-health-manager-listen-port']['mac_address'] %}
-{%- set amp_hm_port_id = port_mine_data[0]['octavia-health-manager-listen-port']['id'] %}
-{%- set amp_hm_port_ip = port_mine_data[0]['octavia-health-manager-listen-port']['fixed_ips'][0]['ip_address'] %}
+{%- for port in port_mine_data %}
+{%- for port_name, port_params in port.iteritems() %}
+{%- if grains.get('host') in port_name or port_name == 'octavia-health-manager-listen-port'%}
+{%- set amp_hm_port_mac = port_params['mac_address'] %}
+{%- set amp_hm_port_id = port_params['id'] %}
+{%- set amp_hm_port_ip = port_params['fixed_ips'][0]['ip_address'] %}
+
 
 health_manager_ovs_port:
   cmd.run:
@@ -128,6 +132,9 @@
   - watch:
     - file: /etc/octavia/octavia_manager.conf
 {%- endif %}
+{%- endfor %}
+{%- endfor %}
+{%- endif %}
 {%- endif %}
 
 {%- endif %}
diff --git a/tests/pillar/manager_cluster.sls b/tests/pillar/manager_cluster.sls
new file mode 100644
index 0000000..22db297
--- /dev/null
+++ b/tests/pillar/manager_cluster.sls
@@ -0,0 +1,76 @@
+octavia:
+  manager:
+    enabled: true
+    version: ocata
+    database:
+      engine: mysql
+      host: 127.0.0.1
+      port: 3306
+      name: octavia
+      user: octavia
+      password: password
+    identity:
+      engine: keystone
+      region: RegionOne
+      host: 127.0.0.1
+      port: 35357
+      user: octavia
+      password: password
+      tenant: service
+    message_queue:
+      engine: rabbitmq
+      host: 127.0.0.1
+      port: 5672
+      user: openstack
+      password: password
+      virtual_host: '/openstack'
+    certificates:
+      ca_private_key: '/etc/octavia/certs/ca.key'
+      ca_certificate: '/etc/octavia/certs/ca_01.pem'
+    controller_worker:
+      amp_flavor_id: '967972bb-ab54-4679-9f53-bf81d5e28154'
+      amp_image_tag: amphora
+      amp_ssh_key_name: octavia_ssh_key
+      loadbalancer_topology: 'SINGLE'
+    haproxy_amphora:
+      client_cert: '/etc/octavia/certs/client.pem'
+      client_cert_key: '/etc/octavia/certs/client.key'
+      client_cert_all: '/etc/octavia/certs/client_all.pem'
+      server_ca: '/etc/octavia/certs/ca_01.pem'
+    health_manager:
+      bind_ip: 192.168.0.12
+      heartbeat_key: 'insecure'
+      controller_ip_port_list: 192.168.0.10:5555,192.168.0.11:5555,192.168.0.12:5555
+    house_keeping:
+      spare_amphora_pool_size: 0
+    ssh:
+      private_key: |
+        -----BEGIN RSA PRIVATE KEY-----
+        MIIEpAIBAAKCAQEAtjnPDJsQToHBtoqIo15mdSYpfi8z6DFMi8Gbo0KCN33OUn5u
+        OctbdtjUfeuhvI6px1SCnvyWi09Ft8eWwq+KwLCGKbUxLvqKltuJ7K3LIrGXkt+m
+        qZN4O9XKeVKfZH+mQWkkxRWgX2r8RKNV3GkdNtd74VjhP+R6XSKJQ1Z8b7eHM10v
+        6IjTY/jPczjK+eyCeEj4qbSnV8eKlqLhhquuSQRmUO2DRSjLVdpdf2BB4/BdWFsD
+        YOmX7mb8kpEr9vQ+c1JKMXDwD6ehzyU8kE+1kVm5zOeEy4HdYIMpvUfN49P1anRV
+        2ISQ1ZE+r22IAMKl0tekrGH0e/1NP1DF5rINMwIDAQABAoIBAQCkP/cgpaRNHyg8
+        ISKIHs67SWqdEm73G3ijgB+JSKmW2w7dzJgN//6xYUAnP/zIuM7PnJ0gMQyBBTMS
+        NBTv5spqZLKJZYivj6Tb1Ya8jupKm0jEWlMfBo2ZYVrfgFmrfGOfEebSvmuPlh9M
+        vuzlftmWVSSUOkjODmM9D6QpzgrbpktBuA/WpX+6esMTwJpOcQ5xZWEnHXnVzuTc
+        SncodVweE4gz6F1qorbqIJz8UAUQ5T0OZTdHzIS1IbamACHWaxQfixAO2s4+BoUK
+        ANGGZWkfneCxx7lthvY8DiKn7M5cSRnqFyDToGqaLezdkMNlGC7v3U11FF5blSEW
+        fL1o/HwBAoGBAOavhTr8eqezTchqZvarorFIq7HFWk/l0vguIotu6/wlh1V/KdF+
+        aLLHgPgJ5j+RrCMvTBoKqMeeHfVGrS2udEy8L1mK6b3meG+tMxU05OA55abmhYn7
+        7vF0q8XJmYIHIXmuCgF90R8Piscb0eaMlmHW9unKTKo8EOs5j+D8+AMJAoGBAMo4
+        8WW+D3XiD7fsymsfXalf7VpAt/H834QTbNZJweUWhg11eLutyahyyfjjHV200nNZ
+        cnU09DWKpBbLg7d1pyT69CNLXpNnxuWCt8oiUjhWCUpNqVm2nDJbUdlRFTzYb2fS
+        ZC4r0oQaPD5kMLSipjcwzMWe0PniySxNvKXKInFbAoGBAKxW2qD7uKKKuQSOQUft
+        aAksMmEIAHWKTDdvOA2VG6XvX5DHBLXmy08s7rPfqW06ZjCPCDq4Velzvgvc9koX
+        d/lP6cvqlL9za+x6p5wjPQ4rEt/CfmdcmOE4eY+1EgLrUt314LHGjjG3ScWAiirE
+        QyDrGOIGaYoQf89L3KqIMr0JAoGARYAklw8nSSCUvmXHe+Gf0yKA9M/haG28dCwo
+        780RsqZ3FBEXmYk1EYvCFqQX56jJ25MWX2n/tJcdpifz8Q2ikHcfiTHSI187YI34
+        lKQPFgWb08m1NnwoWrY//yx63BqWz1vjymqNQ5GwutC8XJi5/6Xp+tGGiRuEgJGH
+        EIPUKpkCgYAjBIVMkpNiLCREZ6b+qjrPV96ed3iTUt7TqP7yGlFI/OkORFS38xqC
+        hBP6Fk8iNWuOWQD+ohM/vMMnvIhk5jwlcwn+kF0ra04gi5KBFWSh/ddWMJxUtPC1
+        2htvlEc6zQAR6QfqXHmwhg1hP81JcpqpicQzCMhkzLoR1DC6stXdLg==
+        -----END RSA PRIVATE KEY-----
+      user: octavia
+      group: octavia
diff --git a/tests/pillar/manager_single.sls b/tests/pillar/manager_single.sls
index 716acb8..a6eab24 100644
--- a/tests/pillar/manager_single.sls
+++ b/tests/pillar/manager_single.sls
@@ -40,6 +40,7 @@
     health_manager:
       bind_ip: 192.168.0.12
       heartbeat_key: 'insecure'
+      controller_ip_port_list: 192.168.0.12:5555
     house_keeping:
       spare_amphora_pool_size: 0
     ssh: