Merge "Revert "Switch to internal mirror for docker-ee packages""
diff --git a/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp0-gtw0.yaml b/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp0-gtw0.yaml
index 5376ac4..8da45aa 100644
--- a/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp0-gtw0.yaml
+++ b/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp0-gtw0.yaml
@@ -103,7 +103,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             # The roles will be assigned based on node labels.
             # roles:
@@ -121,7 +121,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.102
@@ -134,7 +134,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.103
@@ -147,7 +147,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.104
@@ -160,7 +160,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.105
@@ -173,7 +173,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.106
diff --git a/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp2-gtw0.yaml b/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp2-gtw0.yaml
index f45829b..3c9ae9d 100644
--- a/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp2-gtw0.yaml
+++ b/de/heat-templates/env/bmt402-mstr1-wrkr3-cmp2-gtw0.yaml
@@ -103,7 +103,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             # The roles will be assigned based on node labels.
             # roles:
@@ -121,7 +121,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.102
@@ -134,7 +134,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.103
@@ -147,7 +147,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.104
@@ -160,7 +160,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.105
@@ -173,7 +173,7 @@
               - name: sdb
                 role: hdd
                 sizeGb: 450
-            ram: 64
+            ramGb: 64
             cores: 12
             ips:
               - 192.168.122.106
diff --git a/de/heat-templates/env/bmt410-mstr3-wrkr3-cmp0-gtw0-osd3.yaml b/de/heat-templates/env/bmt410-mstr3-wrkr3-cmp0-gtw0-osd3.yaml
new file mode 100644
index 0000000..ec785e9
--- /dev/null
+++ b/de/heat-templates/env/bmt410-mstr3-wrkr3-cmp0-gtw0-osd3.yaml
@@ -0,0 +1,536 @@
+resource_registry:
+  "MCP2::NetworkAcc": ../fragments/NetworkAccBM.yaml
+  "MCP2::NetworkAccStorage": ../fragments/NetworkAccVMStorage.yaml
+  "MCP2::NetworkPrvFl": ../fragments/NetworkPrvFl.yaml
+  "MCP2::SrvInstances": ../fragments/SrvInstancesBM.yaml
+  "MCP2::SrvInstancesCeph": ../fragments/SrvInstancesBMCeph.yaml
+  "MCP2::SrvInstancesCephOSD": ../fragments/SrvInstancesBMCephOSD.yaml
+
+parameters:
+  image: bionic-server-cloudimg-amd64-20190612
+  ucp_flavor: si-bm-410-128-16-hdd0-ssd2x1t
+  masters_flavor: si-bm-410-128-16-hdd0-ssd2x1t
+  workers_flavor: si-bm-410-128-32-hdd0-ssd2x1t
+  cmps_flavor: oscore-bm-lab-01
+  gtws_flavor: oscore-bm-lab-01
+  lmas_flavor: oscore-bm-lab-01
+  osds_flavor: si-bm-410-64-16-hdd2x2t-ssd1x500g
+  spares_flavor: si-bm-410-64-16-hdd2x2t-ssd1x500g
+  masters_size: 2  # UCP nodes
+  worker_size: 3  # OpenStack controller nodes
+  cmp_size: 0
+  gtw_size: 0
+  lma_size: 0
+  osd_size: 3
+  spare_size: 1
+  ucp_boot_timeout: 1200
+  cluster_public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCp0evjOaK8c8SKYK4r2+0BN7g+8YSvQ2n8nFgOURCyvkJqOHi1qPGZmuN0CclYVdVuZiXbWw3VxRbSW3EH736VzgY1U0JmoTiSamzLHaWsXvEIW8VCi7boli539QJP0ikJiBaNAgZILyCrVPN+A6mfqtacs1KXdZ0zlMq1BPtFciR1JTCRcVs5vP2Wwz5QtY2jMIh3aiwkePjMTQPcfmh1TkOlxYu5IbQyZ3G1ahA0mNKI9a0dtF282av/F6pwB/N1R1nEZ/9VtcN2I1mf1NW/tTHEEcTzXYo1R/8K9vlqAN8QvvGLZtZduGviNVNoNWvoxaXxDt8CPv2B2NCdQFZp
+  public_net_id: 'physnet1-410'
+  control_network_cidr: '172.16.50.0/25'
+  private_floating_network_cidr: '172.16.51.0/25'
+  private_floating_network_pool_start: '172.16.51.10'
+  private_floating_network_pool_end: '172.16.51.30'
+  private_floating_network_ipam_pool_start: '172.16.51.31'
+  private_floating_network_ipam_pool_end: '172.16.51.62'
+  private_floating_network_gateway: '172.16.51.1'
+  private_floating_interface: 'os-floating'
+  tunnel_interface: 'os-tenant'
+  live_migration_interface: 'os-tenant'
+  #master_metadata: {"labels": {"role": "stacklight", "stacklight": "enabled"}}
+  worker_metadata: {"labels": {"openstack-control-plane":"enabled", "openstack-compute-node":"enabled", "openvswitch":"enabled", "gateway": "enabled"}}
+  osd_metadata: {"labels": {"role": "ceph-osd-node"}}
+  spare_metadata: {"labels": {"role": "spare-node"}}
+
+
+  hardware_metadata: |
+    '0c:c4:7a:33:26:7c':
+      write_files:
+      - path: '/etc/netplan/99-cz7714.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eno1:
+                dhcp4: false
+              ens3f1:
+                dhcp4: false
+              ens3f2:
+                dhcp4: false
+              ens3f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [eno1, ens3f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens3f2, ens3f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:33:26:6e':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eno1:
+                dhcp4: false
+              ens3f1:
+                dhcp4: false
+              ens3f2:
+                dhcp4: false
+              ens3f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [eno1, ens3f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens3f2, ens3f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:33:26:6a':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eno1:
+                dhcp4: false
+              ens3f1:
+                dhcp4: false
+              ens3f2:
+                dhcp4: false
+              ens3f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [eno1, ens3f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens3f2, ens3f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:34:92:94':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eno1:
+                dhcp4: false
+              ens3f1:
+                dhcp4: false
+              ens3f2:
+                dhcp4: false
+              ens3f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [eno1, ens3f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens3f2, ens3f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:aa:e0:9e':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              ens11f0:
+                dhcp4: false
+              ens11f1:
+                dhcp4: false
+              ens11f2:
+                dhcp4: false
+              ens11f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [ens11f0, ens11f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens11f2, ens11f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:aa:e0:9c':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              ens11f0:
+                dhcp4: false
+              ens11f1:
+                dhcp4: false
+              ens11f2:
+                dhcp4: false
+              ens11f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [ens11f0, ens11f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens11f2, ens11f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:aa:78:2c':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              ens11f0:
+                dhcp4: false
+              ens11f1:
+                dhcp4: false
+              ens11f2:
+                dhcp4: false
+              ens11f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [ens11f0, ens11f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens11f2, ens11f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:aa:78:7a':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eno1:
+                dhcp4: false
+              ens4f1:
+                dhcp4: false
+              ens4f2:
+                dhcp4: false
+              ens4f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [eno1, ens4f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens4f2, ens4f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:aa:e0:98':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eno1:
+                dhcp4: false
+              ens4f1:
+                dhcp4: false
+              ens4f2:
+                dhcp4: false
+              ens4f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [eno1, ens4f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens4f2, ens4f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+    '0c:c4:7a:aa:78:7c':
+      write_files:
+      - path: '/etc/netplan/99-cz7725.yaml'
+        content: |
+          network:
+            version: 2
+            ethernets:
+              eno1:
+                dhcp4: false
+              ens4f1:
+                dhcp4: false
+              ens4f2:
+                dhcp4: false
+              ens4f3:
+                dhcp4: false
+            bonds:
+              bond0:
+                dhcp4: no
+                interfaces: [eno1, ens4f1]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+              bond1:
+                dhcp4: no
+                interfaces: [ens4f2, ens4f3]
+                parameters:
+                  mode: active-backup
+                  mii-monitor-interval: 1
+            vlans:
+              k8s-control:
+                id: 411
+                link: bond0
+              os-floating:
+                id: 412
+                link: bond1
+              k8s-metallb:
+                id: 413
+                link: bond0
+              k8s-dataplane:
+                id: 414
+                link: bond0
+              os-storage:
+                id: 415
+                link: bond1
+              os-tenant:
+                id: 416
+                link: bond1
+
+  functions_override: |
+
+    function network_config {
+
+        DEBIAN_FRONTEND=noninteractive apt -y install bridge-utils ifenslave vlan
+        sudo modprobe bonding
+        netplan --debug apply
+        sleep 15
+
+        ip a a ${PUBLIC_INTERFACE_IP}/${PUBLIC_INTERFACE_NETMASK} dev ${PUBLIC_INTERFACE}
+    }
diff --git a/de/heat-templates/scripts/instance_boot.sh b/de/heat-templates/scripts/instance_boot.sh
index 4436054..6f281a8 100644
--- a/de/heat-templates/scripts/instance_boot.sh
+++ b/de/heat-templates/scripts/instance_boot.sh
@@ -400,6 +400,16 @@
         wait_for_node
         set_node_labels
         ;;
+    spare)
+        prepare_metadata_files
+        prepare_network
+        update_docker_network
+        install_required_packages
+        install_docker
+        network_config
+        download_bundles
+        workaround_default_forward_policy
+        ;;
     *)
         echo "Usage: $0 {ucp|master|worker}"
         exit 1
diff --git a/de/heat-templates/top.yaml b/de/heat-templates/top.yaml
index 0064c38..d2c4048 100644
--- a/de/heat-templates/top.yaml
+++ b/de/heat-templates/top.yaml
@@ -38,6 +38,10 @@
     type: number
     description: Number of gtw workers to deploy
     default: 0
+  spare_size:
+    type: number
+    description: Number of gtw workers to deploy
+    default: 0
   ntw_size:
     type: number
     description: Number of contrail nodes to deploy
@@ -98,6 +102,9 @@
   osd_metadata:
     type: json
     default: {}
+  spare_metadata:
+    type: json
+    default: {}
   ntw_metadata:
     type: json
     default: {}
@@ -122,6 +129,9 @@
   osds_flavor:
     type: string
     default: 'system.virtual.ceph.osd'
+  spares_flavor:
+    type: string
+    default: 'system.compact.openstack.control'
   ntws_flavor:
     type: string
     default: 'system.compact.openstack.control'
@@ -387,6 +397,38 @@
           storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
           hardware_metadata: { get_param: hardware_metadata}
 
+  spares:  # spares for workers
+    type: OS::Heat::ResourceGroup
+    depends_on:
+     - ucp
+    properties:
+      count: { get_param: spare_size }
+      resource_def:
+        type: MCP2::SrvInstancesCeph
+        properties:
+          metadata: { get_param: spare_metadata }
+          node_type: "spare"
+          key_name: { get_param: "OS::stack_name" }
+          image: { get_param: image }
+          flavor: { get_param: spares_flavor }
+          docker_ee_url: { get_param: docker_ee_url }
+          docker_ee_release: { get_param: docker_ee_release }
+          accessible_network: { get_attr: [accessible_network, public_network] }
+          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
+          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
+          private_floating_interface: { get_param: private_floating_interface }
+          private_floating_network_cidr: { get_param: private_floating_network_cidr }
+          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
+          public_net_id: { get_param: public_net_id }
+          control_network_cidr: { get_param: control_network_cidr }
+          ucp_master_host: { get_attr: [ucp, server_private_ip] }
+          functions_override: { get_param: functions_override }
+          storage_frontend_interface: { get_param: storage_frontend_interface }
+          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
+          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
+          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
+          hardware_metadata: { get_param: hardware_metadata}
+
   ntws:
     type: OS::Heat::ResourceGroup
     depends_on:
@@ -437,6 +479,9 @@
   osds_ips:
     description: Private IP addresses of the deployed osd instances
     value: { get_attr: [osds, server_public_ip] }
+  spares_ips:
+    description: Private IP addresses of the deployed spare instances
+    value: { get_attr: [spares, server_public_ip] }
   ntws_ips:
     description: Private IP addresses of the deployed contrail instances
     value: { get_attr: [ntws, server_public_ip] }