[ironic] Configure vxlan for multi-tenancy mode

Related-Bug: PRODX-1821
Change-Id: I1b3b0b80fba64161c63e4765ebf2545b4ee6585b
diff --git a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
index e96a0e3..bea8d12 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
@@ -44,6 +44,10 @@
     type: string
   ironic_baremetal_network_cidr:
     type: string
+  ironic_baremetal_tunnel_cidr:
+    type: string
+  ironic_mt_enabled:
+    type: boolean
   functions_override:
     type: string
   boot_timeout:
@@ -93,6 +97,8 @@
             $storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
             $ironic_baremetal_interface_ip: { get_attr: [ironic_baremetal_server_port, fixed_ips, 0, ip_address] }
             $ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+            $ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+            $ironic_mt_enabled: { get_param: ironic_mt_enabled }
 
   inject_files:
     type: "OS::Heat::CloudConfig"
diff --git a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
index b03830f..162da83 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
@@ -52,6 +52,10 @@
     type: string
   ironic_baremetal_network_cidr:
     type: string
+  ironic_baremetal_tunnel_cidr:
+    type: string
+  ironic_mt_enabled:
+    type: boolean
   tungstenfabric_enabled:
     type: boolean
   tf_data_network:
@@ -114,6 +118,8 @@
             $storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
             $ironic_baremetal_interface_ip: { get_attr: [ironic_baremetal_server_port, fixed_ips, 0, ip_address] }
             $ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+            $ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+            $ironic_mt_enabled: { get_param: ironic_mt_enabled }
 
   inject_files:
     type: "OS::Heat::CloudConfig"
diff --git a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
index a367c7f..de965d1 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
@@ -42,6 +42,10 @@
     type: string
   ironic_baremetal_network_cidr:
     type: string
+  ironic_baremetal_tunnel_cidr:
+    type: string
+  ironic_mt_enabled:
+    type: boolean
   control_network_cidr:
     type: string
   functions_override:
@@ -101,6 +105,8 @@
             $storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
             $ironic_baremetal_interface_ip: { get_attr: [ironic_baremetal_server_port, fixed_ips, 0, ip_address] }
             $ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+            $ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+            $ironic_mt_enabled: { get_param: ironic_mt_enabled }
 
   inject_files:
     type: "OS::Heat::CloudConfig"
diff --git a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
index 588b0c9..406e896 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
@@ -52,6 +52,10 @@
     type: string
   ironic_baremetal_network_cidr:
     type: string
+  ironic_baremetal_tunnel_cidr:
+    type: string
+  ironic_mt_enabled:
+    type: boolean
   tungstenfabric_enabled:
     type: boolean
   tf_data_network:
@@ -122,6 +126,8 @@
             $storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
             $ironic_baremetal_interface_ip: { get_attr: [ironic_baremetal_server_port, fixed_ips, 0, ip_address] }
             $ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+            $ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+            $ironic_mt_enabled: { get_param: ironic_mt_enabled }
 
   inject_files:
     type: "OS::Heat::CloudConfig"
diff --git a/de/heat-templates/scripts/instance_boot.sh b/de/heat-templates/scripts/instance_boot.sh
index 1d4275b..4186b29 100644
--- a/de/heat-templates/scripts/instance_boot.sh
+++ b/de/heat-templates/scripts/instance_boot.sh
@@ -23,6 +23,7 @@
 IRONIC_BAREMETAL_INTERFACE_IP=$ironic_baremetal_interface_ip
 IRONIC_BAREMETAL_NETWORK_NETMASK=$(echo ${IRONIC_BAREMETAL_NETWORK} | cut -d'/' -f2)
 IRONIC_BAREMETAL_INTERFACE=$(ip route get ${IRONIC_BAREMETAL_NETWORK%/*} | awk '/^broadcast / {print $4}')
+IRONIC_BAREMETAL_TUNNEL_NETWORK=$ironic_baremetal_tunnel_cidr
 NODE_TYPE=$node_type
 UCP_MASTER_HOST=$ucp_master_host
 NODE_METADATA='$node_metadata'
@@ -30,6 +31,9 @@
 DOCKER_EE_RELEASE='$docker_ee_release'
 DOCKER_UCP_IMAGE='$docker_ucp_image'
 FLOATING_NETWORK_PREFIXES=$private_floating_network_cidr
+IRONIC_MT_ENABLED=$ironic_mt_enabled
+IRONIC_BAREMETAL_NETWORK_PREFIX=$(sed 's/[0-9]*\/[0-9]*$//' <<< $IRONIC_BAREMETAL_NETWORK)
+IRONIC_BAREMETAL_TUNNEL_NETWORK_PREFIX=$(sed 's/[0-9]*\/[0-9]*$//' <<< $IRONIC_BAREMETAL_TUNNEL_NETWORK)
 #
 # End of block
 #
@@ -61,6 +65,7 @@
 UCP_IP_ADDRESS=${UCP_IP_ADDRESS:-$CONTROL_IP_ADDRESS}
 NTP_SERVERS=${NTP_SERVERS:-"ldap.scc.mirantis.net ldap.bud.mirantis.net"}
 DEFAULT_INTERFACE=$(ip route show default | awk '{print $5}')
+IRONIC_BAREMETAL_VXLAN_INTERFACE='vxlan10'
 
 
 function retry {
@@ -419,8 +424,30 @@
     sed -i 's/.*ethernets:.*/&\n        veth-vbmc: {}/' ${cloud_netplan_cfg}
     sed -i 's/.*ethernets:.*/&\n        veth-vbmc-br: {}/' ${cloud_netplan_cfg}
 
-    ironic_baremetal_address_match_ip_line=$(grep -nm1 "${IRONIC_BAREMETAL_INTERFACE_IP}/${IRONIC_BAREMETAL_NETWORK_NETMASK}" ${cloud_netplan_cfg} | cut -d: -f1)
-    sed -i "$((${ironic_baremetal_address_match_ip_line}-1)),$((${ironic_baremetal_address_match_ip_line}))d" ${cloud_netplan_cfg}
+    if [[ "${IRONIC_MT_ENABLED,,}" == true ]] ; then
+        sed -i "s/.*ethernets:.*/&\n        ${IRONIC_BAREMETAL_VXLAN_INTERFACE}: {}/" ${cloud_netplan_cfg}
+        sed -i 's/.*ethernets:.*/&\n        veth-pxe: {}/' ${cloud_netplan_cfg}
+        sed -i 's/.*ethernets:.*/&\n        veth-pxe-br: {}/' ${cloud_netplan_cfg}
+cat << EOF > /etc/systemd/network/13-veth-pxe.netdev
+[NetDev]
+Name=veth-pxe
+Kind=veth
+[Peer]
+Name=veth-pxe-br
+EOF
+        sed -i "s/- ${IRONIC_BAREMETAL_NETWORK_PREFIX}\([0-9]*\)/- ${IRONIC_BAREMETAL_TUNNEL_NETWORK_PREFIX}\1/" ${cloud_netplan_cfg}
+cat << EOF >> ${cloud_netplan_cfg}
+    vlans:
+        pxe.1000:
+            id: 1000
+            link: veth-pxe
+            addresses:
+            - ${IRONIC_BAREMETAL_INTERFACE_IP}/${IRONIC_BAREMETAL_NETWORK_NETMASK}
+EOF
+    else
+        ironic_baremetal_address_match_ip_line=$(grep -nm1 "${IRONIC_BAREMETAL_INTERFACE_IP}/${IRONIC_BAREMETAL_NETWORK_NETMASK}" ${cloud_netplan_cfg} | cut -d: -f1)
+        sed -i "$((${ironic_baremetal_address_match_ip_line}-1)),$((${ironic_baremetal_address_match_ip_line}))d" ${cloud_netplan_cfg}
+    fi
 fi
 
     public_address_match_ip_line=$(grep -nm1 "${PUBLIC_NODE_IP_ADDRESS}/${PUBLIC_NODE_IP_NETMASK}" ${cloud_netplan_cfg} | cut -d: -f1)
@@ -442,13 +469,38 @@
         br-baremetal:
             dhcp4: false
             interfaces:
-            - ${IRONIC_BAREMETAL_INTERFACE}
             - veth-bm-br
             - veth-vbmc-br
+EOF
+    if [[ "${IRONIC_MT_ENABLED,,}" != true ]] ; then
+cat << EOF >> ${cloud_netplan_cfg}
+            - ${IRONIC_BAREMETAL_INTERFACE}
             addresses:
             - ${IRONIC_BAREMETAL_INTERFACE_IP}/${IRONIC_BAREMETAL_NETWORK_NETMASK}
-
 EOF
+    else
+cat << EOF >> ${cloud_netplan_cfg}
+            - ${IRONIC_BAREMETAL_VXLAN_INTERFACE}
+            - veth-pxe-br
+EOF
+cat << EOF > /etc/systemd/system/ironic-vxlan-tunnel.service
+[Unit]
+Description=Ironic VXLAN tunnel
+After=network.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStart=/sbin/ip link add ${IRONIC_BAREMETAL_VXLAN_INTERFACE} type vxlan id 10 group 239.1.1.10 dstport 0 dev ${IRONIC_BAREMETAL_INTERFACE}
+ExecStart=/sbin/ip link set ${IRONIC_BAREMETAL_VXLAN_INTERFACE} up
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+        systemctl enable ironic-vxlan-tunnel.service
+        systemctl start ironic-vxlan-tunnel.service
+    fi
 fi
     netplan --debug apply
 
diff --git a/de/heat-templates/top.yaml b/de/heat-templates/top.yaml
index bfccd7a..58b634e 100644
--- a/de/heat-templates/top.yaml
+++ b/de/heat-templates/top.yaml
@@ -121,6 +121,9 @@
   ironic_baremetal_network_gateway:
     type: string
     default: ''
+  ironic_baremetal_tunnel_cidr:
+    type: string
+    default: '10.16.0.0/24'
   ironic_baremetal_network_interface:
     type: string
     default: 'veth-bm'
@@ -133,6 +136,13 @@
     type: string
     default: 'br-baremetal'
     description: 'The name of interface on node to which IP ironic services will bind.'
+  ironic_baremetal_pxe_interface:
+    type: string
+    default: 'pxe.1000'
+    description: 'Will be used as provisioning_interface for multi-tenancy mode'
+  ironic_mt_enabled:
+    type: boolean
+    default: false
   tungstenfabric_enabled:
     type: boolean
     default: false
@@ -381,6 +391,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
           tf_data_network: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_network_id] }, ""]}
           tf_data_subnet_id: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_subnet_id] }, ""]}
@@ -424,6 +436,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
           tf_data_network: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_network_id] }, ""]}
           tf_data_subnet_id: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_subnet_id] }, ""]}
@@ -463,6 +477,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           hardware_metadata: { get_param: hardware_metadata}
 
   lmas:
@@ -499,6 +515,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           hardware_metadata: { get_param: hardware_metadata}
 
   osds:
@@ -539,6 +557,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
           tf_data_network: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_network_id] }, ""]}
           tf_data_subnet_id: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_subnet_id] }, ""]}
@@ -582,6 +602,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
           tf_data_network: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_network_id] }, ""]}
           tf_data_subnet_id: {if: ["create_tf_data_network_res", { get_attr: [tf_data_network, tf_data_subnet_id] }, ""]}
@@ -621,6 +643,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           hardware_metadata: { get_param: hardware_metadata}
 
   vbmcs:
@@ -657,6 +681,8 @@
           ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
           ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
           ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
+          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
+          ironic_mt_enabled: { get_param: ironic_mt_enabled }
           hardware_metadata: { get_param: hardware_metadata}
 
 outputs:
@@ -716,6 +742,8 @@
     value: { get_param: ironic_baremetal_network_vbmc_interface }
   ironic_baremetal_provisioning_interface:
     value: { get_param: ironic_baremetal_provisioning_interface }
+  ironic_baremetal_pxe_interface:
+    value: { get_param: ironic_baremetal_pxe_interface }
   tunnel_interface:
     value: { get_param: tunnel_interface }
   live_migration_interface: