de/heat-templates/top.yaml

heat_template_version: queens

parameters:
  image:
    type: string
    description: Name of image to use for servers
  public_net_id:
    type: string
    default: ''
    description: >
      ID of public network for which floating IP addresses will be allocated/
      for baremetal case flat provision network for nodes
  dns_nameservers:
    type: json
    default: []
  control_network_cidr:
    type: string
    description: The CIDR of control network, used to detect control interface.
    default: '10.10.0.0/24'
  control_network_host_routes:
    type: json
    description: >
      List of dicts with host routes in control subnet, e.g:
      - "destination": "10.100.100.1/32"
        "nexthop": "172.16.1.1"
    default: []
  control_network_ext_router_ip:
    type: string
    description: IP address of vsrx/vmx in control network
    default: ''
  docker_default_address_pool:
    type: string
    description: Default address pool for Docker ucp specific local networks
    default: '10.10.1.0/16'
  masters_size:
    type: number
    description: Number of masters instances to deploy
    default: 2
  worker_size:
    type: number
    description: Number of workers to deploy
    default: 5
  cmp_size:
    type: number
    description: Number of cmp workers to deploy
    default: 0
  acmp_size:
    type: number
    description: Number of advanced cmp workers to deploy
    default: 0
  gtw_size:
    type: number
    description: Number of gtw workers to deploy
    default: 0
  lma_size:
    type: number
    description: Number of gtw workers to deploy
    default: 0
  osd_size:
    type: number
    description: Number of gtw workers to deploy
    default: 0
  frr_size:
    type: number
    description: Number of frr nodes to deploy
    default: 0
  ntw_size:
    type: number
    description: Number of contrail nodes to deploy
    default: 0
  vbmc_size:
    type: number
    description: Number of nodes for virtualbmc BM node simulator
    default: 0
  tsrv_size:
    type: number
    description: Number of nodes for test servers
    default: 1
  ucp_boot_timeout:
    type: number
    description: Boot timeout for UCP instance
    default: 3600
  cluster_public_key:
    type: string
  private_floating_network_cidr:
    type: string
    default: '10.11.12.0/24'
  private_floating_network_ipam_pool_start:
    type: string
    description: pool start which is used as pool for IPAM and assigned to instances port
    default: '10.11.12.2'
  private_floating_network_ipam_pool_end:
    type: string
    description: pool end which is used as pool for IPAM and assigned to instances port
    default: '10.11.12.70'
  private_floating_network_pool_start:
    type: string
    default: '10.11.12.100'
  private_floating_network_pool_end:
    type: string
    default: '10.11.12.200'
  private_floating_network_gateway:
    type: string
    default: ''
  private_floating_network_interface:
    type: string
    default: 'veth-phy'
    description: 'The name of interface that should be added to external neutron bridge for floating network.'
  private_floating_interface:
    description: Interface which carries floating network for child OpenStack.
    type: string
  storage_frontend_network_cidr:
    type: string
    default: '10.12.0.0/24'
  storage_frontend_network_gateway:
    type: string
    default: '10.12.0.1/24'
  storage_frontend_network_ipam_pool_start:
    type: string
    default: '10.12.0.3'
  storage_frontend_network_ipam_pool_end:
    type: string
    default: '10.12.0.254'
  storage_frontend_interface:
    type: string
    default: 'ens5'
  storage_backend_network_cidr:
    type: string
    default: '10.11.0.0/24'
  storage_backend_interface:
    type: string
    default: 'ens6'
  ironic_baremetal_network_cidr:
    type: string
    default: '10.13.0.0/24'
  ironic_baremetal_network_ipam_pool_start:
    type: string
    default: '10.13.0.2'
  ironic_baremetal_network_ipam_pool_end:
    type: string
    default: '10.13.0.99'
  ironic_baremetal_network_pool_start:
    type: string
    default: '10.13.0.100'
  ironic_baremetal_network_pool_end:
    type: string
    default: '10.13.0.200'
  ironic_baremetal_network_gateway:
    type: string
    default: ''
  ironic_baremetal_tunnel_cidr:
    type: string
    default: '10.16.0.0/24'
  ironic_baremetal_network_interface:
    type: string
    default: 'veth-bm'
    description: 'The name of interface that should be added to ovs bridge in Neutron'
  ironic_baremetal_network_vbmc_interface:
    type: string
    default: 'veth-vbmc'
    description: 'The name of interface that should be added to ovs bridge with virtualbmc VMs.'
  ironic_baremetal_provisioning_interface:
    type: string
    default: 'br-baremetal'
    description: 'The name of interface on node to which IP ironic services will bind.'
  ironic_baremetal_pxe_interface:
    type: string
    default: 'pxe.1000'
    description: 'Will be used as provisioning_interface for multi-tenancy mode'
  ironic_mt_enabled:
    type: boolean
    default: false
  tungstenfabric_enabled:
    type: boolean
    default: false
  vmx_linux_img:
    type: string
    description: Name of image to use for servers
    default: "vPFC-20170216"
  vmx_linux_flav:
    type: string
    description: Name of image to use for servers
    default: "vfp.lite"
  vmx_junos_img:
    type: string
    description: Name of image to use for servers
    default: "vmx-x86-64-17.1R1.8"
  vmx_junos_flav:
    type: string
    description: Name of image to use for servers
    default: "vcp.lite"
  vmx_gateway_ip:
    type: string
    description: Default GW for vmx
    default: '10.10.0.1'
  vmx_lo_ip:
    type: string
    description: Ip of lo interface for vmx
    default: '10.199.199.199'
  tun_network_cidr:
    type: string
    default: '10.15.0.0/24'
  # When tun interface is added to ovs bridge inside vm,
  # and arp responder is enabled on undercloud,
  # ip adresses from tun network pool can't be used for
  # configuring vxlan tunnels through ovs bridge (https://mirantis.jira.com/browse/PRODX-9337).
  # Ip addresses unknown for arp responder should be used.
  # In deployment pipeline addresses assigned by openstack to advanced compute nodes
  # are changed inside instance to 10.15.0.1XX for this case.
  tun_network_pool_start:
    type: string
    default: '10.15.0.2'
  tun_network_pool_end:
    type: string
    default: '10.15.0.99'
  tun_network_ext_router_ip:
    type: string
    description: IP address of vsrx/vmx in data network
    default: ''
  tun_network_host_routes:
    type: json
    description: >
      List of dicts with host routes in tunnel subnet, e.g:
      - "destination": "10.100.100.1/32"
        "nexthop": "172.16.1.1"
    default: []
  evpn_network_cidr:
    type: string
    default: '10.20.100.0/24'
  evpn_network_ipam_pool_start:
    type: string
    description: pool start which is used as pool for IPAM and assigned to instances port
    default: '10.20.100.2'
  evpn_network_ipam_pool_end:
    type: string
    description: pool end which is used as pool for IPAM and assigned to instances port
    default: '10.20.100.99'
  evpn_network_pool_start:
    type: string
    default: '10.20.100.105'
  evpn_network_pool_end:
    type: string
    default: '10.20.100.200'
  evpn_network_vmx_ip:
    type: string
    description: IP address of vmx in a network for evpn test
    default: ''
  ucp_metadata:
    type: json
    default: {"role":"ucp"}
  ucp_audit_log_level:
    type: string
    default: ""
  master_metadata:
    type: json
    default: {"role":"master"}
  worker_metadata:
    type: json
    default: {}
  cmp_metadata:
    type: json
    default: {}
  acmp_metadata:
    type: json
    default: {}
  gtw_metadata:
    type: json
    default: {}
  lma_metadata:
    type: json
    default: {}
  osd_metadata:
    type: json
    default: {}
  frr_metadata:
    type: json
    default: {}
  ntw_metadata:
    type: json
    default: {}
  vbmc_metadata:
    type: json
    default: {}
  tsrv_metadata:
    type: json
    default: {"labels": {"tempest": "enabled", "stepler": "enabled", "local-volume-provisioner": "enabled"}}
  ucp_flavor:
    type: string
    default: 'mosk.s.ucp'
  masters_flavor:
    type: string
    default: 'system.compact.openstack.proxy'
  workers_flavor:
    type: string
    default: 'system.compact.openstack.control'
  cmps_flavor:
    type: string
    default: 'system.compact.openstack.control'
  acmps_flavor:
    type: string
    default: 'system.compact.openstack.control.ephemeral.numa'
  gtws_flavor:
    type: string
    default: 'system.compact.openstack.control'
  lmas_flavor:
    type: string
    default: 'system.compact.openstack.control'
  osds_flavor:
    type: string
    default: 'system.virtual.ceph.osd'
  frrs_flavor:
    type: string
    default: 'system.compact.openstack.control'
  ntws_flavor:
    type: string
    default: 'mos.tf.control'
  vbmcs_flavor:
    type: string
    default: 'system.compact.openstack.control'
  tsrvs_flavor:
    type: string
    default: 'mosk.s.testserver'
  functions_override:
    type: string
    default: ''
  docker_ee_url:
    type: string
    default: 'https://repos.mirantis.com/ubuntu'
  docker_ee_release:
    type: string
    default: 'stable-19.03'
  docker_ee_packages:
    type: string
    default: "docker-ee"
  kubernetes_container_runtime:
    type: string
    default: 'docker'
  docker_ucp_image:
    type: string
    default: 'docker/ucp:3.2.4'
  docker_ucp_swarm_data_port:
    type: string
    default: 4789
  binary_base_url:
    type: string
    default: 'http://binary.mirantis.com'
  tunnel_interface:
    type: string
    default: ''
  live_migration_interface:
    type: string
    default: ''
  metallb_address_pools:
    type: comma_delimited_list
    default: ''
  hardware_metadata:
    description: The content of lab metadata.
    default: ''
    type: string
  huge_pages:
    description: >
      String formatted as '<Number of Gb for 1Gb huge pages>,<Number of Mb for 2Mb huge pages'
      If number is equal to 0 alloacation of huge pages won't be configured.
    default: '0,0'
    type: string
  single_node:
    description: If set true then AIO is going to be deployed.
    type: string
    default: 'false'
  vsrx_image:
    type: string
    default: mos-vsrx-acc-fl-tun
  vsrx_flavor:
    type: string
    default: oc_vsrx
  cmp_lvm_loop_device_size:
    description: >
      The size of loop device for computes to configure LVM on, in gigabytes
    type: number
    default: 50
  cmp_cinder_lvm_loop_device_size:
    description: >
      The size of loop device for computes to configure LVM on, in gigabytes
    type: number
    default: 0
  acmp_lvm_loop_device_size:
    description: >
      The size of loop device for advanced computes to configure LVM on, in gigabytes
    type: number
    default: 0
  acmp_cinder_lvm_loop_device_size:
    description: >
      The size of loop device for advanced computes to configure LVM on, in gigabytes
    type: number
    default: 0
  kubernetes_installer:
    type: string
    default: "ucp"
  kubectl_version:
    type: string
    default: 1.18.8
  volumes_per_osd_instance:
    description: >
      Number of extra volumes for OSD instance
    type: number
    default: 0
  availability_zone:
    type: string
    default: nova
  secure_overlay_enabled:
    type: boolean
    default: false
  vsrx_enabled:
    type: boolean
    default: false
  vmx_enabled:
    type: boolean
    default: false
  devops_utils_refspec:
    type: string
    default: 'master'
  k0s_version:
    type: string
    default: ''
  external_k8s_service_network_cidr:
    type: string
    default: '10.172.1.0/24'

conditions:
  aio_deploy:
    equals:
    - get_param: single_node
    - 'true'
  tf:
    equals:
      - get_param: tungstenfabric_enabled
      - true
  vsrx_deploy:
    equals:
      - get_param: vsrx_enabled
      - true
  vmx_deploy:
    and:
      - tf
      - equals:
          - get_param: vmx_enabled
          - true
      - not:
          vsrx_deploy
  ext_router_deploy:
    or:
      - vsrx_deploy
      - vmx_deploy

resources:
  keypair_name:
    type: OS::Heat::RandomString
    properties:
      character_classes: [{"class": "hexdigits", "min": 1}]
      length: 128
      salt: constant
  key_pair:
    type: OS::Nova::KeyPair
    properties:
      name: { get_attr: [keypair_name, value] }
      public_key: { get_param: cluster_public_key }
      save_private_key: false

  tun_network:
    type: MCP2::NetworkTun
    properties:
      tun_network_cidr: { get_param: tun_network_cidr }
      tun_network_pool_start: { get_param: tun_network_pool_start }
      tun_network_pool_end: { get_param: tun_network_pool_end }
      tun_network_host_routes: { get_param: tun_network_host_routes }

  private_floating_network:
    type: MCP2::NetworkPrvFl
    properties:
      private_floating_network_cidr: { get_param: private_floating_network_cidr }
      private_floating_network_ipam_pool_start: { get_param: private_floating_network_ipam_pool_start }
      private_floating_network_ipam_pool_end: { get_param: private_floating_network_ipam_pool_end }

  storage_network:
    type: MCP2::NetworkAccStorage
    properties:
      storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
      storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
      storage_frontend_network_ipam_pool_start: { get_param: storage_frontend_network_ipam_pool_start }
      storage_frontend_network_ipam_pool_end: { get_param: storage_frontend_network_ipam_pool_end }

  accessible_network:
    depends_on:
      - private_floating_network
      - storage_network
    type: MCP2::NetworkAcc
    properties:
      public_net_id: { get_param: public_net_id }
      control_network_cidr: { get_param: control_network_cidr }
      dns_nameservers: { get_param: dns_nameservers }
      control_network_host_routes: { get_param: control_network_host_routes }
      tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
      vsrx_enabled: { get_param: vsrx_enabled }
      control_network_ext_router_ip: { get_param: control_network_ext_router_ip }
      private_floating_network: {get_attr: [private_floating_network, private_floating_network_id]}
      private_floating_network_cidr: { get_param: private_floating_network_cidr }
      private_floating_network_gateway: { get_param: private_floating_network_gateway }
      private_floating_subnet: {get_attr: [private_floating_network, private_floating_subnet_id]}
      storage_frontend_network: {get_attr: [storage_network, storage_frontend_network_id]}
      storage_frontend_network_gateway: { get_param: storage_frontend_network_gateway }
      storage_frontend_subnet: {get_attr: [storage_network, storage_frontend_subnet_id]}

  vmx:
    depends_on:
      - private_floating_network
      - accessible_network
      - tun_network
    condition: vmx_deploy
    type: MCP2::VMX
    properties:
      linux_img: { get_param: vmx_linux_img }
      linux_flav: { get_param: vmx_linux_flav }
      junos_img: { get_param: vmx_junos_img }
      junos_flav: { get_param: vmx_junos_flav }
      vmx_gateway_ip: { get_param: vmx_gateway_ip }
      vmx_lo_ip: { get_param: vmx_lo_ip }
      accessible_network_id: { get_attr: [ accessible_network, public_network ] }
      control_network_cidr: { get_param: control_network_cidr }
      control_network_vmx_ip: { get_param: control_network_ext_router_ip }
      tun_network_id: { get_attr: [ tun_network, tun_network_id ] }
      tun_network_vmx_ip: { get_param: tun_network_ext_router_ip }
      tun_network_cidr: { get_param: tun_network_cidr }
      private_floating_network_id: { get_attr: [private_floating_network, private_floating_network_id] }
      private_floating_network_cidr: { get_param: private_floating_network_cidr }
      private_floating_network_gateway: { get_param: private_floating_network_gateway }
      evpn_network_cidr: { get_param: evpn_network_cidr }
      evpn_network_ipam_pool_start: { get_param: evpn_network_ipam_pool_start }
      evpn_network_ipam_pool_end: { get_param: evpn_network_ipam_pool_end }
      evpn_network_vmx_ip: { get_param: evpn_network_vmx_ip }
      public_net_id: { get_param: public_net_id }

  vsrx:
    depends_on:
      - private_floating_network
      - accessible_network
      - tun_network
    condition: vsrx_deploy
    type: MCP2::VSRX
    properties:
      private_floating_network_id: { get_attr: [private_floating_network, private_floating_network_id] }
      private_floating_network_gateway: { get_param: private_floating_network_gateway }
      control_network_vsrx_ip: { get_param: control_network_ext_router_ip }
      accessible_network: { get_attr: [ accessible_network, public_network ] }
      data_network_vsrx_ip: { get_param: tun_network_ext_router_ip }
      tun_network: { get_attr: [ tun_network, tun_network_id ] }
      vsrx_image: { get_param: vsrx_image }
      vsrx_flavor: { get_param: vsrx_flavor }
      public_net_id: { get_param: public_net_id }

  ironic_baremetal_network:
    type: MCP2::NetworkIronicFlat
    properties:
      ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
      ironic_baremetal_network_ipam_pool_start: { get_param: ironic_baremetal_network_ipam_pool_start }
      ironic_baremetal_network_ipam_pool_end: { get_param: ironic_baremetal_network_ipam_pool_end }
      ironic_baremetal_network_gateway: {get_param: ironic_baremetal_network_gateway}

  ucp:
    depends_on:
     - accessible_network
    type: MCP2::SrvInstances
    properties:
      metadata: { get_param: ucp_metadata}
      docker_ee_url: { get_param: docker_ee_url }
      docker_ee_release: { get_param: docker_ee_release }
      docker_ee_packages: { get_param: docker_ee_packages }
      kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
      docker_ucp_image: { get_param: docker_ucp_image}
      docker_ucp_swarm_data_port: { get_param: docker_ucp_swarm_data_port }
      docker_default_address_pool: { get_param: docker_default_address_pool }
      binary_base_url: { get_param: binary_base_url }
      node_type: "ucp"
      kubernetes_installer: { get_param: kubernetes_installer }
      key_name: { get_attr: [keypair_name, value] }
      image: { get_param: image }
      flavor: { get_param: ucp_flavor }
      accessible_network: { get_attr: [accessible_network, public_network] }
      private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
      private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
      private_floating_interface: { get_param: private_floating_interface }
      private_floating_network_cidr: { get_param: private_floating_network_cidr }
      accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
      public_net_id: { get_param: public_net_id }
      control_network_cidr: { get_param: control_network_cidr }
      functions_override: { get_param: functions_override }
      hardware_metadata: { get_param: hardware_metadata}
      single_node: { get_param: single_node}
      availability_zone: { get_param: availability_zone }
      secure_overlay_enabled: { get_param: secure_overlay_enabled }
      kubectl_version: { get_param: kubectl_version }
      devops_utils_refspec: { get_param: devops_utils_refspec }
      k0s_version: { get_param: k0s_version }
      ucp_audit_log_level: { get_param: ucp_audit_log_level }

  masters:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: masters_size }
      resource_def:
        type: MCP2::SrvInstances
        properties:
          metadata: { get_param: master_metadata}
          node_type: "master"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: masters_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          hardware_metadata: { get_param: hardware_metadata}
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  workers:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: worker_size }
      resource_def:
        type: MCP2::SrvInstancesCephOSD
        properties:
          metadata: { get_param: worker_metadata}
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: workers_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          storage_backend_interface: { get_param: storage_backend_interface }
          storage_backend_network: { get_attr: [storage_network, storage_backend_network_id] }
          storage_backend_subnet_id: { get_attr: [storage_network, storage_backend_subnet_id] }
          storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
          tun_network: { get_attr: [tun_network, tun_network_id] }
          tun_subnet_id: { get_attr: [tun_network, tun_subnet_id] }
          hardware_metadata: { get_param: hardware_metadata}
          lvm_loop_device_size: { get_param: cmp_lvm_loop_device_size }
          cinder_lvm_loop_device_size: { get_param: cmp_cinder_lvm_loop_device_size }
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  cmps:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: cmp_size }
      resource_def:
        type: MCP2::SrvInstancesCephOSD
        properties:
          metadata: { get_param: cmp_metadata }
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: cmps_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          storage_backend_interface: { get_param: storage_backend_interface }
          storage_backend_network: { get_attr: [storage_network, storage_backend_network_id] }
          storage_backend_subnet_id: { get_attr: [storage_network, storage_backend_subnet_id] }
          storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
          tun_network: { get_attr: [tun_network, tun_network_id] }
          tun_subnet_id: { get_attr: [tun_network, tun_subnet_id] }
          hardware_metadata: { get_param: hardware_metadata}
          lvm_loop_device_size: { get_param: cmp_lvm_loop_device_size }
          cinder_lvm_loop_device_size: { get_param: cmp_cinder_lvm_loop_device_size }
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  acmps:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: acmp_size }
      resource_def:
        type: MCP2::SrvInstancesCephOSD
        properties:
          metadata: { get_param: acmp_metadata }
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: acmps_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          storage_backend_interface: { get_param: storage_backend_interface }
          storage_backend_network: { get_attr: [storage_network, storage_backend_network_id] }
          storage_backend_subnet_id: { get_attr: [storage_network, storage_backend_subnet_id] }
          storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
          tun_network: { get_attr: [tun_network, tun_network_id] }
          tun_subnet_id: { get_attr: [tun_network, tun_subnet_id] }
          hardware_metadata: { get_param: hardware_metadata}
          huge_pages: { get_param: huge_pages }
          lvm_loop_device_size: { get_param: acmp_lvm_loop_device_size }
          cinder_lvm_loop_device_size: { get_param: acmp_cinder_lvm_loop_device_size }
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  gtws:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: gtw_size }
      resource_def:
        type: MCP2::SrvInstancesCeph
        properties:
          metadata: { get_param: gtw_metadata }
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: gtws_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          hardware_metadata: { get_param: hardware_metadata}
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  lmas:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: lma_size }
      resource_def:
        type: MCP2::SrvInstancesCeph
        properties:
          metadata: { get_param: lma_metadata }
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: lmas_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          hardware_metadata: { get_param: hardware_metadata}
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  osds:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: osd_size }
      resource_def:
        type: MCP2::SrvInstancesCephOSD
        properties:
          metadata: { get_param: osd_metadata }
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: osds_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          storage_backend_interface: { get_param: storage_backend_interface }
          storage_backend_network: { get_attr: [storage_network, storage_backend_network_id] }
          storage_backend_subnet_id: { get_attr: [storage_network, storage_backend_subnet_id] }
          storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
          tun_network: { get_attr: [tun_network, tun_network_id] }
          tun_subnet_id: { get_attr: [tun_network, tun_subnet_id] }
          hardware_metadata: { get_param: hardware_metadata}
          num_volumes: { get_param: volumes_per_osd_instance }
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  frrs:  # spares for osds/cmps
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: frr_size }
      resource_def:
        type: MCP2::SrvInstancesCephOSD
        properties:
          metadata: { get_param: frr_metadata }
          node_type: "frr"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: frrs_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          storage_backend_interface: { get_param: storage_backend_interface }
          storage_backend_network: { get_attr: [storage_network, storage_backend_network_id] }
          storage_backend_subnet_id: { get_attr: [storage_network, storage_backend_subnet_id] }
          storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
          tun_network: { get_attr: [tun_network, tun_network_id] }
          tun_subnet_id: { get_attr: [tun_network, tun_subnet_id] }
          hardware_metadata: { get_param: hardware_metadata}
          frr_bgp_neighbors: { list_join: [',', {get_attr: [workers, server_tun_ip]}] }
          availability_zone: { get_param: availability_zone }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  ntws:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: ntw_size }
      resource_def:
        type: MCP2::SrvInstancesCephOSD
        properties:
          metadata: { get_param: ntw_metadata }
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: ntws_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          storage_backend_interface: { get_param: storage_backend_interface }
          storage_backend_network: { get_attr: [ storage_network, storage_backend_network_id ] }
          storage_backend_subnet_id: { get_attr: [ storage_network, storage_backend_subnet_id ] }
          storage_backend_network_cidr: { get_param: storage_backend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          tungstenfabric_enabled: { get_param: tungstenfabric_enabled }
          tun_network: { get_attr: [tun_network, tun_network_id] }
          tun_subnet_id: { get_attr: [tun_network, tun_subnet_id] }
          hardware_metadata: { get_param: hardware_metadata}
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  vbmcs:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: vbmc_size }
      resource_def:
        type: MCP2::SrvInstancesCeph
        properties:
          metadata: { get_param: vbmc_metadata}
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: vbmcs_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          hardware_metadata: { get_param: hardware_metadata}
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  tsrvs:
    type: OS::Heat::ResourceGroup
    depends_on:
     - ucp
    properties:
      count: { get_param: tsrv_size }
      resource_def:
        type: MCP2::SrvInstancesCeph
        properties:
          metadata: { get_param: tsrv_metadata}
          node_type: "worker"
          kubernetes_installer: { get_param: kubernetes_installer }
          key_name: { get_attr: [keypair_name, value] }
          image: { get_param: image }
          flavor: { get_param: tsrvs_flavor }
          docker_ee_url: { get_param: docker_ee_url }
          docker_ee_release: { get_param: docker_ee_release }
          docker_ee_packages: { get_param: docker_ee_packages }
          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
          docker_ucp_image: { get_param: docker_ucp_image}
          docker_default_address_pool: { get_param: docker_default_address_pool }
          binary_base_url: { get_param: binary_base_url }
          accessible_network: { get_attr: [accessible_network, public_network] }
          private_floating_network: { get_attr: [private_floating_network, private_floating_network_id] }
          private_floating_subnet_id: { get_attr: [private_floating_network, private_floating_subnet_id] }
          private_floating_interface: { get_param: private_floating_interface }
          private_floating_network_cidr: { get_param: private_floating_network_cidr }
          accessible_subnet_id: { get_attr: [accessible_network, accessible_subnet_id]}
          public_net_id: { get_param: public_net_id }
          control_network_cidr: { get_param: control_network_cidr }
          ucp_master_host: { get_attr: [ucp, server_private_ip] }
          functions_override: { get_param: functions_override }
          storage_frontend_interface: { get_param: storage_frontend_interface }
          storage_frontend_network: { get_attr: [storage_network, storage_frontend_network_id] }
          storage_frontend_subnet_id: { get_attr: [storage_network, storage_frontend_subnet_id] }
          storage_frontend_network_cidr: { get_param: storage_frontend_network_cidr }
          ironic_baremetal_network: { get_attr: [ironic_baremetal_network, ironic_baremetal_network_id] }
          ironic_baremetal_subnet_id: { get_attr: [ironic_baremetal_network, ironic_baremetal_subnet_id] }
          ironic_baremetal_network_cidr: { get_param: ironic_baremetal_network_cidr }
          ironic_baremetal_tunnel_cidr: { get_param: ironic_baremetal_tunnel_cidr }
          ironic_mt_enabled: { get_param: ironic_mt_enabled }
          hardware_metadata: { get_param: hardware_metadata}
          availability_zone: { get_param: availability_zone }
          kubectl_version: { get_param: kubectl_version }
          devops_utils_refspec: { get_param: devops_utils_refspec }

  k8s_external_service_route:
    type: OS::Neutron::ExtraRoute
    depends_on:
     - tsrvs
    condition: { not: aio_deploy }
    properties:
      destination: { get_param: external_k8s_service_network_cidr }
      nexthop: { get_attr: [tsrvs, resource.0, server_private_ip ] }
      router_id: { get_attr: [accessible_network, accessible_router] }

outputs:
  ucp_ips:
    description: Private IP addresses of the deployed ucp instances
    value: { get_attr: [ucp, server_public_ip] }
  masters_ips:
    description: Private IP addresses of the deployed masters instances
    value: { get_attr: [masters, server_public_ip] }
  workers_ips:
    description: Private IP addresses of the deployed worker instances
    value: { get_attr: [workers, server_public_ip] }
  cmps_ips:
    description: Private IP addresses of the deployed cmp instances
    value: { get_attr: [cmps, server_public_ip] }
  acmps_ips:
    description: Private IP addresses of the deployed cmp instances with high computing features
    value: { get_attr: [acmps, server_public_ip] }
  gtws_ips:
    description: Private IP addresses of the deployed gtw instances
    value: { get_attr: [gtws, server_public_ip] }
  lmas_ips:
    description: Private IP addresses of the deployed lma instances
    value: { get_attr: [lmas, server_public_ip] }
  osds_ips:
    description: Private IP addresses of the deployed osd instances
    value: { get_attr: [osds, server_public_ip] }
  ntws_ips:
    description: Private IP addresses of the deployed contrail instances
    value: { get_attr: [ntws, server_public_ip] }
  vbmcs_ips:
    description: Private IP addresses of the deployed virtualbmcs instances
    value: { get_attr: [vbmcs, server_public_ip] }
  vbmc_ironic_baremetal_ips:
    description: The IPs of ironic baremetal network assigned to vbmc hosts.
    value: { get_attr: [vbmcs, server_ironic_baremetal_ip] }
  frrs_ips:
    description: Private IP addresses of the deployed FRR instances
    value: { get_attr: [frrs, server_public_ip] }
  frrs_tunnel_ips:
    description: The tunnel IPs of FRR nodes
    value: { get_attr: [frrs, server_tun_ip] }
  ucp_private_floating_ip:
    description: Private IP addresses of the deployed ucp instances
    value: { get_attr: [ucp, server_private_floating_ip] }
  worker_private_floating_ips:
    description: IPs might be used as gateway
    value: {if: [aio_deploy, {get_attr: [ucp, server_private_floating_ip]}, {get_attr: [workers, server_private_floating_ip]}]}
  tsrvs_ips:
    description: Public IP addresses of the deployed tsrv instances
    value: { get_attr: [tsrvs, server_public_ip] }
  tsrvs_private_floating_ips:
    description: Private IP addresses of the deployed tsrv instances
    value: { get_attr: [tsrvs, server_private_floating_ip] }
  private_floating_cidr:
    value: { get_param: private_floating_network_cidr }
  private_floating_network_pool_start:
    value: { get_param: private_floating_network_pool_start }
  private_floating_network_pool_end:
    value: { get_param: private_floating_network_pool_end }
  private_floating_network_gateway:
    value: { get_param: private_floating_network_gateway }
  private_floating_network_interface:
    value: { get_param: private_floating_network_interface }
  ironic_baremetal_network_pool_start:
    value: { get_param: ironic_baremetal_network_pool_start }
  ironic_baremetal_network_pool_end:
    value: { get_param: ironic_baremetal_network_pool_end }
  ironic_baremetal_network_gateway:
    value: { get_param: ironic_baremetal_network_gateway }
  ironic_baremetal_network_interface:
    value: { get_param: ironic_baremetal_network_interface }
  ironic_baremetal_network_vbmc_interface:
    value: { get_param: ironic_baremetal_network_vbmc_interface }
  ironic_baremetal_provisioning_interface:
    value: { get_param: ironic_baremetal_provisioning_interface }
  ironic_baremetal_pxe_interface:
    value: { get_param: ironic_baremetal_pxe_interface }
  tunnel_interface:
    value: { get_param: tunnel_interface }
  live_migration_interface:
    value: { get_param: live_migration_interface }
  metallb_address_pools:
    value: { get_param: metallb_address_pools }
  ucp_wc_data:
    description: Metadata from ucp
    value: { get_attr: [ucp, wc_data] }
  masters_wc_data:
    description: Metadata from masters
    value: { get_attr: [masters, wc_data] }
  workers_wc_data:
    description: Metadata from workers
    value: { get_attr: [workers, wc_data] }
  cmps_wc_data:
    description: Metadata from cmps
    value: { get_attr: [cmps, wc_data] }
  acmps_wc_data:
    description: Metadata from cmps with high computing features
    value: { get_attr: [acmps, wc_data] }
  gtws_wc_data:
    description: Metadata from gtws
    value: { get_attr: [gtws, wc_data] }
  lmas_wc_data:
    description: Metadata from lmas
    value: { get_attr: [lmas, wc_data] }
  osds_wc_data:
    description: Metadata from osds
    value: { get_attr: [osds, wc_data] }
  ntws_wc_data:
    description: Metadata from ntws
    value: { get_attr: [ntws, wc_data] }
  storage_frontend_network_cidr:
    description: Storage network which is used as clientNet in Ceph CR
    value: { get_param: storage_frontend_network_cidr }
  storage_backend_network_cidr:
    description: Storage network which is used as clusterNet in Ceph CR
    value: { get_param: storage_backend_network_cidr }
  tungstenfabric_enabled:
    value: { get_param: tungstenfabric_enabled }
  tun_network_ext_router_ip:
    condition: ext_router_deploy
    description: IP address of vsrx/vmx in data network
    value: { get_param: tun_network_ext_router_ip }
  control_network_ext_router_ip:
    condition: ext_router_deploy
    description: IP address of vsrx/vmx in control/accessible network
    value: { get_param: control_network_ext_router_ip }
  public_network_ext_router_ip:
    condition: ext_router_deploy
    description: Public IP of vsrx/vmx instance
    value: { if: [ vmx_deploy, get_attr: [ vmx, re_floating_ip ], get_attr: [ vsrx, server_public_ip ] ] }
  accessible_router:
    description: Router from accessible network to public
    value: { get_attr: [accessible_network, accessible_router] }
  evpn_network_cidr:
    description: Network CIDR for evpn tests
    value: { get_param: evpn_network_cidr }
  evpn_network_pool_start:
    description: IPAM pool start for evpn tests
    value: { get_param: evpn_network_pool_start }
  evpn_network_pool_end:
    description: IPAM pool end for evpn tests
    value: { get_param: evpn_network_pool_end }