Fix stack readiness
The patch reworks heat templates to ensure stack is ready
only when all nodes are bootstrapped and wait conditions
are completed.
Related-Prod: PRODX-2063
Change-Id: I03905a80c05ab5933d8641529f97afe4d0097825
diff --git a/de/heat-templates/scripts/instance_boot.sh b/de/heat-templates/scripts/instance_boot.sh
index b48ff49..c03f7b2 100644
--- a/de/heat-templates/scripts/instance_boot.sh
+++ b/de/heat-templates/scripts/instance_boot.sh
@@ -11,6 +11,7 @@
UCP_PASSWORD=${UCP_PASSWORD:-administrator}
OS_CODENAME=$(lsb_release -c -s)
KUBECTL_VERSION=${KUBECTL_VERSION:-v1.14.0}
+NODE_DEPLOYMENT_RETRIES=${NODE_DEPLOYMENT_RETRIES:-15}
NODE_TYPE=$node_type
UCP_MASTER_HOST=$ucp_master_host
@@ -122,6 +123,14 @@
popd
}
+function wait_for_node {
+ function retry_wait {
+ kubectl --kubeconfig /etc/kubernetes/admin.conf get nodes |awk '{print $1}' |grep -q $(hostname)
+ }
+ retry $NODE_DEPLOYMENT_RETRIES "The node didn't come up." retry_wait
+}
+
+
function join_node {
env -i $(docker swarm join-token $1 |grep 'docker swarm join' | xargs)
}
@@ -166,6 +175,7 @@
download_bundles
rm_ucp_config
install_kubectl
+ wait_for_node
;;
master)
prepare_network
@@ -174,6 +184,7 @@
download_bundles
join_node manager
install_kubectl
+ wait_for_node
;;
worker)
prepare_network
@@ -181,6 +192,8 @@
install_docker_ce
download_bundles
join_node worker
+ install_kubectl
+ wait_for_node
;;
*)
echo "Usage: $0 {ucp|master|worker}"
diff --git a/de/heat-templates/srv-group.yaml b/de/heat-templates/srv-group.yaml
index e654bb2..a2cf684 100644
--- a/de/heat-templates/srv-group.yaml
+++ b/de/heat-templates/srv-group.yaml
@@ -23,13 +23,33 @@
private_floating_network:
type: string
description: ID of network that will be used for floating in nested openstack
- user_data:
- type: string
+ boot_timeout:
+ type: number
+ description: Boot timeout for instance
+ default: 1200
metadata:
type: json
+ node_type:
+ type: string
+ ucp_master_host:
+ type: string
+ default: ''
resources:
+ software_config:
+ type: OS::Heat::SoftwareConfig
+ properties:
+ group: ungrouped
+ config:
+ str_replace:
+ template: { get_file: ./scripts/instance_boot.sh }
+ params:
+ $node_type: { get_param: node_type }
+ $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
+ $ucp_license_key: { get_file: ./scripts/license.lic }
+ $ucp_master_host: { get_param: ucp_master_host }
+
server:
type: OS::Nova::Server
properties:
@@ -41,7 +61,7 @@
- port: { get_resource: server_port }
- network: { get_param: private_floating_network }
user_data_format: RAW
- user_data: { get_param: user_data }
+ user_data: { get_resource: software_config }
metadata: { get_param: metadata }
server_port:
@@ -58,6 +78,13 @@
floating_network_id: { get_param: public_net_id }
port_id: { get_resource: server_port }
+ wait_handle:
+ type: OS::Heat::WaitConditionHandle
+ wait_condition:
+ type: OS::Heat::WaitCondition
+ properties:
+ handle: { get_resource: wait_handle }
+ timeout: { get_param: boot_timeout }
outputs:
server_private_ip:
diff --git a/de/heat-templates/top.yaml b/de/heat-templates/top.yaml
index 7051873..f6a5e6f 100644
--- a/de/heat-templates/top.yaml
+++ b/de/heat-templates/top.yaml
@@ -84,17 +84,6 @@
enable_dhcp: false
gateway_ip: ~
- ucp_config:
- type: OS::Heat::SoftwareConfig
- properties:
- group: ungrouped
- config:
- str_replace:
- template: { get_file: ./scripts/instance_boot.sh }
- params:
- $node_type: ucp
- $wait_condition_notify: { get_attr: [ ucp_wait_handle, curl_cli ] }
- $ucp_license_key: { get_file: ./scripts/license.lic }
ucp:
depends_on: router_iface
type: ./srv-group.yaml
@@ -106,34 +95,13 @@
private_net_id: { get_resource: network }
private_subnet_id: { get_resource: subnet }
private_floating_network: { get_resource: private_floating_network }
- user_data: { get_resource: ucp_config }
metadata: {"role":"ucp"}
-
- ucp_wait_handle:
- type: OS::Heat::WaitConditionHandle
- ucp_wait_condition:
- type: OS::Heat::WaitCondition
- properties:
- handle: { get_resource: ucp_wait_handle }
- timeout: { get_param: ucp_boot_timeout }
-
- master_config:
- type: OS::Heat::SoftwareConfig
- properties:
- group: ungrouped
- config:
- str_replace:
- template: { get_file: ./scripts/instance_boot.sh }
- params:
- $node_type: master
- $wait_condition_notify: { get_attr: [ ucp_wait_handle, curl_cli ] }
- $ucp_license_key: { get_file: ./scripts/license.lic }
- $ucp_master_host: { get_attr: [ucp, server_private_ip] }
+ node_type: "ucp"
masters:
type: OS::Heat::ResourceGroup
depends_on:
- - ucp_wait_condition
+ - ucp
properties:
count: { get_param: masters_size }
resource_def:
@@ -146,26 +114,14 @@
private_net_id: { get_resource: network }
private_subnet_id: { get_resource: subnet }
private_floating_network: { get_resource: private_floating_network }
- user_data: { get_resource: master_config }
metadata: {"role":"master"}
-
- worker_config:
- type: OS::Heat::SoftwareConfig
- properties:
- group: ungrouped
- config:
- str_replace:
- template: { get_file: ./scripts/instance_boot.sh }
- params:
- $node_type: worker
- $wait_condition_notify: { get_attr: [ ucp_wait_handle, curl_cli ] }
- $ucp_license_key: { get_file: ./scripts/license.lic }
- $ucp_master_host: { get_attr: [ucp, server_private_ip] }
+ node_type: "master"
+ ucp_master_host: { get_attr: [ucp, server_private_ip] }
workers:
type: OS::Heat::ResourceGroup
depends_on:
- - ucp_wait_condition
+ - ucp
properties:
count: { get_param: worker_size }
resource_def:
@@ -178,12 +134,14 @@
private_net_id: { get_resource: network }
private_subnet_id: { get_resource: subnet }
private_floating_network: { get_resource: private_floating_network }
- user_data: { get_resource: worker_config }
metadata: { get_param: worker_metadata}
+ node_type: "worker"
+ ucp_master_host: { get_attr: [ucp, server_private_ip] }
+
cmps:
type: OS::Heat::ResourceGroup
depends_on:
- - ucp_wait_condition
+ - ucp
properties:
count: { get_param: cmp_size }
resource_def:
@@ -196,12 +154,14 @@
private_net_id: { get_resource: network }
private_subnet_id: { get_resource: subnet }
private_floating_network: { get_resource: private_floating_network }
- user_data: { get_resource: worker_config }
metadata: { get_param: cmp_metadata}
+ node_type: "worker"
+ ucp_master_host: { get_attr: [ucp, server_private_ip] }
+
gtws:
type: OS::Heat::ResourceGroup
depends_on:
- - ucp_wait_condition
+ - ucp
properties:
count: { get_param: gtw_size }
resource_def:
@@ -214,8 +174,9 @@
private_net_id: { get_resource: network }
private_subnet_id: { get_resource: subnet }
private_floating_network: { get_resource: private_floating_network }
- user_data: { get_resource: worker_config }
metadata: { get_param: gtw_metadata}
+ node_type: "worker"
+ ucp_master_host: { get_attr: [ucp, server_private_ip] }
outputs: