Allow to deploy with custom kubernetes runtime
Related-Prod: PRODX-42391
Change-Id: I20c6cc7a8f5a6e19cea66e158a781a9f7394187d
diff --git a/de/heat-templates/fragments/SrvInstancesBM.yaml b/de/heat-templates/fragments/SrvInstancesBM.yaml
index 3cb80ee..196f985 100644
--- a/de/heat-templates/fragments/SrvInstancesBM.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBM.yaml
@@ -48,6 +48,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
docker_ucp_image:
@@ -79,6 +81,7 @@
$wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$docker_ee_release: { get_param: docker_ee_release }
$ucp_master_host: { get_param: ucp_master_host }
$docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
index ad2a702..ded0ea3 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
@@ -72,6 +72,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
hardware_metadata:
@@ -96,6 +98,7 @@
$wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$docker_ee_release: { get_param: docker_ee_release }
$binary_base_url: { get_param: binary_base_url }
$ucp_master_host: { get_param: ucp_master_host }
diff --git a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
index 0c92f7a..6791060 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
@@ -86,6 +86,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
hardware_metadata:
@@ -117,6 +119,7 @@
$wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$docker_ee_release: { get_param: docker_ee_release }
$ucp_master_host: { get_param: ucp_master_host }
$docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/SrvInstancesVM.yaml b/de/heat-templates/fragments/SrvInstancesVM.yaml
index 374ec92..fd83b44 100644
--- a/de/heat-templates/fragments/SrvInstancesVM.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVM.yaml
@@ -47,6 +47,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
docker_ucp_image:
@@ -101,6 +103,7 @@
$wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$docker_ee_release: { get_param: docker_ee_release }
$ucp_master_host: { get_param: ucp_master_host }
$node_metadata: { get_param: metadata }
diff --git a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
index 320b564..cc1099b 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
@@ -71,6 +71,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
hardware_metadata:
@@ -110,6 +112,7 @@
$wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$docker_ee_release: { get_param: docker_ee_release }
$ucp_master_host: { get_param: ucp_master_host }
$docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
index dff89f5..e7834e2 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
@@ -85,6 +85,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
hardware_metadata:
@@ -143,6 +145,7 @@
$wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$docker_ee_release: { get_param: docker_ee_release }
$ucp_master_host: { get_param: ucp_master_host }
$docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/multirack/CentralSite.yaml b/de/heat-templates/fragments/multirack/CentralSite.yaml
index c8b98b2..e56b413 100644
--- a/de/heat-templates/fragments/multirack/CentralSite.yaml
+++ b/de/heat-templates/fragments/multirack/CentralSite.yaml
@@ -23,6 +23,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
docker_ucp_image:
@@ -170,6 +172,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
node_type: "ucp"
@@ -201,6 +204,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
node_type: "worker"
@@ -242,6 +246,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
node_type: "worker"
diff --git a/de/heat-templates/fragments/multirack/Rack.yaml b/de/heat-templates/fragments/multirack/Rack.yaml
index cfbd1d7..16dc350 100644
--- a/de/heat-templates/fragments/multirack/Rack.yaml
+++ b/de/heat-templates/fragments/multirack/Rack.yaml
@@ -27,6 +27,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
docker_ucp_image:
@@ -84,6 +86,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
node_type: "worker"
kubernetes_installer: { get_param: kubernetes_installer }
diff --git a/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml b/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml
index dc437b2..ac28eed 100644
--- a/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml
+++ b/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml
@@ -49,6 +49,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ucp_image:
type: string
default: 'docker/ucp:3.2.4'
@@ -75,6 +77,7 @@
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_release: { get_param: docker_ee_release }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$ucp_master_host: { get_param: ucp_master_host }
$docker_ucp_image: { get_param: docker_ucp_image }
$node_metadata: { get_param: metadata }
diff --git a/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml b/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml
index 288d03c..04f5aaf 100644
--- a/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml
+++ b/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml
@@ -46,6 +46,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
docker_ucp_image:
@@ -104,6 +106,7 @@
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_release: { get_param: docker_ee_release }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$ucp_master_host: { get_param: ucp_master_host }
$docker_ucp_image: { get_param: docker_ucp_image }
$node_metadata: { get_param: metadata }
diff --git a/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml b/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml
index 8efb30a..118e376 100644
--- a/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml
+++ b/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml
@@ -39,6 +39,8 @@
type: string
docker_ee_packages:
type: string
+ kubernetes_container_runtime:
+ type: string
docker_ee_url:
type: string
docker_ucp_image:
@@ -68,6 +70,7 @@
$wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
$docker_ee_url: { get_param: docker_ee_url }
$docker_ee_packages: { get_param: docker_ee_packages }
+ $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
$docker_ee_release: { get_param: docker_ee_release }
$ucp_master_host: { get_param: ucp_master_host }
$docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/scripts/functions.sh b/de/heat-templates/scripts/functions.sh
index 404c0bc..7e089cb 100644
--- a/de/heat-templates/scripts/functions.sh
+++ b/de/heat-templates/scripts/functions.sh
@@ -115,7 +115,7 @@
function install_pkgs {
local pkgs="$@"
function _retry_install {
- apt update && DEBIAN_FRONTEND=noninteractive apt install -y $pkgs
+ apt update && DEBIAN_FRONTEND=noninteractive apt install -o DPkg::Options::=--force-confdef -y $pkgs
}
retry 10 "Labeling node failed" _retry_install
}
@@ -221,6 +221,11 @@
curl --retry 6 --retry-delay 5 -fsSL "${DOCKER_EE_URL}/gpg" | sudo apt-key add -
add-apt-repository "deb [arch=amd64] ${DOCKER_EE_URL}/ ${OS_CODENAME} ${DOCKER_EE_RELEASE}"
install_pkgs ${DOCKER_EE_PACKAGES}
+ if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+ NERDCTL_VERSION=1.7.6
+ curl --retry 6 --retry-delay 5 -fsSL https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz --output nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz
+ tar zxvf nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz -C /usr/local/bin
+ fi
}
retry 10 "Failed to install docker" install_retry
}
@@ -241,11 +246,37 @@
}
EOF
+ if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+ mkdir -p /etc/containerd/
+cat <<EOF> /etc/containerd/config.toml
+disabled_plugins = []
+
+#root = "/var/lib/containerd"
+#state = "/run/containerd"
+#subreaper = true
+#oom_score = 0
+
+#[grpc]
+# address = "/run/containerd/containerd.sock"
+# uid = 0
+# gid = 0
+
+#[debug]
+# address = "/run/containerd/debug.sock"
+# uid = 0
+# gid = 0
+# level = "info"
+EOF
+ fi
+
}
function pull_image {
local image="$1"
function pull_image_retry {
+ if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+ nerdctl --address unix:///run/containerd/containerd.sock -n k8s.io pull $image
+ fi
docker pull ${image}
}
retry 10 "Can't pool docker image $image" pull_image_retry
@@ -259,6 +290,9 @@
local image_prefix=${DOCKER_UCP_IMAGE%/*}
for image in $(docker container run --rm --name ucp -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_UCP_IMAGE} images --list); do
pull_image "${image_prefix}/${image##*/}"
+ if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+ nerdctl --address unix:///run/containerd/containerd.sock -n k8s.io tag ${image_prefix}/${image##*/} mirantis/${image##*/}
+ fi
docker tag ${image_prefix}/${image##*/} mirantis/${image##*/}
done
}
@@ -396,6 +430,10 @@
secure_overlay="secure_overlay = true
calico_vxlan = false"
fi
+ kubelet_custom_flags=""
+ if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+ kubelet_custom_flags='custom_kubelet_flags=["--container-runtime-endpoint=unix:///run/containerd/containerd.sock"]'
+ fi
if docker config ls | grep com.docker.ucp.config ; then
echo "Config com.docker.ucp.config already exists"
else
@@ -408,6 +446,7 @@
calico_vxlan_port = \"$UCP_DOCKER_CALICO_VXLAN_PORT\"
${max_pods}
${secure_overlay}
+ ${kubelet_custom_flags}
" | docker config create com.docker.ucp.config -
fi
}
diff --git a/de/heat-templates/scripts/launch.sh b/de/heat-templates/scripts/launch.sh
index 09c4068..6f5704e 100644
--- a/de/heat-templates/scripts/launch.sh
+++ b/de/heat-templates/scripts/launch.sh
@@ -53,6 +53,7 @@
CINDER_LVM_LOOP_DEVICE_SIZE=${CINDER_LVM_LOOP_DEVICE_SIZE:-$cinder_lvm_loop_device_size}
SECURE_OVERLAY_ENABLED=${SECURE_OVERLAY_ENABLED:-$secure_overlay_enabled}
KUBECTL_VERSION=${KUBECTL_VERSION:-$kubectl_version}
+KUBERNETES_CONTAINER_RUNTIME=${KUBERNETES_CONTAINER_RUNTIME:-$kubernetes_container_runtime}
DEVOPS_UTILS_REFSPEC=${DEVOPS_UTILS_REFSPEC:-$devops_utils_refspec}
diff --git a/de/heat-templates/top.yaml b/de/heat-templates/top.yaml
index e4dccf7..56f939e 100644
--- a/de/heat-templates/top.yaml
+++ b/de/heat-templates/top.yaml
@@ -320,6 +320,9 @@
docker_ee_packages:
type: string
default: "docker-ee"
+ kubernetes_container_runtime:
+ type: string
+ default: 'docker'
docker_ucp_image:
type: string
default: 'docker/ucp:3.2.4'
@@ -542,6 +545,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_ucp_swarm_data_port: { get_param: docker_ucp_swarm_data_port }
docker_default_address_pool: { get_param: docker_default_address_pool }
@@ -585,6 +589,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -621,6 +626,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -675,6 +681,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -729,6 +736,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -784,6 +792,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -829,6 +838,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -874,6 +884,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -927,6 +938,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -978,6 +990,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -1030,6 +1043,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }
@@ -1075,6 +1089,7 @@
docker_ee_url: { get_param: docker_ee_url }
docker_ee_release: { get_param: docker_ee_release }
docker_ee_packages: { get_param: docker_ee_packages }
+ kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
docker_ucp_image: { get_param: docker_ucp_image}
docker_default_address_pool: { get_param: docker_default_address_pool }
binary_base_url: { get_param: binary_base_url }