Allow to deploy with custom kubernetes runtime

Related-Prod: PRODX-42391

Change-Id: I20c6cc7a8f5a6e19cea66e158a781a9f7394187d
diff --git a/de/heat-templates/fragments/SrvInstancesBM.yaml b/de/heat-templates/fragments/SrvInstancesBM.yaml
index 3cb80ee..196f985 100644
--- a/de/heat-templates/fragments/SrvInstancesBM.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBM.yaml
@@ -48,6 +48,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   docker_ucp_image:
@@ -79,6 +81,7 @@
             $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $docker_ee_release: { get_param: docker_ee_release }
             $ucp_master_host: { get_param: ucp_master_host }
             $docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
index ad2a702..ded0ea3 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
@@ -72,6 +72,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   hardware_metadata:
@@ -96,6 +98,7 @@
             $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $docker_ee_release: { get_param: docker_ee_release }
             $binary_base_url: { get_param: binary_base_url }
             $ucp_master_host: { get_param: ucp_master_host }
diff --git a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
index 0c92f7a..6791060 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
@@ -86,6 +86,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   hardware_metadata:
@@ -117,6 +119,7 @@
             $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $docker_ee_release: { get_param: docker_ee_release }
             $ucp_master_host: { get_param: ucp_master_host }
             $docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/SrvInstancesVM.yaml b/de/heat-templates/fragments/SrvInstancesVM.yaml
index 374ec92..fd83b44 100644
--- a/de/heat-templates/fragments/SrvInstancesVM.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVM.yaml
@@ -47,6 +47,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   docker_ucp_image:
@@ -101,6 +103,7 @@
             $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $docker_ee_release: { get_param: docker_ee_release }
             $ucp_master_host: { get_param: ucp_master_host }
             $node_metadata: { get_param: metadata }
diff --git a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
index 320b564..cc1099b 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
@@ -71,6 +71,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   hardware_metadata:
@@ -110,6 +112,7 @@
             $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $docker_ee_release: { get_param: docker_ee_release }
             $ucp_master_host: { get_param: ucp_master_host }
             $docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
index dff89f5..e7834e2 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
@@ -85,6 +85,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   hardware_metadata:
@@ -143,6 +145,7 @@
             $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $docker_ee_release: { get_param: docker_ee_release }
             $ucp_master_host: { get_param: ucp_master_host }
             $docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/fragments/multirack/CentralSite.yaml b/de/heat-templates/fragments/multirack/CentralSite.yaml
index c8b98b2..e56b413 100644
--- a/de/heat-templates/fragments/multirack/CentralSite.yaml
+++ b/de/heat-templates/fragments/multirack/CentralSite.yaml
@@ -23,6 +23,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   docker_ucp_image:
@@ -170,6 +172,7 @@
       docker_ee_url: { get_param: docker_ee_url }
       docker_ee_release: { get_param: docker_ee_release }
       docker_ee_packages: { get_param: docker_ee_packages }
+      kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
       docker_ucp_image: { get_param: docker_ucp_image}
       docker_default_address_pool: { get_param: docker_default_address_pool }
       node_type: "ucp"
@@ -201,6 +204,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           node_type: "worker"
@@ -242,6 +246,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           node_type: "worker"
diff --git a/de/heat-templates/fragments/multirack/Rack.yaml b/de/heat-templates/fragments/multirack/Rack.yaml
index cfbd1d7..16dc350 100644
--- a/de/heat-templates/fragments/multirack/Rack.yaml
+++ b/de/heat-templates/fragments/multirack/Rack.yaml
@@ -27,6 +27,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   docker_ucp_image:
@@ -84,6 +86,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           node_type: "worker"
           kubernetes_installer: { get_param: kubernetes_installer }
diff --git a/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml b/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml
index dc437b2..ac28eed 100644
--- a/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml
+++ b/de/heat-templates/fragments/multirack/SrvInstancesVM.yaml
@@ -49,6 +49,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ucp_image:
     type: string
     default: 'docker/ucp:3.2.4'
@@ -75,6 +77,7 @@
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_release: { get_param: docker_ee_release }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $ucp_master_host: { get_param: ucp_master_host }
             $docker_ucp_image: { get_param: docker_ucp_image }
             $node_metadata: { get_param: metadata }
diff --git a/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml b/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml
index 288d03c..04f5aaf 100644
--- a/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml
+++ b/de/heat-templates/fragments/multirack/SrvInstancesVMCeph.yaml
@@ -46,6 +46,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   docker_ucp_image:
@@ -104,6 +106,7 @@
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_release: { get_param: docker_ee_release }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $ucp_master_host: { get_param: ucp_master_host }
             $docker_ucp_image: { get_param: docker_ucp_image }
             $node_metadata: { get_param: metadata }
diff --git a/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml b/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml
index 8efb30a..118e376 100644
--- a/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml
+++ b/de/heat-templates/fragments/multirack/SrvInstancesVMRack.yaml
@@ -39,6 +39,8 @@
     type: string
   docker_ee_packages:
     type: string
+  kubernetes_container_runtime:
+    type: string
   docker_ee_url:
     type: string
   docker_ucp_image:
@@ -68,6 +70,7 @@
             $wait_condition_notify: { get_attr: [ wait_handle, curl_cli ] }
             $docker_ee_url: { get_param: docker_ee_url }
             $docker_ee_packages: { get_param: docker_ee_packages }
+            $kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
             $docker_ee_release: { get_param: docker_ee_release }
             $ucp_master_host: { get_param: ucp_master_host }
             $docker_ucp_image: { get_param: docker_ucp_image }
diff --git a/de/heat-templates/scripts/functions.sh b/de/heat-templates/scripts/functions.sh
index 404c0bc..7e089cb 100644
--- a/de/heat-templates/scripts/functions.sh
+++ b/de/heat-templates/scripts/functions.sh
@@ -115,7 +115,7 @@
 function install_pkgs {
     local pkgs="$@"
     function _retry_install {
-        apt update && DEBIAN_FRONTEND=noninteractive apt install -y $pkgs
+        apt update && DEBIAN_FRONTEND=noninteractive apt install -o DPkg::Options::=--force-confdef -y $pkgs
     }
     retry 10 "Labeling node failed" _retry_install
 }
@@ -221,6 +221,11 @@
         curl --retry 6 --retry-delay 5 -fsSL "${DOCKER_EE_URL}/gpg" | sudo apt-key add -
         add-apt-repository "deb [arch=amd64] ${DOCKER_EE_URL}/ ${OS_CODENAME} ${DOCKER_EE_RELEASE}"
         install_pkgs ${DOCKER_EE_PACKAGES}
+        if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+            NERDCTL_VERSION=1.7.6
+            curl --retry 6 --retry-delay 5 -fsSL https://github.com/containerd/nerdctl/releases/download/v${NERDCTL_VERSION}/nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz --output nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz
+            tar zxvf nerdctl-${NERDCTL_VERSION}-linux-amd64.tar.gz -C /usr/local/bin
+        fi
     }
     retry 10 "Failed to install docker" install_retry
 }
@@ -241,11 +246,37 @@
 }
 EOF
 
+    if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+        mkdir -p /etc/containerd/
+cat <<EOF> /etc/containerd/config.toml
+disabled_plugins = []
+
+#root = "/var/lib/containerd"
+#state = "/run/containerd"
+#subreaper = true
+#oom_score = 0
+
+#[grpc]
+#  address = "/run/containerd/containerd.sock"
+#  uid = 0
+#  gid = 0
+
+#[debug]
+#  address = "/run/containerd/debug.sock"
+#  uid = 0
+#  gid = 0
+#  level = "info"
+EOF
+    fi
+
 }
 
 function pull_image {
     local image="$1"
     function pull_image_retry {
+        if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+            nerdctl --address  unix:///run/containerd/containerd.sock -n k8s.io pull $image
+        fi
         docker pull ${image}
     }
     retry 10 "Can't pool docker image $image" pull_image_retry
@@ -259,6 +290,9 @@
     local image_prefix=${DOCKER_UCP_IMAGE%/*}
     for image in $(docker container run --rm --name ucp -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_UCP_IMAGE} images --list); do
         pull_image "${image_prefix}/${image##*/}"
+        if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+            nerdctl --address  unix:///run/containerd/containerd.sock -n k8s.io  tag ${image_prefix}/${image##*/} mirantis/${image##*/}
+        fi
         docker tag ${image_prefix}/${image##*/} mirantis/${image##*/}
     done
 }
@@ -396,6 +430,10 @@
         secure_overlay="secure_overlay = true
     calico_vxlan = false"
     fi
+    kubelet_custom_flags=""
+    if [[ $KUBERNETES_CONTAINER_RUNTIME == "crio" ]]; then
+        kubelet_custom_flags='custom_kubelet_flags=["--container-runtime-endpoint=unix:///run/containerd/containerd.sock"]'
+    fi
     if docker config ls | grep com.docker.ucp.config ; then
         echo "Config com.docker.ucp.config already exists"
     else
@@ -408,6 +446,7 @@
     calico_vxlan_port = \"$UCP_DOCKER_CALICO_VXLAN_PORT\"
     ${max_pods}
     ${secure_overlay}
+    ${kubelet_custom_flags}
 " | docker config create com.docker.ucp.config -
     fi
 }
diff --git a/de/heat-templates/scripts/launch.sh b/de/heat-templates/scripts/launch.sh
index 09c4068..6f5704e 100644
--- a/de/heat-templates/scripts/launch.sh
+++ b/de/heat-templates/scripts/launch.sh
@@ -53,6 +53,7 @@
 CINDER_LVM_LOOP_DEVICE_SIZE=${CINDER_LVM_LOOP_DEVICE_SIZE:-$cinder_lvm_loop_device_size}
 SECURE_OVERLAY_ENABLED=${SECURE_OVERLAY_ENABLED:-$secure_overlay_enabled}
 KUBECTL_VERSION=${KUBECTL_VERSION:-$kubectl_version}
+KUBERNETES_CONTAINER_RUNTIME=${KUBERNETES_CONTAINER_RUNTIME:-$kubernetes_container_runtime}
 
 DEVOPS_UTILS_REFSPEC=${DEVOPS_UTILS_REFSPEC:-$devops_utils_refspec}
 
diff --git a/de/heat-templates/top.yaml b/de/heat-templates/top.yaml
index e4dccf7..56f939e 100644
--- a/de/heat-templates/top.yaml
+++ b/de/heat-templates/top.yaml
@@ -320,6 +320,9 @@
   docker_ee_packages:
     type: string
     default: "docker-ee"
+  kubernetes_container_runtime:
+    type: string
+    default: 'docker'
   docker_ucp_image:
     type: string
     default: 'docker/ucp:3.2.4'
@@ -542,6 +545,7 @@
       docker_ee_url: { get_param: docker_ee_url }
       docker_ee_release: { get_param: docker_ee_release }
       docker_ee_packages: { get_param: docker_ee_packages }
+      kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
       docker_ucp_image: { get_param: docker_ucp_image}
       docker_ucp_swarm_data_port: { get_param: docker_ucp_swarm_data_port }
       docker_default_address_pool: { get_param: docker_default_address_pool }
@@ -585,6 +589,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -621,6 +626,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -675,6 +681,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -729,6 +736,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -784,6 +792,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -829,6 +838,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -874,6 +884,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -927,6 +938,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -978,6 +990,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -1030,6 +1043,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }
@@ -1075,6 +1089,7 @@
           docker_ee_url: { get_param: docker_ee_url }
           docker_ee_release: { get_param: docker_ee_release }
           docker_ee_packages: { get_param: docker_ee_packages }
+          kubernetes_container_runtime: { get_param: kubernetes_container_runtime }
           docker_ucp_image: { get_param: docker_ucp_image}
           docker_default_address_pool: { get_param: docker_default_address_pool }
           binary_base_url: { get_param: binary_base_url }