Install cron for disabling calico offloading
After switch to ucp 3.4.5 and kernel 5.4 issue [1]
started to reproduce each time on node reboot or adding.
[1] https://github.com/kubernetes/kubernetes/issues/96868
Related-Prod: https://mirantis.jira.com/browse/PRODX-20052
Change-Id: I63c879947889adb24f8ef7e3e3fb243deef07841
diff --git a/de/heat-templates/fragments/SrvInstancesBM.yaml b/de/heat-templates/fragments/SrvInstancesBM.yaml
index 5599d11..c3ffa7d 100644
--- a/de/heat-templates/fragments/SrvInstancesBM.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBM.yaml
@@ -98,6 +98,10 @@
owner: "root:root"
permissions: "0644"
content: { get_param: hardware_metadata}
+ - path: /usr/sbin/calico_disable_offloading.sh
+ owner: "root:root"
+ permissions: "0755"
+ content: {get_file: ../scripts/calico_disable_offloading.sh}
install_config_agent:
type: "OS::Heat::MultipartMime"
diff --git a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
index 7c6a76d..c9a337c 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCeph.yaml
@@ -122,6 +122,10 @@
owner: "root:root"
permissions: "0644"
content: { get_param: hardware_metadata}
+ - path: /usr/sbin/calico_disable_offloading.sh
+ owner: "root:root"
+ permissions: "0755"
+ content: {get_file: ../scripts/calico_disable_offloading.sh}
install_config_agent:
type: "OS::Heat::MultipartMime"
diff --git a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
index 4fa2615..c8bdd4b 100644
--- a/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesBMCephOSD.yaml
@@ -148,6 +148,10 @@
owner: "root:root"
permissions: "0644"
content: { get_param: hardware_metadata}
+ - path: /usr/sbin/calico_disable_offloading.sh
+ owner: "root:root"
+ permissions: "0755"
+ content: {get_file: ../scripts/calico_disable_offloading.sh}
install_config_agent:
type: "OS::Heat::MultipartMime"
diff --git a/de/heat-templates/fragments/SrvInstancesVM.yaml b/de/heat-templates/fragments/SrvInstancesVM.yaml
index d005f8d..93372b0 100644
--- a/de/heat-templates/fragments/SrvInstancesVM.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVM.yaml
@@ -114,6 +114,10 @@
owner: "root:root"
permissions: "0644"
content: { get_param: hardware_metadata}
+ - path: /usr/sbin/calico_disable_offloading.sh
+ owner: "root:root"
+ permissions: "0755"
+ content: {get_file: ../scripts/calico_disable_offloading.sh}
install_config_agent:
type: "OS::Heat::MultipartMime"
diff --git a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
index 17b2593..3fc96d5 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCeph.yaml
@@ -130,6 +130,10 @@
owner: "root:root"
permissions: "0644"
content: { get_param: hardware_metadata}
+ - path: /usr/sbin/calico_disable_offloading.sh
+ owner: "root:root"
+ permissions: "0755"
+ content: {get_file: ../scripts/calico_disable_offloading.sh}
install_config_agent:
type: "OS::Heat::MultipartMime"
diff --git a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
index f9c2ce9..d66dae9 100644
--- a/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
+++ b/de/heat-templates/fragments/SrvInstancesVMCephOSD.yaml
@@ -172,6 +172,10 @@
owner: "root:root"
permissions: "0644"
content: { get_param: hardware_metadata}
+ - path: /usr/sbin/calico_disable_offloading.sh
+ owner: "root:root"
+ permissions: "0755"
+ content: {get_file: ../scripts/calico_disable_offloading.sh}
install_config_agent:
type: "OS::Heat::MultipartMime"
diff --git a/de/heat-templates/scripts/calico_disable_offloading.sh b/de/heat-templates/scripts/calico_disable_offloading.sh
new file mode 100644
index 0000000..26ce61b
--- /dev/null
+++ b/de/heat-templates/scripts/calico_disable_offloading.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# script is taken from https://gerrit.mcp.mirantis.com/plugins/gitiles/kubernetes/lcm-ansible/+/refs/heads/master/roles/kubernetes-postinstall/templates/calico_disable_offloading.sh
+# This script apply a workaround for a bug encountered on Kubernetes with vxlan
+# and iptables >= 1.6.2.
+# You can find more details on this bug here:
+# https://github.com/kubernetes/kubernetes/issues/96868
+# https://github.com/projectcalico/calico/issues/3145
+#
+# The workaround is to disable offloading on vxlan interface
+
+nic_name='vxlan.calico'
+if [ -e /etc/system-release ]; then
+ #RHEL
+ ethtool_cmd="/usr/sbin/ethtool"
+ ip_cmd="/usr/sbin/ip"
+else
+ #Ubuntu
+ ethtool_cmd="/sbin/ethtool"
+ ip_cmd="/sbin/ip"
+fi
+
+_ethtool() {
+ $ethtool_cmd "$@"
+}
+
+_ip () {
+ $ip_cmd "$@"
+}
+
+is_nic_available() {
+ _ip a show dev $nic_name > /dev/null 2>&1
+}
+
+deactivate_offloading() {
+ echo "Disabling offloading for ${nic_name}"
+ _ethtool --offload $nic_name rx off tx off > /dev/null
+}
+
+is_offloading_disabled() {
+ # Return an error if at least one offload is enabled (rx or tx)
+ if _ethtool --show-offload $nic_name | grep -E '^.x-checksumming:' | grep -q ': on'; then
+ return 1
+ else
+ return 0
+ fi
+}
+
+if is_nic_available; then
+ if ! is_offloading_disabled; then
+ deactivate_offloading
+ exit $?
+ fi
+fi
\ No newline at end of file
diff --git a/de/heat-templates/scripts/instance_boot.sh b/de/heat-templates/scripts/instance_boot.sh
index f81ac55..68a4a62 100644
--- a/de/heat-templates/scripts/instance_boot.sh
+++ b/de/heat-templates/scripts/instance_boot.sh
@@ -975,6 +975,12 @@
setup_evpn
}
+function cron_disable_calico_offloading {
+ cat << EOF >> /etc/cron.d/disable_calico_offloading
+* * * * * root /usr/sbin/calico_disable_offloading.sh 2>&1 | /usr/bin/logger -t calico_disable_offloading
+EOF
+}
+
# Exit on any errors
function handle_exit {
if [ $? != 0 ] ; then
@@ -1024,6 +1030,7 @@
disable_master_taint
collect_interfaces_metadata
fi
+ cron_disable_calico_offloading
;;
master)
nested_virt_config
@@ -1053,6 +1060,7 @@
disable_iptables_for_bridges
fi
collect_interfaces_metadata
+ cron_disable_calico_offloading
;;
worker)
if [[ "${CONFIGURE_HUGE_PAGES}" == true ]]; then
@@ -1091,6 +1099,7 @@
fi
collect_interfaces_metadata
configure_lvm
+ cron_disable_calico_offloading
;;
spare)
prepare_metadata_files
@@ -1113,6 +1122,7 @@
if [[ "${KUBERNETES_INSTALLER}" == "ucp" ]]; then
disable_iptables_for_bridges
fi
+ cron_disable_calico_offloading
;;
frr)
prepare_metadata_files
@@ -1134,6 +1144,7 @@
configure_contrack
disable_iptables_for_bridges
install_frr
+ cron_disable_calico_offloading
;;
*)
echo "Usage: $0 {ucp|master|worker}"