Add verification of Calico policy setting during upgrade
In order to perform Calico upgrade properly, it's required to
determine whether Calico policy is enabled on the environment already
and to align policy settings for Calico v3.x (that will be running
after upgrade) accordingly.
The following actions are performed:
1. Check CNI config for policy settings.
2. Check Kubernetes addons for calico_policy addon.
3. Check whether kube-controllers pod is running.
4. Remove v2.6.x calico_policy addon on masters to not interfere with
v3.x kube-controllers.
5. Check consistency between Calico policy entities detected during
pp.1-3. Show a warning and do not touch policy turn on/off flag in
case of non-consistency.
6. Check Calico policy setting for v3.x (that will be effective after
upgrade) if Calico policy was enabled for currently running
environment (so for Calico v2.6.x) already.
7. Issue a warning message if Calico policy is enabled currently but
it is going to be disabled after upgrade.
It's safe to enable Calico policy any time, but it may be unsafe to
disable it. So, no need to disable Calico policy for v3.x if it's
not in use currently. But if Calico policy is in use already, it
should be enabled after upgrade as well.
Related-Prod: PROD-25161 (PROD:25161)
Change-Id: I35634adef52c29037b0184440aae9cbb4872ffd5
diff --git a/k8s-upgrade-pipeline.groovy b/k8s-upgrade-pipeline.groovy
index 980aa2f..f9a16e0 100644
--- a/k8s-upgrade-pipeline.groovy
+++ b/k8s-upgrade-pipeline.groovy
@@ -24,6 +24,8 @@
* CALICO_UPGRADE_VERSION Version of "calico-upgrade" utility to be used ("v1.0.5" for Calico v3.1.3 target).
*
**/
+import groovy.json.JsonSlurper
+
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
@@ -323,6 +325,85 @@
}
}
+def checkCalicoPolicySetting(pepperEnv, target) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
+
+ stage("Checking of Calico network policy setting") {
+ // check Calico policy enabled
+ def cniPolicy = false
+ def addonsPolicy = false
+ def kubeCtrlRunning = false
+
+ // check CNI config
+ def cniCfgResult = salt.cmdRun(
+ pepperEnv, target, "cat /etc/cni/net.d/10-calico.conf"
+ )['return'][0].values()[0].toString()
+ def cniCfg = new JsonSlurper().parseText(cniCfgResult)
+ if (cniCfg.get("policy") != null) {
+ if (cniCfg["policy"].get("type") == "k8s") {
+ cniPolicy = true
+ } else {
+ common.warningMsg("Calico policy type is unknown or not set.")
+ }
+ }
+
+ // check k8s addons
+ def addonsResult = salt.cmdRun(
+ pepperEnv, target, "ls /etc/kubernetes/addons"
+ )['return'][0].values()[0].toString()
+ if (addonsResult.contains("calico_policy")) {
+ addonsPolicy = true
+ }
+
+ // check kube-controllers is running
+ def kubeCtrlResult = salt.cmdRun(
+ pepperEnv, target, "kubectl get pod -n kube-system --selector=k8s-app=calico-kube-controllers"
+ )['return'][0].values()[0].toString()
+ if (kubeCtrlResult.contains("Running")) {
+ kubeCtrlRunning = true
+ }
+
+ // It's safe to enable Calico policy any time, but it may be unsafe to disable it.
+ // So, no need to disable Calico policy for v3.x if it's not in use currently.
+ // But if Calico policy is in use already, it should be enabled after upgrade as well.
+
+ // check for consistency
+ if ((cniPolicy != addonsPolicy) || (addonsPolicy != kubeCtrlRunning)) {
+ caution = "ATTENTION. Calico policy setting cannot be determined reliably (enabled in CNI config: ${cniPolicy}, " +
+ "presence in k8s addons: ${addonsPolicy}, kube-controllers is running: ${kubeCtrlRunning})."
+ currentBuild.description += "<br><b>${caution}</b><br><br>"
+ common.warningMsg(caution)
+ } else {
+ common.infoMsg("Current Calico policy state is detected as: ${cniPolicy}")
+ if (cniPolicy) {
+ // Calico policy is in use. Check policy setting for v3.x.
+ common.infoMsg("Calico policy is in use. It should be enabled for v3.x as well.")
+ def saltPolicyResult = salt.getPillar(
+ pepperEnv, target, "kubernetes:pool:network:calico:policy"
+ )["return"][0].values()[0].toString()
+
+ common.infoMsg("kubernetes.pool.network.calico.policy: ${saltPolicyResult}")
+ if (saltPolicyResult.toLowerCase().contains("true")) {
+ common.infoMsg("Calico policy setting for v3.x is detected as: true")
+ } else {
+ caution = "ATTENTION. Currently, Calico is running with policy switched on. " +
+ "Calico policy setting for v3.x is not set to true. " +
+ "After upgrade is completed, Calico policy will be switched off. " +
+ "You will need to switch it on manually if required."
+ currentBuild.description += "<br><b>${caution}</b><br><br>"
+ common.warningMsg(caution)
+ }
+ }
+ }
+
+ if (addonsPolicy) {
+ // Remove v2.6.x policy-related addons on masters to not interfere with v3.x kube-controllers
+ salt.cmdRun(pepperEnv, CTL_TARGET, "rm -rf /etc/kubernetes/addons/calico_policy")
+ }
+ }
+}
+
timeout(time: 12, unit: 'HOURS') {
node() {
try {
@@ -378,6 +459,9 @@
pullCalicoImages(pepperEnv, POOL)
}
+ // check and adjust Calico policy setting
+ checkCalicoPolicySetting(pepperEnv, ctl_node)
+
// this sequence implies workloads operations downtime
startCalicoUpgrade(pepperEnv, ctl_node)
performCalicoConfigurationUpdateAndServicesRestart(pepperEnv, POOL)