Merge "Fix failure of gpg key creation in case of empty RequesterEmail"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 8802cea..f05735a 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -447,6 +447,9 @@
}
}
+ // install docker swarm
+ orchestrate.installDockerSwarm(venvPepper, extra_tgt)
+
// install openstack
if (common.checkContains('STACK_INSTALL', 'openstack')) {
// install control, tests, ...
@@ -513,7 +516,6 @@
if (common.checkContains('STACK_INSTALL', 'cicd')) {
stage('Install Cicd') {
orchestrate.installInfra(venvPepper, extra_tgt)
- orchestrate.installDockerSwarm(venvPepper, extra_tgt)
orchestrate.installCicd(venvPepper, extra_tgt)
}
}
@@ -527,7 +529,6 @@
if (common.checkContains('STACK_INSTALL', 'stacklight')) {
stage('Install StackLight') {
- orchestrate.installDockerSwarm(venvPepper, extra_tgt)
orchestrate.installStacklight(venvPepper, extra_tgt)
}
}
@@ -536,7 +537,6 @@
stage('Install OSS') {
if (!common.checkContains('STACK_INSTALL', 'stacklight')) {
// In case if StackLightv2 enabled containers already started
- orchestrate.installDockerSwarm(venvPepper, extra_tgt)
salt.enforceState(venvPepper, "I@docker:swarm:role:master and I@devops_portal:config ${extra_tgt}", 'docker.client', true)
}
orchestrate.installOss(venvPepper, extra_tgt)
diff --git a/gating-pipeline.groovy b/gating-pipeline.groovy
index d449cd8..e1c80f0 100644
--- a/gating-pipeline.groovy
+++ b/gating-pipeline.groovy
@@ -10,7 +10,7 @@
def gerrit = new com.mirantis.mk.Gerrit()
def ssh = new com.mirantis.mk.Ssh()
-slaveNode = env.SLAVE_NODE ?: 'docker'
+slaveNode = env.SLAVE_NODE ?: 'virtual'
giveVerify = false
@NonCPS
diff --git a/k8s-upgrade-pipeline.groovy b/k8s-upgrade-pipeline.groovy
index a09ae85..1c168f3 100644
--- a/k8s-upgrade-pipeline.groovy
+++ b/k8s-upgrade-pipeline.groovy
@@ -139,11 +139,15 @@
}
}
-def performCalicoConfigurationUpdateAndServicesRestart(pepperEnv, target) {
+def performCalicoConfigurationUpdateAndServicesRestart(pepperEnv, target, ctl_node) {
def salt = new com.mirantis.mk.Salt()
stage("Performing Calico configuration update and services restart") {
- salt.enforceState(pepperEnv, target, "kubernetes.pool.calico")
+ if (containerDenabled(pepperEnv, ctl_node)) {
+ salt.enforceState(pepperEnv, target, "kubernetes.pool")
+ } else {
+ salt.enforceState(pepperEnv, target, "kubernetes.pool.calico")
+ }
salt.runSaltProcessStep(pepperEnv, target, 'service.restart', ['kubelet'])
}
}
@@ -279,14 +283,141 @@
)['return'][0].values()[0].replaceAll('Salt command execution success','').trim().toBoolean()
}
-def checkCalicoUpgradeSuccessful(pepperEnv, target) {
+def calicoEnabled(pepperEnv, target) {
+ def salt = new com.mirantis.mk.Salt()
+ return salt.getPillar(pepperEnv, target, "kubernetes:pool:network:calico:enabled"
+ )["return"][0].values()[0].toBoolean()
+}
+
+def checkCalicoClusterState(pepperEnv, target) {
+ def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
- stage("Checking cluster state after Calico upgrade") {
- // TODO add auto-check of results
- salt.cmdRun(pepperEnv, target, "calicoctl version | grep -i version")
- salt.cmdRun(pepperEnv, target, "calicoctl node status")
- salt.cmdRun(pepperEnv, target, "calicoctl node checksystem")
+ stage("Checking Calico cluster state after upgrade") {
+ // check Calico cluster and cli clients versions
+ def checkVer = [
+ "Client Version:": [verStr: "", dif: false, wrong: false],
+ "Cluster Version:": [verStr: "", dif: false, wrong: false]
+ ]
+ def checkVerPassed = true
+ def versionResults = salt.cmdRun(pepperEnv, target, "calicoctl version | grep -i version")['return'][0]
+ versionResults.each { k, v ->
+ // println("Node:\n${k}\nResult:\n${v}")
+ for (verLine in v.split("\n")) {
+ for (verType in checkVer.keySet()) {
+ if (verLine.contains(verType)) {
+ def verRec = checkVer[verType]
+ ver = (verLine - verType).trim()
+ if (!verRec.verStr) {
+ verRec.verStr = ver
+ }
+ if (verRec.verStr != ver) {
+ verRec.dif = true
+ checkVerPassed = false
+ }
+ version = ver.tokenize(".")
+ if ((version.size() < 3) || (version[0] != "v3")) {
+ verRec.wrong = true
+ checkVerPassed = false
+ }
+ checkVer[verType] = verRec
+ }
+ }
+ }
+ }
+ if (checkVerPassed) {
+ common.infoMsg("Calico version verification passed")
+ }
+ else {
+ def warningMsg = "Calico version verification failed.\n"
+ checkVer.each { k, rec ->
+ if (rec.dif) {
+ warningMsg += "${k} versions are different across nodes.\n"
+ }
+ if (rec.wrong) {
+ warningMsg += "${k} (some) versions are wrong - should be v3.x.\n"
+ }
+ }
+ common.warningMsg(warningMsg)
+ currentBuild.description += "<br><b>${warningMsg}</b><br><br>"
+ }
+
+ // check Calico nodes' statuses
+ def nodeStatusResults = salt.cmdRun(pepperEnv, target, "calicoctl node status")['return'][0]
+ def nodesRunning = true
+ def peersNotFound = []
+ def peersNotOnline = []
+ nodeStatusResults.each { k, v ->
+ // println("Node:\n${k}\nResult:\n${v}")
+ if (!v.contains("Calico process is running")) {
+ nodesRunning = false
+ def warningMsg = "Node ${k}: Calico node is not running."
+ common.warningMsg(warningMsg)
+ currentBuild.description += "<br><b>${warningMsg}</b><br><br>"
+ }
+ def nodePeersFound = false
+ def nodePeersOnline = true
+ for (nodeLine in v.split("\n")) {
+ if (nodeLine.contains("|") && (!nodeLine.contains("STATE"))) {
+ def col = nodeLine.tokenize("|").collect{it.trim()}
+ if (col.size() == 5) {
+ nodePeersFound = true
+ if ((col[2] != "up") || (col[4] != "Established")) {
+ def warningMsg = "Node ${k}: BGP peer '${col[0]}' is out of reach. Peer state: '${col[2]}', connection info: '${col[4]}'."
+ common.warningMsg(warningMsg)
+ currentBuild.description += "<br><b>${warningMsg}</b><br><br>"
+ nodePeersOnline = false
+ }
+ }
+ }
+ }
+ if (!nodePeersFound) {
+ peersNotFound += k
+ }
+ if (!nodePeersOnline) {
+ peersNotOnline += k
+ }
+ }
+ if (nodesRunning) {
+ common.infoMsg("All the Calico nodes are running")
+ }
+ if (peersNotFound) {
+ def warningMsg = "BGP peers not found for the node(s): " + peersNotFound.join(', ') + "."
+ common.warningMsg(warningMsg)
+ currentBuild.description += "<br><b>${warningMsg}</b><br><br>"
+ } else {
+ common.infoMsg("BGP peers were found for all the nodes")
+ }
+ if (!peersNotOnline) {
+ common.infoMsg("All reported BGP peers are reachable")
+ }
+
+ // check that 'calico-kube-controllers' is running
+ // one CTL node will be used to get pod's state using kubectl
+ def ctl_node = salt.getMinionsSorted(pepperEnv, CTL_TARGET)[0]
+ def kubeCtrlResult = salt.cmdRun(
+ pepperEnv, ctl_node, "kubectl get pod -n kube-system --selector=k8s-app=calico-kube-controllers"
+ )['return'][0].values()[0].toString()
+ if (kubeCtrlResult.contains("calico-kube-controllers")) {
+ for (line in kubeCtrlResult.split("\n")) {
+ if (line.contains("calico-kube-controllers")) {
+ col = line.tokenize(" ")
+ if ((col[1] != "1/1") || (col[2] != "Running")) {
+ def warningMsg = "Calico kube-controllers pod is not running properly."
+ common.warningMsg(warningMsg)
+ currentBuild.description += "<br><b>${warningMsg}</b><br><br>"
+ }
+ else {
+ common.infoMsg("Calico kube-controllers pod is running.")
+ }
+ break
+ }
+ }
+ } else {
+ def warningMsg = "Calico kube-controllers pod was not scheduled."
+ common.warningMsg(warningMsg)
+ currentBuild.description += "<br><b>${warningMsg}</b><br><br>"
+ }
}
}
@@ -494,11 +625,9 @@
// this sequence implies workloads operations downtime
startCalicoUpgrade(pepperEnv, ctl_node)
- performCalicoConfigurationUpdateAndServicesRestart(pepperEnv, POOL)
+ performCalicoConfigurationUpdateAndServicesRestart(pepperEnv, POOL, ctl_node)
completeCalicoUpgrade(pepperEnv, ctl_node)
- // after that no downtime is expected
-
- checkCalicoUpgradeSuccessful(pepperEnv, POOL)
+ // no downtime is expected after this point
}
/*
@@ -561,6 +690,11 @@
}
}
+ def ctl_node = salt.getMinionsSorted(pepperEnv, CTL_TARGET)[0]
+ if (calicoEnabled(pepperEnv, ctl_node)) {
+ checkCalicoClusterState(pepperEnv, POOL)
+ }
+
if (CONFORMANCE_RUN_AFTER.toBoolean()) {
def target = CTL_TARGET
def mcp_repo = ARTIFACTORY_URL
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index 79a1632..27313c7 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -27,7 +27,7 @@
extraVars = [:]
}
-slaveNode = env.SLAVE_NODE ?: 'docker'
+slaveNode = env.SLAVE_NODE ?: 'virtual'
checkIncludeOrder = env.CHECK_INCLUDE_ORDER ?: false
// Global var's
diff --git a/test-salt-formulas-pipeline.groovy b/test-salt-formulas-pipeline.groovy
index 45eeef8..52c7d79 100644
--- a/test-salt-formulas-pipeline.groovy
+++ b/test-salt-formulas-pipeline.groovy
@@ -62,7 +62,7 @@
]
}
-timeout(time: 2, unit: 'HOURS') {
+timeout(time: 4, unit: 'HOURS') {
node(slaveNode) {
try {
if (fileExists("tests/build")) {
diff --git a/test-salt-model-wrapper.groovy b/test-salt-model-wrapper.groovy
index 9f61db8..f49b7fc 100644
--- a/test-salt-model-wrapper.groovy
+++ b/test-salt-model-wrapper.groovy
@@ -12,7 +12,7 @@
cookiecutterTemplatesRepo='mk/cookiecutter-templates'
reclassSystemRepo='salt-models/reclass-system'
-slaveNode = env.getProperty('SLAVE_NODE') ?: 'python&&docker'
+slaveNode = env.getProperty('SLAVE_NODE') ?: 'virtual'
voteMatrix = [
'test-mk-cookiecutter-templates': true,