Merge "Add calico upgrade path into k8s upgrade pipeline"
diff --git a/cloud-update.groovy b/cloud-update.groovy
index ab72f76..8dea65b 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -42,6 +42,7 @@
*
**/
def common = new com.mirantis.mk.Common()
+def orchestrate = new com.mirantis.mk.Orchestrate()
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
def virsh = new com.mirantis.mk.Virsh()
@@ -864,6 +865,10 @@
/*
* Update section
*/
+
+ // Go through applications that using orchestrated deployment.
+ orchestrate.OrchestrateApplications(pepperEnv, "I@salt:master", "orchestration.deploy.applications")
+
if (updates.contains("cfg")) {
def target = 'I@salt:master'
def type = 'cfg'
diff --git a/gating-pipeline.groovy b/gating-pipeline.groovy
index aeaee9a..15518d4 100644
--- a/gating-pipeline.groovy
+++ b/gating-pipeline.groovy
@@ -4,6 +4,7 @@
* JOBS_NAMESPACE - Gerrit gating jobs namespace (mk, contrail, ...)
*
**/
+import groovy.json.JsonOutput
def common = new com.mirantis.mk.Common()
def gerrit = new com.mirantis.mk.Gerrit()
@@ -18,12 +19,7 @@
}
def callJobWithExtraVars(String jobName) {
- def gerritVars = '\n---'
- for (envVar in env.getEnvironment()) {
- if (envVar.key.startsWith("GERRIT_")) {
- gerritVars += "\n${envVar.key}: '${envVar.value}'"
- }
- }
+ def gerritVars = JsonOutput.toJson(env.getEnvironment().findAll{ it.key.startsWith('GERRIT_') })
testJob = build job: jobName, parameters: [
[$class: 'TextParameterValue', name: 'EXTRA_VARIABLES_YAML', value: gerritVars]
]
diff --git a/opencontrail40-upgrade.groovy b/opencontrail40-upgrade.groovy
index 01adb0c..63cb5b4 100644
--- a/opencontrail40-upgrade.groovy
+++ b/opencontrail40-upgrade.groovy
@@ -29,18 +29,17 @@
def thirdPartyControlPkgsToRemove = 'zookeeper,libzookeeper-java,kafka,cassandra,redis-server,ifmap-server,supervisor'
def analyticsPkgs = 'contrail-analytics,contrail-lib,contrail-nodemgr,contrail-utils,python-contrail,contrail-database'
def thirdPartyAnalyticsPkgsToRemove = 'zookeeper,libzookeeper-java,kafka,cassandra,python-cassandra,cassandra-cpp-driver,redis-server,supervisor'
-//def cmpPkgs = ['contrail-lib', 'contrail-nodemgr', 'contrail-utils', 'contrail-vrouter-agent', 'contrail-vrouter-utils', 'python-contrail', 'python-contrail-vrouter-api', 'python-opencontrail-vrouter-netns', 'contrail-vrouter-dkms']
-def CMP_PKGS = 'contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms'
+def cmpPkgs = 'contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms'
def neutronServerPkgs = 'neutron-plugin-contrail,contrail-heat,python-contrail'
def dashboardPanelPkg = 'openstack-dashboard-contrail-panels'
-def KERNEL_MODULE_RELOAD = 'service supervisor-vrouter stop; rmmod vrouter; sync && echo 3 > /proc/sys/vm/drop_caches && echo 1 > /proc/sys/vm/compact_memory; service contrail-vrouter-agent start; service contrail-vrouter-nodemgr start'
+def kernelModuleReloadCmd = 'service supervisor-vrouter stop; rmmod vrouter; sync && echo 3 > /proc/sys/vm/drop_caches && echo 1 > /proc/sys/vm/compact_memory; service contrail-vrouter-agent start; service contrail-vrouter-nodemgr start'
def analyticsServices = ['supervisor-analytics', 'supervisor-database', 'zookeeper', 'redis-server']
def configServices = ['contrail-webui-jobserver', 'contrail-webui-webserver', 'supervisor-config', 'supervisor-database', 'zookeeper']
def controlServices = ['ifmap-server', 'supervisor-control', 'redis-server']
def thirdPartyServicesToDisable = ['kafka', 'zookeeper', 'cassandra']
def config4Services = ['zookeeper', 'contrail-webui-middleware', 'contrail-webui', 'contrail-api', 'contrail-schema', 'contrail-svc-monitor', 'contrail-device-manager', 'contrail-config-nodemgr', 'contrail-database']
-def void runCommonCommands(target, command, args, check, salt, pepperEnv, common) {
+def runCommonCommands(target, command, args, check, salt, pepperEnv) {
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, null, args, null)
salt.printSaltCommandResult(out)
@@ -52,9 +51,6 @@
} else if ( check == "contrail-status" ) {
salt.commandStatus(pepperEnv, target, "${check} | grep -v == | grep -v FOR | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup | grep -v -F /var/crashes/", null, false, true, null, true, 500)
}
- //out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, null, check, null)
- //salt.printSaltCommandResult(out)
- //input message: "Please check the output of \'${check}\' and continue if it is correct."
}
timeout(time: 12, unit: 'HOURS') {
node() {
@@ -136,7 +132,7 @@
}
check = 'doctrail all contrail-status'
salt.enforceState(pepperEnv, 'I@opencontrail:collector', 'docker.client')
- runCommonCommands('I@opencontrail:collector:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:primary', command, args, check, salt, pepperEnv)
} catch (Exception er) {
common.errorMsg("Opencontrail Analytics failed to be upgraded.")
throw er
@@ -160,7 +156,7 @@
salt.enforceState(pepperEnv, 'I@opencontrail:control:role:secondary', 'docker.client')
- runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv)
sleep(120)
@@ -259,11 +255,11 @@
salt.runSaltProcessStep(pepperEnv, targetLiveSubset, 'file.remove', ["/etc/apt/sources.list.d/mcp_opencontrail.list"], null, true)
salt.enforceState(pepperEnv, targetLiveSubset, 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check availability of contrail packages before continuing.")
throw er
}
- args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${cmpPkgs} -y;"
check = 'contrail-status'
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
@@ -275,7 +271,7 @@
common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
}
- salt.runSaltProcessStep(pepperEnv, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+ salt.runSaltProcessStep(pepperEnv, targetLiveSubset, 'cmd.shell', [kernelModuleReloadCmd], null, true)
//sleep(10)
salt.commandStatus(pepperEnv, targetLiveSubset, "${check} | grep -v == | grep -v active | grep -v -F /var/crashes/", null, false)
@@ -287,6 +283,7 @@
stage('Confirm upgrade on all targeted nodes') {
input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
}
+
stage("Opencontrail compute upgrade on all targeted nodes") {
try {
@@ -295,11 +292,11 @@
salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'file.remove', ["/etc/apt/sources.list.d/mcp_opencontrail.list"], null, true)
salt.enforceState(pepperEnv, targetLiveAll, 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check availability of contrail packages before continuing.")
throw er
}
- args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${cmpPkgs} -y;"
check = 'contrail-status'
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
@@ -311,8 +308,7 @@
common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
}
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
- //sleep(10)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'cmd.shell', [kernelModuleReloadCmd], null, true)
salt.commandStatus(pepperEnv, targetLiveAll, "${check} | grep -v == | grep -v active | grep -v -F /var/crashes/", null, false)
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
@@ -334,7 +330,7 @@
input message: "Do you want to continue with the Opencontrail nodes rollback?"
}
- stage('Opencontrail controllers rollback') {
+ stage('Opencontrail controllers rollback') {
salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:database or I@neutron:server or I@horizon:server', 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(pepperEnv, 'I@opencontrail:database or I@neutron:server or I@horizon:server', 'linux.system.repo')
@@ -350,7 +346,7 @@
salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:secondary', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
check = 'contrail-status'
- runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv)
sleep(120)
@@ -404,11 +400,11 @@
salt.runSaltProcessStep(pepperEnv, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(pepperEnv, targetLiveSubset, 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check availability of contrail packages before continuing.")
throw er
}
- args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${cmpPkgs} -y;"
check = 'contrail-status'
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
@@ -420,8 +416,7 @@
common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
}
- salt.runSaltProcessStep(pepperEnv, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
- //sleep(10)
+ salt.runSaltProcessStep(pepperEnv, targetLiveSubset, 'cmd.shell', [kernelModuleReloadCmd], null, true)
salt.commandStatus(pepperEnv, targetLiveSubset, "${check} | grep -v == | grep -v active | grep -v -F /var/crashes/", null, false)
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
@@ -439,11 +434,11 @@
salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(pepperEnv, targetLiveAll, 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check availability of contrail packages before continuing.")
throw er
}
- args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${cmpPkgs} -y;"
check = 'contrail-status'
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
@@ -455,8 +450,7 @@
common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
}
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
- //sleep(10)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'cmd.shell', [kernelModuleReloadCmd], null, true)
salt.commandStatus(pepperEnv, targetLiveAll, "${check} | grep -v == | grep -v active | grep -v -F /var/crashes/", null, false)
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index c20c3a0..b585e7e 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -19,6 +19,19 @@
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
+ stage('Opencontrail controllers health check') {
+ try {
+ salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'opencontrail.upgrade.verify', true, true)
+ } catch (Exception er) {
+ common.errorMsg("Opencontrail controllers health check stage found issues with services. Please take a look at the logs above.")
+ throw er
+ }
+ }
+
+ stage('Backup') {
+ salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', 'bash /usr/local/bin/cassandra-backup-runner-call.sh')
+ }
+
stage('Restore') {
// get opencontrail version
def _pillar = salt.getPillar(pepperEnv, "I@opencontrail:control", '_param:opencontrail_version')
@@ -55,11 +68,9 @@
// the lovely wait-60-seconds mantra before restarting supervisor-database service
sleep(60)
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "doctrail controller systemctl restart contrail-database")
- // another mantra
+ // another mantra, wait till all services are up
sleep(60)
- salt.cmdRun(pepperEnv, 'I@opencontrail:control', "doctrail controller contrail-status")
- }
- else {
+ } else {
try {
salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.stop', ['neutron-server'], null, true)
} catch (Exception er) {
@@ -125,5 +136,12 @@
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "contrail-status")
}
}
+
+ stage('Opencontrail controllers health check') {
+ common.retry(3, 20){
+ salt.cmdRun(pepperEnv, 'I@opencontrail:control', "doctrail controller contrail-status")
+ salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'opencontrail.upgrade.verify', true, true)
+ }
+ }
}
}
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index bd3373c..33ab3f5 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -158,14 +158,14 @@
if (env.GERRIT_PROJECT) {
messages.add("<font color='red'>GerritTrigger detected! We are in auto-mode:</font>")
messages.add("Test env variables has been changed:")
- messages.add("COOKIECUTTER_TEMPLATE_BRANCH => ${gerritDataCC['gerritBranch']}")
- messages.add("RECLASS_MODEL_BRANCH => ${gerritDataRS['gerritBranch']}")
// TODO are we going to have such branches?
if (!['nightly', 'testing', 'stable', 'proposed', 'master'].contains(env.GERRIT_BRANCH)) {
gerritDataCC['gerritBranch'] = env.GERRIT_BRANCH
gerritDataRS['gerritBranch'] = env.GERRIT_BRANCH
testDistribRevision = env.GERRIT_BRANCH
}
+ messages.add("COOKIECUTTER_TEMPLATE_BRANCH => ${gerritDataCC['gerritBranch']}")
+ messages.add("RECLASS_SYSTEM_BRANCH => ${gerritDataRS['gerritBranch']}")
// Identify, who triggered. To whom we should pass refspec
if (env.GERRIT_PROJECT == 'salt-models/reclass-system') {
gerritDataRS['gerritRefSpec'] = env.GERRIT_REFSPEC
diff --git a/test-salt-formulas-env.groovy b/test-salt-formulas-env.groovy
index 465b048..486bd3a 100644
--- a/test-salt-formulas-env.groovy
+++ b/test-salt-formulas-env.groovy
@@ -24,6 +24,7 @@
def checkouted = false
def openstackTest = false
+def cleanEnv = ''
throttle(['test-formula']) {
timeout(time: 1, unit: 'HOURS') {
@@ -88,10 +89,10 @@
}
common.infoMsg("Running part of kitchen test")
if (KITCHEN_ENV != null && !KITCHEN_ENV.isEmpty() && KITCHEN_ENV != "") {
- def cleanEnv = KITCHEN_ENV.replaceAll("\\s?SUITE=[^\\s]*", "")
+ cleanEnv = KITCHEN_ENV.replaceAll("\\s?SUITE=[^\\s]*", "")
if (openstackTest) { cleanEnv = "KITCHEN_YAML=.kitchen.openstack.yml " + cleanEnv }
- sh("find . -type f -exec sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {} \\;")
- sh("find . -type f -exec sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {} \\;")
+ sh("grep apt.mirantis.com -Ril | xargs -I{} bash -c \"echo {}; sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {}\"")
+ sh("grep apt-mk.mirantis.com -Ril | xargs -I{} bash -c \"echo {}; sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {}\"")
def suite = ruby.getSuiteName(KITCHEN_ENV)
if (suite && suite != "") {
common.infoMsg("Running kitchen test with environment:" + KITCHEN_ENV.trim())
@@ -111,7 +112,8 @@
} catch (Throwable e) {
// If there was an error or exception thrown, the build failed
currentBuild.result = "FAILURE"
- ruby.runKitchenCommand("destroy")
+ sh(script: 'find .kitchen/logs/ -type f -iname "*.log" | xargs -I{} bash -c "echo {}; cat {}"')
+ ruby.runKitchenCommand("destroy", cleanEnv)
throw e
} finally {
if (currentBuild.result == "FAILURE" && fileExists(".kitchen/logs/kitchen.log")) {
diff --git a/test-system-reclass-pipeline.groovy b/test-system-reclass-pipeline.groovy
index 04eafeb..a8e10b7 100644
--- a/test-system-reclass-pipeline.groovy
+++ b/test-system-reclass-pipeline.groovy
@@ -1,3 +1,5 @@
+import groovy.json.JsonOutput
+
def gerrit = new com.mirantis.mk.Gerrit()
def common = new com.mirantis.mk.Common()
@@ -7,12 +9,7 @@
if (extraVarsYaml != '') {
common.mergeEnv(env, extraVarsYaml)
} else {
- extraVarsYaml = '\n---'
- for (envVar in env.getEnvironment()) {
- if (envVar.key.startsWith("GERRIT_")) {
- extraVarsYaml += "\n${envVar.key}: '${envVar.value}'"
- }
- }
+ extraVarsYaml = JsonOutput.toJson(env.getEnvironment().findAll{ it.key.startsWith('GERRIT_') })
}
def slaveNode = env.SLAVE_NODE ?: 'python&&docker'
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 566caa9..7dfe312 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -5,7 +5,7 @@
* Expected parameters:
* SALT_MASTER_URL Salt API server location
* SALT_MASTER_CREDENTIALS Credentials to the Salt API
- * MCP_VERSION Version of MCP to upgrade to
+ * TARGET_MCP_VERSION Version of MCP to upgrade to
* UPGRADE_SALTSTACK Upgrade SaltStack packages to new version.
* UPDATE_CLUSTER_MODEL Update MCP version parameter in cluster model
* UPDATE_PIPELINES Update pipeline repositories on Gerrit
@@ -20,7 +20,7 @@
venvPepper = "venvPepper"
workspace = ""
-def triggerMirrorJob(jobName){
+def triggerMirrorJob(jobName) {
params = jenkinsUtils.getJobParameters(jobName)
build job: jobName, parameters: [
[$class: 'StringParameterValue', name: 'BRANCHES', value: params.get("BRANCHES")],
@@ -30,16 +30,17 @@
]
}
-def updateSaltStack(target, pkgs){
- try{
+def updateSaltStack(target, pkgs) {
+ try {
salt.runSaltProcessStep(venvPepper, target, 'pkg.install', ["force_yes=True", "pkgs='$pkgs'"], null, true, 5)
- }catch(Exception ex){}
+ } catch (Exception ex) {
+ }
- common.retry(20, 60){
+ common.retry(20, 60) {
salt.minionsReachable(venvPepper, 'I@salt:master', '*')
def running = salt.runSaltProcessStep(venvPepper, target, 'saltutil.running', [], null, true, 5)
- for(value in running.get("return")[0].values()){
- if(value != []){
+ for (value in running.get("return")[0].values()) {
+ if (value != []) {
throw new Exception("Not all salt-minions are ready for execution")
}
}
@@ -49,15 +50,15 @@
def saltMinionVersions = salt.cmdRun(venvPepper, target, "apt-cache policy salt-common | awk '/Installed/ && /$saltVersion/'").get("return")
def saltMinionVersion = ""
- for(minion in saltMinionVersions[0].keySet()){
- saltMinionVersion = saltMinionVersions[0].get(minion).replace("Salt command execution success","").trim()
- if(saltMinionVersion == ""){
+ for (minion in saltMinionVersions[0].keySet()) {
+ saltMinionVersion = saltMinionVersions[0].get(minion).replace("Salt command execution success", "").trim()
+ if (saltMinionVersion == "") {
error("Installed version of Salt on $minion doesn't match specified version in the model.")
}
}
}
-def archiveReclassInventory(filename){
+def archiveReclassInventory(filename) {
def ret = salt.cmdRun(venvPepper, 'I@salt:master', "reclass -i", true, null, false)
def reclassInv = ret.values()[0]
writeFile file: filename, text: reclassInv.toString()
@@ -72,46 +73,49 @@
timeout(time: pipelineTimeout, unit: 'HOURS') {
node("python") {
try {
- def gitMcpVersion = MCP_VERSION
workspace = common.getWorkspace()
+ if (env.TARGET_MCP_VERSION == "") {
+ error("You must specify MCP version in TARGET_MCP_VERSION variable ")
+ }
+ def gitTargetMcpVersion = env.TARGET_MCP_VERSION
+ if (env.TARGET_MCP_VERSION == "testing") {
+ gitTargetMcpVersion = "master"
+ }
python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
- if(MCP_VERSION == ""){
- error("You must specify MCP version")
- }
- if(MCP_VERSION == "testing"){
- gitMcpVersion = "master"
- }
-
- stage("Update Reclass"){
+ stage("Update Reclass") {
def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
- try{
+ try {
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/ && git diff-index --quiet HEAD --")
}
- catch(Exception ex){
+ catch (Exception ex) {
error("You have uncommited changes in your Reclass cluster model repository. Please commit or reset them and rerun the pipeline.")
}
- if(UPDATE_CLUSTER_MODEL.toBoolean()){
+ if (UPDATE_CLUSTER_MODEL.toBoolean()) {
+ common.infoMsg('Perform: UPDATE_CLUSTER_MODEL')
def dateTime = common.getDatetime()
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
- "grep -r --exclude-dir=apty -l 'apt_mk_version: .*' * | xargs sed -i 's/apt_mk_version: .*/apt_mk_version: \"$MCP_VERSION\"/g'")
- salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout $gitMcpVersion")
+ "grep -r --exclude-dir=aptly -l 'mcp_version: .*' * | xargs sed -i 's/mcp_version: .*/mcp_version: \"$env.TARGET_MCP_VERSION\"/g'")
+ // Do the same, for deprecated variable-duplicate
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
+ "grep -r --exclude-dir=aptly -l 'apt_mk_version: .*' * | xargs sed -i 's/apt_mk_version: .*/apt_mk_version: \"$env.TARGET_MCP_VERSION\"/g'")
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout $gitTargetMcpVersion")
// Add new defaults
common.infoMsg("Add new defaults")
salt.cmdRun(venvPepper, 'I@salt:master', "grep '^- system.defaults\$' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +
- "sed -i 's/^classes:/classes:\\n- system.defaults/' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml")
+ "sed -i 's/^classes:/classes:\\n- system.defaults/' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml")
common.infoMsg("The following changes were made to the cluster model and will be commited. " +
- "Please consider if you want to push them to the remote repository or not. You have to do this manually when the run is finished.")
+ "Please consider if you want to push them to the remote repository or not. You have to do this manually when the run is finished.")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && git diff")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && git status && " +
- "git add -u && git commit --allow-empty -m 'Cluster model update to the release $MCP_VERSION on $dateTime'")
+ "git add -u && git commit --allow-empty -m 'Cluster model update to the release $env.TARGET_MCP_VERSION on $dateTime'")
}
salt.enforceState(venvPepper, 'I@salt:master', 'reclass.storage', true)
}
- if(UPDATE_LOCAL_REPOS.toBoolean()){
+ if (UPDATE_LOCAL_REPOS.toBoolean()) {
def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
- stage("Update local repos"){
+ stage("Update local repos") {
common.infoMsg("Updating local repositories")
def engine = salt.getPillar(venvPepper, 'I@aptly:publisher', "aptly:publisher:source:engine")
@@ -119,25 +123,22 @@
if (runningOnDocker) {
common.infoMsg("Aptly is running as Docker container")
- }
- else {
+ } else {
common.infoMsg("Aptly isn't running as Docker container. Going to use aptly user for executing aptly commands")
}
- if(runningOnDocker){
+ if (runningOnDocker) {
salt.cmdRun(venvPepper, 'I@aptly:publisher', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true)
- }
- else{
- salt.cmdRun(venvPepper, 'I@aptly:publisher', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true, ['runas=aptly'])
+ } else {
+ salt.cmdRun(venvPepper, 'I@aptly:publisher', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true, ['runas=aptly'])
}
salt.enforceState(venvPepper, 'I@aptly:publisher', 'aptly', true)
- if(runningOnDocker){
+ if (runningOnDocker) {
salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv"], null, true)
salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-frv -u http://10.99.0.1:8080"], null, true)
- }
- else{
+ } else {
salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv", 'runas=aptly'], null, true)
salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-afrv", 'runas=aptly'], null, true)
}
@@ -152,12 +153,12 @@
}
}
- stage("Update Drivetrain"){
- salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_salt.list")
+ stage("Update Drivetrain") {
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$env.TARGET_MCP_VERSION/4' /etc/apt/sources.list.d/mcp_salt.list")
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_salt.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
// Workaround for PROD-22108
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get purge -y salt-formula-octavia && " +
- "apt-get install -y salt-formula-octavia")
+ "apt-get install -y salt-formula-octavia")
// End workaround for PROD-22108
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades salt-formula-*")
@@ -166,25 +167,25 @@
archiveReclassInventory(inventoryBeforeFilename)
- salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_extra.list")
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$env.TARGET_MCP_VERSION/4' /etc/apt/sources.list.d/mcp_extra.list")
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_extra.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades reclass")
salt.fullRefresh(venvPepper, 'I@salt:master')
- try{
+ try {
salt.enforceState(venvPepper, "I@salt:master", 'reclass', true)
}
- catch(Exception ex){
+ catch (Exception ex) {
error("Reclass fails rendering. Pay attention to your cluster model.")
}
salt.fullRefresh(venvPepper, '*')
- try{
+ try {
salt.cmdRun(venvPepper, 'I@salt:master', "reclass-salt --top")
}
- catch(Exception ex){
+ catch (Exception ex) {
error("Reclass fails rendering. Pay attention to your cluster model.")
}
@@ -193,7 +194,7 @@
sh "diff -u $inventoryBeforeFilename $inventoryAfterFilename > reclass-inventory-diff.out || true"
archiveArtifacts artifacts: "reclass-inventory-diff.out"
- if(UPGRADE_SALTSTACK.toBoolean()){
+ if (UPGRADE_SALTSTACK.toBoolean()) {
salt.enforceState(venvPepper, "I@linux:system", 'linux.system.repo', true)
updateSaltStack("I@salt:master", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
@@ -201,7 +202,7 @@
updateSaltStack("I@salt:minion and not I@salt:master", '["salt-minion"]')
}
- if(UPDATE_PIPELINES.toBoolean()){
+ if (UPDATE_PIPELINES.toBoolean()) {
triggerMirrorJob("git-mirror-downstream-mk-pipelines")
triggerMirrorJob("git-mirror-downstream-pipeline-library")
}
@@ -214,12 +215,12 @@
common.infoMsg("Checking if Docker containers are up")
- try{
- common.retry(10, 30){
+ try {
+ common.retry(10, 30) {
salt.cmdRun(venvPepper, 'I@jenkins:client and I@docker:client', "! docker service ls | tail -n +2 | grep -v -E '\\s([0-9])/\\1\\s'")
}
}
- catch(Exception ex){
+ catch (Exception ex) {
error("Docker containers for CI/CD services are having troubles with starting.")
}
}