Align upgrade-mcp-release pipeline with release branch
Change-Id: I3b3fa0f0b940fd010fa8fcc907ceeee135ec3a0a
Related-Prod: PROD-30893 (PROD:30893)
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index e865a06..161029e 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -22,14 +22,25 @@
venvPepper = "venvPepper"
workspace = ""
-def triggerMirrorJob(jobName) {
+def triggerMirrorJob(String jobName, String reclassSystemBranch) {
params = jenkinsUtils.getJobParameters(jobName)
- build job: jobName, parameters: [
- [$class: 'StringParameterValue', name: 'BRANCHES', value: params.get("BRANCHES")],
- [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get("CREDENTIALS_ID")],
- [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get("SOURCE_URL")],
- [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get("TARGET_URL")]
- ]
+ try {
+ build job: jobName, parameters: [
+ [$class: 'StringParameterValue', name: 'BRANCHES', value: params.get('BRANCHES')],
+ [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get('CREDENTIALS_ID')],
+ [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get('SOURCE_URL')],
+ [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get('TARGET_URL')]
+ ]
+ } catch (Exception updateErr) {
+ common.warningMsg(updateErr)
+ common.warningMsg('Attempt to update git repo in failsafe manner')
+ build job: jobName, parameters: [
+ [$class: 'StringParameterValue', name: 'BRANCHES', value: reclassSystemBranch.replace('origin/', '')],
+ [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get('CREDENTIALS_ID')],
+ [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get('SOURCE_URL')],
+ [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get('TARGET_URL')]
+ ]
+ }
}
def updateSaltStack(target, pkgs) {
@@ -58,64 +69,179 @@
}
}
+def wa29352(ArrayList saltMinions, String cname) {
+ // WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/12/openssh/client/root.yml
+ // Default soft-param has been removed, what now makes not possible to render some old env's.
+ // Like fix, we found copy-paste already generated key from backups, to secrets.yml with correct key name
+ def wa29352ClassName = 'cluster.' + cname + '.infra.secrets_root_wa29352'
+ def wa29352File = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets_root_wa29352.yml"
+ def wa29352SecretsFile = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets.yml"
+ def _tempFile = '/tmp/wa29352_' + UUID.randomUUID().toString().take(8)
+ try {
+ salt.cmdRun(venvPepper, 'I@salt:master', "grep -qiv root_private_key ${wa29352SecretsFile}", true, null, false)
+ salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29352File}", true, null, false)
+ }
+ catch (Exception ex) {
+ common.infoMsg('Work-around for PROD-29352 already apply, nothing todo')
+ return
+ }
+ def rKeysDict = [
+ 'parameters': [
+ '_param': [
+ 'root_private_key': salt.getPillar(venvPepper, 'I@salt:master', '_param:root_private_key').get('return')[0].values()[0].trim(),
+ 'root_public_key' : '',
+ ]
+ ]
+ ]
+ // save root key,and generate public one from it
+ writeFile file: _tempFile, text: rKeysDict['parameters']['_param']['root_private_key'].toString().trim()
+ sh('chmod 0600 ' + _tempFile)
+ rKeysDict['parameters']['_param']['root_public_key'] = sh(script: "ssh-keygen -q -y -f ${_tempFile}", returnStdout: true).trim()
+ sh('rm -fv ' + _tempFile)
+ writeYaml file: _tempFile, data: rKeysDict
+ def yamlData = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
+ salt.cmdRun(venvPepper, 'I@salt:master', "echo '${yamlData}' | base64 -d > ${wa29352File}", false, null, false)
+ common.infoMsg("Add $wa29352ClassName class into secrets.yml")
+
+ // Add 'classes:' directive
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
+ "grep -q 'classes:' infra/secrets.yml || sed -i '1iclasses:' infra/secrets.yml")
+
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
+ "grep -q '${wa29352ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29352ClassName' infra/secrets.yml")
+ salt.fullRefresh(venvPepper, '*')
+ sh('rm -fv ' + _tempFile)
+}
+
+def wa29155(ArrayList saltMinions, String cname) {
+ // WA for PROD-29155. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/
+ // CHeck for existence cmp nodes, and try to render it. Is failed, apply ssh-key wa
+ def ret = ''
+ def patched = false
+ def wa29155ClassName = 'cluster.' + cname + '.infra.secrets_nova_wa29155'
+ def wa29155File = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets_nova_wa29155.yml"
+
+ try {
+ salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29155File}", true, null, false)
+ }
+ catch (Exception ex) {
+ common.infoMsg('Work-around for PROD-29155 already apply, nothing todo')
+ return
+ }
+ salt.fullRefresh(venvPepper, 'cfg*')
+ salt.fullRefresh(venvPepper, 'cmp*')
+ for (String minion in saltMinions) {
+ if (!minion.startsWith('cmp')) {
+ continue
+ }
+ // First attempt, second will be performed in next validateReclassModel() stages
+ try {
+ salt.cmdRun(venvPepper, 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ } catch (Exception e) {
+ common.errorMsg(e.toString())
+ if (patched) {
+ error("Node: ${minion} failed to render after reclass-system upgrade!WA29155 probably didn't help.")
+ }
+ // check, that failed exactly by our case, by key-length check.
+ def missed_key = salt.getPillar(venvPepper, minion, '_param:nova_compute_ssh_private').get("return")[0].values()[0]
+ if (missed_key != '') {
+ error("Node: ${minion} failed to render after reclass-system upgrade!")
+ }
+ common.warningMsg('Perform: Attempt to apply WA for PROD-29155\n' +
+ 'See https://gerrit.mcp.mirantis.com/#/c/37932/ for more info')
+ common.warningMsg('WA-PROD-29155 Generating new ssh key at master node')
+ def _tempFile = "/tmp/nova_wa29155_" + UUID.randomUUID().toString().take(8)
+ common.infoMsg('Perform: generation NEW ssh-private key for nova-compute')
+ salt.cmdRun(venvPepper, 'I@salt:master', "ssh-keygen -f ${_tempFile} -N '' -q")
+ def _pub_k = salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'cmd.run', "cat ${_tempFile}.pub").get('return')[0].values()[0].trim()
+ def _priv_k = salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'cmd.run', "cat ${_tempFile}").get('return')[0].values()[0].trim()
+ salt.cmdRun(venvPepper, 'I@salt:master', "rm -fv ${_tempFile}", false, null, false)
+ def novaKeysDict = [
+ "parameters": [
+ "_param": [
+ "nova_compute_ssh_private": _priv_k,
+ "nova_compute_ssh_public" : _pub_k
+ ]
+ ]
+ ]
+ writeYaml file: _tempFile, data: novaKeysDict
+ def yamlData = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
+ salt.cmdRun(venvPepper, 'I@salt:master', "echo '${yamlData}' | base64 -d > ${wa29155File}", false, null, false)
+ common.infoMsg("Add $wa29155ClassName class into secrets.yml")
+
+ // Add 'classes:' directive
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
+ "grep -q 'classes:' infra/secrets.yml || sed -i '1iclasses:' infra/secrets.yml")
+
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
+ "grep -q '${wa29155ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29155ClassName' infra/secrets.yml")
+ salt.fullRefresh(venvPepper, 'cfg*')
+ salt.fullRefresh(venvPepper, 'cmp*')
+ patched = true
+ }
+ }
+ if (patched) {
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && git status && " +
+ "git add ${wa29155File} && git add -u && git commit --allow-empty -m 'Cluster model updated with WA for PROD-29155. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/ at ${common.getDatetime()}' ")
+ common.infoMsg('Work-around for PROD-29155 successfully applied')
+ }
+
+}
+
def archiveReclassInventory(filename) {
- def ret = salt.cmdRun(venvPepper, 'I@salt:master', "reclass -i", true, null, false)
- def reclassInv = ret.values()[0]
- writeFile file: filename, text: reclassInv.toString()
- archiveArtifacts artifacts: "$filename"
+ def _tmp_file = '/tmp/' + filename + UUID.randomUUID().toString().take(8)
+ // jenkins may fail at overheap. Compress data with gzip like WA
+ def ret = salt.cmdRun(venvPepper, 'I@salt:master', 'reclass -i 2>/dev/null | gzip -9 -c | base64', true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ def _tmp = sh(script: "echo '$ret' > ${_tmp_file}", returnStdout: false)
+ sh(script: "cat ${_tmp_file} | base64 -d | gzip -d > $filename", returnStdout: false)
+ archiveArtifacts artifacts: filename
+ sh(script: "rm -v ${_tmp_file}|| true")
}
def validateReclassModel(ArrayList saltMinions, String suffix) {
try {
- dir(suffix) {
- for(String minion in saltMinions) {
- common.infoMsg("Reclass model validation for minion ${minion}...")
- def ret = salt.cmdRun("${workspace}/${venvPepper}", 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0]
- writeFile file: minion, text: ret.toString()
- }
+ for (String minion in saltMinions) {
+ common.infoMsg("Reclass model validation for minion ${minion}...")
+ def reclassInv = salt.cmdRun(venvPepper, 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ writeFile file: "inventory-${minion}-${suffix}.out", text: reclassInv.toString()
}
} catch (Exception e) {
common.errorMsg('Can not validate current Reclass model. Inspect failed minion manually.')
- error(e)
+ error(e.toString())
}
}
-def archiveReclassModelChanges(ArrayList saltMinions, String oldSuffix, String newSuffix) {
- def diffDir = 'pillarsDiff'
- dir(diffDir) {
- for(String minion in saltMinions) {
- def fileName = "reclass-model-${minion}-diff.out"
- sh "diff -u ${workspace}/${oldSuffix}/${minion} ${workspace}/${newSuffix}/${minion} > ${fileName} || true"
- }
+def archiveReclassModelChanges(ArrayList saltMinions, String oldSuffix = 'before', String newSuffix = 'after') {
+ for (String minion in saltMinions) {
+ def fileName = "reclass-model-${minion}-diff.out"
+ sh "diff -u inventory-${minion}-${oldSuffix}.out inventory-${minion}-${newSuffix}.out > ${fileName} || true"
+ archiveArtifacts artifacts: "${fileName}"
}
- archiveArtifacts artifacts: "${oldSuffix}/*"
- archiveArtifacts artifacts: "${newSuffix}/*"
- archiveArtifacts artifacts: "${diffDir}/*"
}
if (common.validInputParam('PIPELINE_TIMEOUT')) {
try {
pipelineTimeout = env.PIPELINE_TIMEOUT.toInteger()
- } catch(Exception e) {
+ } catch (Exception e) {
common.warningMsg("Provided PIPELINE_TIMEOUT parameter has invalid value: ${env.PIPELINE_TIMEOUT} - should be interger")
}
}
timeout(time: pipelineTimeout, unit: 'HOURS') {
- node("python && docker") {
+ node("python") {
try {
+ def inventoryBeforeFilename = "reclass-inventory-before.out"
+ def inventoryAfterFilename = "reclass-inventory-after.out"
workspace = common.getWorkspace()
- deleteDir()
targetMcpVersion = null
if (!common.validInputParam('TARGET_MCP_VERSION') && !common.validInputParam('MCP_VERSION')) {
error('You must specify MCP version in TARGET_MCP_VERSION|MCP_VERSION variable')
}
// bw comp. for 2018.X => 2018.11 release
- if (common.validInputParam('MCP_VERSION')){
+ if (common.validInputParam('MCP_VERSION')) {
targetMcpVersion = env.MCP_VERSION
common.warningMsg("targetMcpVersion has been changed to:${targetMcpVersion}, which was taken from deprecated pipeline viriable:MCP_VERSION")
- }
- else {
+ } else {
targetMcpVersion = env.TARGET_MCP_VERSION
}
// end bw comp. for 2018.X => 2018.11 release
@@ -160,26 +286,24 @@
updateLocalRepos = env.getProperty('UPDATE_LOCAL_REPOS').toBoolean()
reclassSystemBranch = reclassSystemBranchDefault
}
-
python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
-
- def pillarsBeforeSuffix = 'pillarsBefore'
- def pillarsAfterSuffix = 'pillarsAfter'
- def inventoryBeforeFilename = "reclass-inventory-before.out"
- def inventoryAfterFilename = "reclass-inventory-after.out"
-
def minions = salt.getMinions(venvPepper, '*')
- def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
- stage("Update Reclass and Salt-Formulas ") {
- validateReclassModel(minions, pillarsBeforeSuffix)
+ stage('Update Reclass and Salt-Formulas') {
+ common.infoMsg('Perform: Full salt sync')
+ salt.fullRefresh(venvPepper, '*')
+ common.infoMsg('Perform: Validate reclass medata before processing')
+ validateReclassModel(minions, 'before')
+
+ common.infoMsg('Perform: archiveReclassInventory before upgrade')
archiveReclassInventory(inventoryBeforeFilename)
+ def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', '_param:cluster_name').get('return')[0].values()[0]
try {
- salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/ && git diff-index --quiet HEAD --")
+ salt.cmdRun(venvPepper, 'I@salt:master', 'cd /srv/salt/reclass/ && git status && git diff-index --quiet HEAD --')
}
catch (Exception ex) {
- error("You have uncommited changes in your Reclass cluster model repository. Please commit or reset them and rerun the pipeline.")
+ error('You have uncommitted changes in your Reclass cluster model repository. Please commit or reset them and rerun the pipeline.')
}
if (updateClusterModel) {
common.infoMsg('Perform: UPDATE_CLUSTER_MODEL')
@@ -202,7 +326,7 @@
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.updates/system.linux.system.repo.mcp.apt_mirantis.update/g'")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
- "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
+ "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout ${reclassSystemBranch}")
// Add kubernetes-extra repo
if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
@@ -217,7 +341,7 @@
}
// Add all update repositories
def repoIncludeBase = '- system.linux.system.repo.mcp.apt_mirantis.'
- def updateRepoList = [ 'cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'percona', 'salt-formulas', 'saltstack', 'ubuntu' ]
+ def updateRepoList = ['cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'percona', 'salt-formulas', 'saltstack', 'ubuntu']
updateRepoList.each { repo ->
def repoNameUpdateInclude = "${repoIncludeBase}update.${repo}"
def filesWithInclude = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -Plr '\\${repoIncludeBase}${repo}\$' . || true", false).get('return')[0].values()[0].trim().tokenize('\n')
@@ -226,7 +350,7 @@
if (updateRepoIncludeExist == 'not_found') {
// Include needs to be added
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
- "sed -i 's/\\( *\\)${repoIncludeBase}${repo}\$/&\\n\\1${repoNameUpdateInclude}/g' ${file}")
+ "sed -i 's/\\( *\\)${repoIncludeBase}${repo}\$/&\\n\\1${repoNameUpdateInclude}/g' ${file}")
common.infoMsg("Update repo for ${repo} is added to ${file}")
}
}
@@ -247,121 +371,22 @@
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && git status && " +
"git add -u && git commit --allow-empty -m 'Cluster model update to the release $targetMcpVersion on $dateTime'")
}
-
- salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'saltutil.refresh_pillar')
- try {
- salt.enforceState(venvPepper, 'I@salt:master', 'linux.system.repo')
- } catch (Exception e) {
- common.errorMsg("Something wrong with model after UPDATE_CLUSTER_MODEL step. Please check model.")
- throw e
- }
-
- common.infoMsg('Running a check for compatibility with new Reclass/Salt-Formulas packages')
- def saltModelDir = 'salt-model'
- def nodesArtifact = 'pillarsFromValidation.tar.gz'
- def reclassModel = 'reclassModel.tar.gz'
- def pillarsAfterValidation = 'pillarsFromValidation'
- try {
- def repos = salt.getPillar(venvPepper, 'I@salt:master', "linux:system:repo").get("return")[0].values()[0]
- def cfgInfo = salt.getPillar(venvPepper, 'I@salt:master', "reclass:storage:node:infra_cfg01_node").get("return")[0].values()[0]
- def docker_image_for_test = salt.getPillar(venvPepper, 'I@salt:master', "_param:docker_image_cvp_sanity_checks").get("return")[0].values()[0]
- def saltModelTesting = new com.mirantis.mk.SaltModelTesting()
- def config = [
- 'dockerHostname': "cfg01",
- 'distribRevision': "${targetMcpVersion}",
- 'baseRepoPreConfig': true,
- 'extraRepoMergeStrategy': 'override',
- 'dockerContainerName': 'new-reclass-package-check',
- 'dockerMaxCpus': 1,
- 'image': docker_image_for_test,
- 'dockerExtraOpts': [
- "-v ${env.WORKSPACE}/${saltModelDir}:/srv/salt/reclass",
- "--entrypoint ''",
- ],
- 'extraRepos': ['repo': repos, 'aprConfD': "APT::Get::AllowUnauthenticated 'true';" ],
- 'envOpts': [ "CLUSTER_NAME=${cluster_name}", "NODES_ARTIFACT_NAME=${nodesArtifact}" ]
- ]
- def tarName = '/tmp/currentModel.tar.gz'
- salt.cmdRun(venvPepper, 'I@salt:master', "tar -cf ${tarName} --mode='a+rwX' --directory=/srv/salt/reclass classes")
- if (cfgInfo == '') {
- // case for old setups when cfg01 node model was static
- def node_name = salt.getPillar(venvPepper, 'I@salt:master', "linux:system:name").get("return")[0].values()[0]
- def node_domain = salt.getPillar(venvPepper, 'I@salt:master', "linux:system:domain").get("return")[0].values()[0]
- salt.cmdRun(venvPepper, 'I@salt:master', "tar -rf ${tarName} --mode='a+rwX' --directory=/srv/salt/reclass nodes/${node_name}.${node_domain}.yml")
- config['envOpts'].add("CFG_NODE_NAME=${node_name}.${node_domain}")
- }
- def modelHash = salt.cmdRun(venvPepper, 'I@salt:master', "cat ${tarName} | gzip -9 -c | base64", false, null, false).get('return')[0].values()[0]
- writeFile file: 'modelHash', text: modelHash
- sh "cat modelHash | base64 -d | gzip -d > ${reclassModel}"
- sh "mkdir ${saltModelDir} && tar -xf ${reclassModel} -C ${saltModelDir}"
-
- config['runCommands'] = [
- '001_Install_Salt_Reclass_Packages': { sh('apt-get install -y reclass salt-formula-*') },
- '002_Get_new_nodes': {
- try {
- sh('''#!/bin/bash
- new_generated_dir=/srv/salt/_new_nodes
- new_pillar_dir=/srv/salt/_new_pillar
- reclass_classes=/srv/salt/reclass/classes/
- mkdir -p ${new_generated_dir} ${new_pillar_dir}
- nodegenerator -b ${reclass_classes} -o ${new_generated_dir} ${CLUSTER_NAME}
- for node in $(ls ${new_generated_dir}); do
- nodeName=$(basename -s .yml ${node})
- reclass -n ${nodeName} -c ${reclass_classes} -u ${new_generated_dir} > ${new_pillar_dir}/${nodeName}
- done
- if [[ -n "${CFG_NODE_NAME}" ]]; then
- reclass -n ${CFG_NODE_NAME} -c ${reclass_classes} -u /srv/salt/reclass/nodes > ${new_pillar_dir}/${CFG_NODE_NAME}
- fi
- tar -czf /tmp/${NODES_ARTIFACT_NAME} -C ${new_pillar_dir}/ .
- ''')
- } catch (Exception e) {
- print "Test new nodegenerator tool is failed: ${e}"
- throw e
- }
- },
- ]
- config['runFinally'] = [ '001_Archive_nodegenerator_artefact': {
- sh(script: "mv /tmp/${nodesArtifact} ${env.WORKSPACE}/${nodesArtifact}")
- archiveArtifacts artifacts: nodesArtifact
- }]
- saltModelTesting.setupDockerAndTest(config)
- def pillarsValidationDiff = "${pillarsAfterValidation}/diffFromOriginal"
- sh "mkdir -p ${pillarsValidationDiff} && tar -xf ${nodesArtifact} --dir ${pillarsAfterValidation}/"
- def changesFound = false
- for(String minion in minions) {
- try {
- sh (script:"diff -u -w -I '^Salt command execution success' -I '^ node: ' -I '^ uri: ' -I '^ timestamp: ' ${pillarsBeforeSuffix}/${minion} ${pillarsAfterValidation}/${minion} > ${pillarsValidationDiff}/${minion}", returnStdout: true)
- } catch(Exception e) {
- changesFound = true
- archiveArtifacts artifacts: "${pillarsValidationDiff}/${minion}"
- def buildUrl = env.BUILD_URL ? env.BUILD_URL : "${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_NUMBER}"
- common.errorMsg("Found diff changes for ${minion} minion: ${buildUrl}/artifact/${pillarsValidationDiff}/${minion}/*view*/ ")
- }
- }
- if (changesFound) {
- common.warningMsg('Found diff changes between current pillar data and updated. Inspect logs above.')
- input message: 'Continue anyway?'
- } else {
- common.infoMsg('Diff between current pillar data and updated one - not found.')
- }
- } catch (Exception updateErr) {
- common.warningMsg(updateErr)
- common.warningMsg('Failed to validate update Salt Formulas repos/packages.')
- input message: 'Continue anyway?'
- } finally {
- sh "rm -rf ${saltModelDir} ${nodesArtifact} ${pillarsAfterValidation} ${reclassModel}"
- }
-
try {
common.infoMsg('Perform: UPDATE Salt Formulas')
+ salt.fullRefresh(venvPepper, '*')
+ salt.enforceState(venvPepper, 'I@salt:master', 'linux.system.repo')
def saltEnv = salt.getPillar(venvPepper, 'I@salt:master', "_param:salt_master_base_environment").get("return")[0].values()[0]
- salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'state.sls_id', ["salt_master_${saltEnv}_pkg_formulas",'salt.master.env'])
+ salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'state.sls_id', ["salt_master_${saltEnv}_pkg_formulas", 'salt.master.env'])
+ salt.fullRefresh(venvPepper, '*')
} catch (Exception updateErr) {
common.warningMsg(updateErr)
common.warningMsg('Failed to update Salt Formulas repos/packages. Check current available documentation on https://docs.mirantis.com/mcp/latest/, how to update packages.')
input message: 'Continue anyway?'
}
+ wa29352(minions, cluster_name)
+ wa29155(minions, cluster_name)
+
try {
common.infoMsg('Perform: UPDATE Reclass package')
salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'pkg.install', ["reclass"])
@@ -381,21 +406,23 @@
}
salt.fullRefresh(venvPepper, '*')
-
try {
salt.cmdRun(venvPepper, 'I@salt:master', "reclass-salt --top")
}
catch (Exception ex) {
- error("Reclass fails rendering. Pay attention to your cluster model.")
+
+ error('Reclass fails rendering. Pay attention to your cluster model.' +
+ 'ErrorMessage:' + ex.toString())
}
+ common.infoMsg('Perform: archiveReclassInventory AFTER upgrade')
archiveReclassInventory(inventoryAfterFilename)
sh "diff -u $inventoryBeforeFilename $inventoryAfterFilename > reclass-inventory-diff.out || true"
archiveArtifacts artifacts: "reclass-inventory-diff.out"
- validateReclassModel(minions, pillarsAfterSuffix)
- archiveReclassModelChanges(minions, pillarsBeforeSuffix, pillarsAfterSuffix)
+ validateReclassModel(minions, 'after')
+ archiveReclassModelChanges(minions)
}
if (updateLocalRepos) {
@@ -437,7 +464,7 @@
}
}
- stage("Update Drivetrain") {
+ stage('Update Drivetrain') {
if (upgradeSaltStack) {
updateSaltStack("I@salt:master", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
@@ -446,28 +473,26 @@
}
if (updatePipelines) {
- triggerMirrorJob("git-mirror-downstream-mk-pipelines")
- triggerMirrorJob("git-mirror-downstream-pipeline-library")
+ common.infoMsg('Perform: UPDATE git repos')
+ triggerMirrorJob('git-mirror-downstream-mk-pipelines', reclassSystemBranch)
+ triggerMirrorJob('git-mirror-downstream-pipeline-library', reclassSystemBranch)
}
// update minions certs
- // call for `salt.minion.ca` state on related nodes to make sure
- // mine was updated with required data after salt-minion/salt-master restart salt:minion:ca
- salt.enforceState(venvPepper, "I@salt:minion:ca", 'salt.minion.ca', true)
salt.enforceState(venvPepper, "I@salt:minion", 'salt.minion.cert', true)
- // run `salt.minion` to refresh all minion configs (for example _keystone.conf)
- salt.enforceState([saltId: venvPepper, target: "I@salt:minion", state: ['salt.minion'], read_timeout: 60, retries: 2])
- // updating users and keys
+
+ // Retry needed only for rare race-condition in user appearance
+ common.infoMsg('Perform: updating users and keys')
salt.enforceState(venvPepper, "I@linux:system", 'linux.system.user', true)
+ common.infoMsg('Perform: updating openssh')
salt.enforceState(venvPepper, "I@linux:system", 'openssh', true)
- salt.enforceState(venvPepper, "I@jenkins:client", 'jenkins.client', true)
-
- salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client' state.sls docker.client --async")
+ salt.enforceState(venvPepper, 'I@jenkins:client and not I@salt:master', 'jenkins.client', true)
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
sleep(180)
- common.infoMsg("Checking if Docker containers are up")
+ common.infoMsg('Perform: Checking if Docker containers are up')
try {
common.retry(10, 30) {
@@ -485,4 +510,4 @@
throw e
}
}
-}
+}
\ No newline at end of file