Merge "Update Shaker pipeline"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index de8d8fd..d59f313 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -467,7 +467,6 @@
}
stage('Install Kubernetes control for kdt') {
- salt.enforceStateWithTest([saltId: venvPepper, target: "I@kubernetes:master ${extra_tgt}", state: 'kubernetes.master.kube-addons'])
salt.enforceStateWithTest([saltId: venvPepper, target: "I@kubernetes:master ${extra_tgt}", state: 'kubernetes.pool.images'])
orchestrate.installKubernetesControl(venvPepper, extra_tgt)
@@ -572,7 +571,6 @@
stage('Install Cicd on kdt') {
extra_tgt_bckp = extra_tgt
extra_tgt = 'and kdt* ' + extra_tgt_bckp
- orchestrate.installInfra(venvPepper, extra_tgt)
orchestrate.installCicd(venvPepper, extra_tgt)
extra_tgt = extra_tgt_bckp
}
diff --git a/test-cookiecutter-reclass-chunk.groovy b/test-cookiecutter-reclass-chunk.groovy
index 8c804a1..de30e65 100644
--- a/test-cookiecutter-reclass-chunk.groovy
+++ b/test-cookiecutter-reclass-chunk.groovy
@@ -15,7 +15,7 @@
extraVars = readYaml text: EXTRA_VARIABLES_YAML
try {
currentBuild.description = extraVars.modelFile
- sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
+ sh(script: 'find . -mindepth 1 -delete || true', returnStatus: true)
sh(script: """
wget --progress=dot:mega --auth-no-challenge -O models.tar.gz ${extraVars.MODELS_TARGZ}
tar -xzf models.tar.gz
@@ -25,14 +25,14 @@
def content = readFile(file: extraVars.modelFile)
def templateContext = readYaml text: content
def config = [
- 'dockerHostname': "cfg01",
- 'domain': "${templateContext.default_context.cluster_domain}",
- 'clusterName': templateContext.default_context.cluster_name,
- 'reclassEnv': extraVars.testReclassEnv,
- 'distribRevision': extraVars.DISTRIB_REVISION,
+ 'dockerHostname' : "cfg01",
+ 'domain' : "${templateContext.default_context.cluster_domain}",
+ 'clusterName' : templateContext.default_context.cluster_name,
+ 'reclassEnv' : extraVars.testReclassEnv,
+ 'distribRevision' : extraVars.DISTRIB_REVISION,
'dockerContainerName': extraVars.DockerCName,
- 'testContext': extraVars.modelFile,
- 'dockerExtraOpts': [ '--memory=3g' ]
+ 'testContext' : extraVars.modelFile,
+ 'dockerExtraOpts' : ['--memory=3g']
]
if (extraVars.DISTRIB_REVISION == 'nightly') {
config['nodegenerator'] = true
@@ -50,9 +50,9 @@
} finally {
stage('Save artifacts to Artifactory') {
def artifactory = new com.mirantis.mcp.MCPArtifactory()
- def envGerritVars = [ "GERRIT_PROJECT=${extraVars.get('GERRIT_PROJECT', '')}", "GERRIT_CHANGE_NUMBER=${extraVars.get('GERRIT_CHANGE_NUMBER', '')}",
- "GERRIT_PATCHSET_NUMBER=${extraVars.get('GERRIT_PATCHSET_NUMBER', '')}", "GERRIT_CHANGE_ID=${extraVars.get('GERRIT_CHANGE_ID', '')}",
- "GERRIT_PATCHSET_REVISION=${extraVars.get('GERRIT_PATCHSET_REVISION', '')}" ]
+ def envGerritVars = ["GERRIT_PROJECT=${extraVars.get('GERRIT_PROJECT', '')}", "GERRIT_CHANGE_NUMBER=${extraVars.get('GERRIT_CHANGE_NUMBER', '')}",
+ "GERRIT_PATCHSET_NUMBER=${extraVars.get('GERRIT_PATCHSET_NUMBER', '')}", "GERRIT_CHANGE_ID=${extraVars.get('GERRIT_CHANGE_ID', '')}",
+ "GERRIT_PATCHSET_REVISION=${extraVars.get('GERRIT_PATCHSET_REVISION', '')}"]
withEnv(envGerritVars) {
def artifactoryLink = artifactory.uploadJobArtifactsToArtifactory(['artifactory': 'mcp-ci', 'artifactoryRepo': "drivetrain-local/${JOB_NAME}/${BUILD_NUMBER}"])
currentBuild.description += "<br/>${artifactoryLink}"
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index aa695f2..b7004f6 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -209,6 +209,12 @@
gerritDataRSHEAD << gerritDataRS
gerritDataRSHEAD['gerritRefSpec'] = null
gerritDataRSHEAD['GERRIT_CHANGE_NUMBER'] = null
+ // check for test XXX vs RELEASE branch, to get correct formulas
+ if (gerritDataCC['gerritBranch'].contains('release/')) {
+ testDistribRevision = gerritDataCC['gerritBranch']
+ } else if (gerritDataRS['gerritBranch'].contains('release')) {
+ testDistribRevision = gerritDataRS['gerritBranch']
+ }
// 'binary' branch logic w\o 'release/' prefix
if (testDistribRevision.contains('/')) {
testDistribRevision = testDistribRevision.split('/')[-1]
@@ -219,8 +225,8 @@
if (!binTest.linux_system_repo_url || !binTest.linux_system_repo_ubuntu_url) {
common.errorMsg("Binary release: ${testDistribRevision} not exist or not full. Fallback to 'proposed'! ")
testDistribRevision = 'proposed'
- messages.add("DISTRIB_REVISION => ${testDistribRevision}")
}
+ messages.add("DISTRIB_REVISION => ${testDistribRevision}")
def message = messages.join(newline) + newline
currentBuild.description = currentBuild.description ? message + currentBuild.description : message
}
@@ -390,7 +396,7 @@
result = '\n' + common.comparePillars(compareRoot, env.BUILD_URL, "-Ev \'infra/secrets.yml|\\.git\'")
currentBuild.description = currentBuild.description ? currentBuild.description + result : result
}
- stage("TestContexts Head/Patched") {
+ stage('TestContexts Head/Patched') {
def stepsForParallel = [:]
stepsForParallel.failFast = true
common.infoMsg("Found: ${contextFileListHead.size()} HEAD contexts to test.")
@@ -406,7 +412,7 @@
parallel stepsForParallel
common.infoMsg('All TestContexts tests done')
}
- stage("Compare NodesInfo Head/Patched") {
+ stage('Compare NodesInfo Head/Patched') {
// Download all artifacts
def stepsForParallel = [:]
stepsForParallel.failFast = true
@@ -480,3 +486,4 @@
}
}
}
+
diff --git a/test-model-generator.groovy b/test-model-generator.groovy
index 39723c6..aff8b8a 100644
--- a/test-model-generator.groovy
+++ b/test-model-generator.groovy
@@ -158,6 +158,7 @@
export TEST_PASSWORD=default
export TEST_MODELD_URL=127.0.0.1
export TEST_MODELD_PORT=3000
+ export TEST_TIMEOUT=30
cd /var/lib/trymcp-tests
pytest -m 'not trymcp' ${component}
"""
@@ -188,11 +189,18 @@
}
sh "rm -rf ${env.WORKSPACE}/venv/"
}
- if (apiImage && apiImage.id) {
- sh "docker rmi ${apiImage.id}"
- }
- if (uiImage && uiImage.id) {
- sh "docker rmi ${uiImage.id}"
+ try {
+ // to avoid issue PROD-29393 "pipeline freezes in `docker rmi` action"
+ timeout(time: 4, unit: 'MINUTES') {
+ if (apiImage && apiImage.id) {
+ sh "docker rmi ${apiImage.id}"
+ }
+ if (uiImage && uiImage.id) {
+ sh "docker rmi ${uiImage.id}"
+ }
+ }
+ } catch (Exception e) {
+ echo "Failed while cleaning docker images: ${e.toString()}"
}
// Remove everything what is owned by root
testImage.inside(testImageOptions) {
diff --git a/update-package.groovy b/update-package.groovy
index 9d36f38..df7655b 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -6,23 +6,12 @@
* SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* TARGET_PACKAGES Space delimited list of packages to be updates [package1=version package2=version], empty string means all updating all packages to the latest version.
- * TARGET_SUBSET_TEST Number of nodes to list package updates, empty string means all targetted nodes.
- * TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
- * TARGET_BATCH_LIVE Batch size for the complete live package update on all nodes, empty string means apply to all targetted nodes.
*
**/
+
pepperEnv = "pepperEnv"
salt = new com.mirantis.mk.Salt()
-def common = new com.mirantis.mk.Common()
-def python = new com.mirantis.mk.Python()
-def targetTestSubset
-def targetLiveSubset
-def targetLiveAll
-def minions
-def result
-def packages
-def command
-def commandKwargs
+common = new com.mirantis.mk.Common()
def installSaltStack(target, pkgs, masterUpdate = false){
salt.cmdRun(pepperEnv, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
@@ -31,131 +20,76 @@
// in case of update Salt Master packages - check all minions are good
minions_reachable = '*'
}
- salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': target, 'target_reachable': minions_reachable])
+ salt.checkTargetMinionsReady(['saltId': pepperEnv, 'target': target, 'target_reachable': minions_reachable])
}
timeout(time: 12, unit: 'HOURS') {
node() {
try {
-
+ def python = new com.mirantis.mk.Python()
+ def command = 'pkg.upgrade'
+ def commandKwargs = null
+ def packages = null
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
- stage('List target servers') {
- minions = salt.getMinions(pepperEnv, TARGET_SERVERS)
-
+ def targetLiveAll = ''
+ stage('Get target servers') {
+ def minions = salt.getMinions(pepperEnv, TARGET_SERVERS)
if (minions.isEmpty()) {
throw new Exception("No minion was targeted")
}
-
- if (TARGET_SUBSET_TEST != "") {
- targetTestSubset = minions.subList(0, Integer.valueOf(TARGET_SUBSET_TEST)).join(' or ')
- } else {
- targetTestSubset = minions.join(' or ')
- }
- targetLiveSubset = minions.subList(0, Integer.valueOf(TARGET_SUBSET_LIVE)).join(' or ')
-
targetLiveAll = minions.join(' or ')
common.infoMsg("Found nodes: ${targetLiveAll}")
- common.infoMsg("Selected test nodes: ${targetTestSubset}")
- common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
}
stage("List package upgrades") {
- common.infoMsg("Listing all the packages that have a new update available on test nodes: ${targetTestSubset}")
- salt.runSaltProcessStep(pepperEnv, targetTestSubset, 'pkg.list_upgrades', [], null, true)
- if(TARGET_PACKAGES != "" && TARGET_PACKAGES != "*"){
- common.infoMsg("Note that only the ${TARGET_PACKAGES} would be installed from the above list of available updates on the ${targetTestSubset}")
- }
- }
-
- stage('Confirm live package upgrades on sample') {
- if(TARGET_PACKAGES==""){
- timeout(time: 2, unit: 'HOURS') {
- def userInput = input(
- id: 'userInput', message: 'Insert package names for update', parameters: [
- [$class: 'TextParameterDefinition', defaultValue: '', description: 'Package names (or *)', name: 'packages']
- ])
- if(userInput!= "" && userInput!= "*"){
- TARGET_PACKAGES = userInput
- }
- }
- }else{
- timeout(time: 2, unit: 'HOURS') {
- input message: "Approve live package upgrades on ${targetLiveSubset} nodes?"
- }
- }
- }
-
- if (TARGET_PACKAGES != "") {
- command = "pkg.install"
- packages = TARGET_PACKAGES.tokenize(' ')
- commandKwargs = ['only_upgrade': 'true']
- }else {
- command = "pkg.upgrade"
- packages = null
- }
-
- stage('Apply package upgrades on sample') {
- if(packages == null || packages.contains("salt-master") || packages.contains("salt-common") || packages.contains("salt-minion") || packages.contains("salt-api")){
- def saltTargets = (targetLiveSubset.split(' or ').collect{it as String})
- for(int i = 0; i < saltTargets.size(); i++ ){
- common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
- common.retry(10, 5) {
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
- }
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
- }
- }
- }
- }
- out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, packages, commandKwargs)
- salt.printSaltCommandResult(out)
- for(value in out.get("return")[0].values()){
- if (value.containsKey('result') && value.result == false) {
- throw new Exception("The package upgrade on sample node has failed. Please check the Salt run result above for more information.")
- }
+ common.infoMsg("Listing all the packages that have a new update available on nodes: ${targetLiveAll}")
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'pkg.list_upgrades', [], null, true)
+ if (TARGET_PACKAGES != '' && TARGET_PACKAGES != '*') {
+ common.warningMsg("Note that only the \"${TARGET_PACKAGES}\" would be installed from the above list of available updates on the ${targetLiveAll}")
+ command = "pkg.install"
+ packages = TARGET_PACKAGES.tokenize(' ')
+ commandKwargs = ['only_upgrade': 'true']
}
}
stage('Confirm package upgrades on all nodes') {
timeout(time: 2, unit: 'HOURS') {
- input message: "Approve live package upgrades on ${targetLiveAll} nodes?"
+ input message: "Approve package upgrades on ${targetLiveAll} nodes?"
}
}
stage('Apply package upgrades on all nodes') {
-
- if(packages == null || packages.contains("salt-master") || packages.contains("salt-common") || packages.contains("salt-minion") || packages.contains("salt-api")){
- def saltTargets = (targetLiveAll.split(' or ').collect{it as String})
- for(int i = 0; i < saltTargets.size(); i++ ){
- common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
+ if (packages == null || packages.contains("salt-master") || packages.contains("salt-common") || packages.contains("salt-minion") || packages.contains("salt-api")) {
+ common.warningMsg('Detected update for some Salt package (master or minion). Updating it first.')
+ def saltTargets = (targetLiveAll.split(' or ').collect { it as String })
+ for (int i = 0; i < saltTargets.size(); i++ ) {
common.retry(10, 5) {
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
+ if (salt.getMinions(pepperEnv, "I@salt:master and ${saltTargets[i]}")) {
installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
- }
- if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
+ } else if (salt.getMinions(pepperEnv, "I@salt:minion and not I@salt:master and ${saltTargets[i]}")) {
installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
+ } else {
+ error("Minion ${saltTargets[i]} is not reachable!")
}
}
}
}
-
+ common.infoMsg('Starting package upgrades...')
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, packages, commandKwargs)
salt.printSaltCommandResult(out)
for(value in out.get("return")[0].values()){
if (value.containsKey('result') && value.result == false) {
- throw new Exception("The package upgrade on sample node has failed. Please check the Salt run result above for more information.")
+ throw new Exception("The package upgrade on nodes has failed. Please check the Salt run result above for more information.")
}
}
- common.warningMsg("Pipeline has finished successfully, but please, check if any packages have been kept back.")
}
+ common.warningMsg("Pipeline has finished successfully, but please, check if any packages have been kept back.")
+
} catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
currentBuild.result = "FAILURE"
currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
throw e
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 02e9270..08d4783 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -88,9 +88,9 @@
sh "diff -u ${workspace}/${oldSuffix}/${minion} ${workspace}/${newSuffix}/${minion} > ${fileName} || true"
}
}
- archiveArtifacts artifacts: "${workspace}/${oldSuffix}"
- archiveArtifacts artifacts: "${workspace}/${newSuffix}"
- archiveArtifacts artifacts: "${workspace}/${diffDir}"
+ archiveArtifacts artifacts: "${oldSuffix}/*"
+ archiveArtifacts artifacts: "${newSuffix}/*"
+ archiveArtifacts artifacts: "${diffDir}/*"
}
if (common.validInputParam('PIPELINE_TIMEOUT')) {
@@ -206,12 +206,31 @@
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout ${reclassSystemBranch}")
// Add kubernetes-extra repo
if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
+ // docker-engine conflicts with the recent containerd versions, so it's removed during upgrade. Thus update source engine
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
+ "grep -r -l 'engine: docker_hybrid' kubernetes | xargs --no-run-if-empty sed -i 's/engine: docker_hybrid/engine: archive/g'")
common.infoMsg("Add kubernetes-extra repo")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -q system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra' kubernetes/common.yml")
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -q system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra' kubernetes/common.yml")
}
+ // Add all update repositories
+ def repoIncludeBase = '- system.linux.system.repo.mcp.apt_mirantis.'
+ def updateRepoList = [ 'cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'percona', 'salt-formulas', 'saltstack', 'ubuntu' ]
+ updateRepoList.each { repo ->
+ def repoNameUpdateInclude = "${repoIncludeBase}update.${repo}"
+ def filesWithInclude = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -Plr '\\${repoIncludeBase}${repo}\$' . || true", false).get('return')[0].values()[0].trim().tokenize('\n')
+ filesWithInclude.each { file ->
+ def updateRepoIncludeExist = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -P '\\${repoNameUpdateInclude}\$' ${file} || echo not_found", false, null, true).get('return')[0].values()[0].trim()
+ if (updateRepoIncludeExist == 'not_found') {
+ // Include needs to be added
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
+ "sed -i 's/\\( *\\)${repoIncludeBase}${repo}\$/&\\n\\1${repoNameUpdateInclude}/g' ${file}")
+ common.infoMsg("Update repo for ${repo} is added to ${file}")
+ }
+ }
+ }
// Add new defaults
common.infoMsg("Add new defaults")
salt.cmdRun(venvPepper, 'I@salt:master', "grep '^ mcp_version: ' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +