Merge "Add kubernetes update pipeline"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 7f7337a..3fd2b8d 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -43,6 +43,8 @@
* which have to be added during bootstrap.
* Format: repo 1, repo priority 1, repo pin 1; repo 2, repo priority 2, repo pin 2;
+ * SALT_VERSION Version of Salt which is going to be installed i.e. 'stable 2016.3' or 'stable 2017.7' etc.
+ *
* Test settings:
* TEST_K8S_API_SERVER Kubernetes API address
* TEST_K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
@@ -208,6 +210,12 @@
}
}
+
+ if (common.validInputParam('SALT_VERSION')) {
+ common.infoMsg("Setting salt version to ${SALT_VERSION}")
+ envParams.put('cfg_saltversion', SALT_VERSION)
+ }
+
openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, venv)
}
diff --git a/cloud-update.groovy b/cloud-update.groovy
index fd7e6ff..07674e2 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -707,13 +707,26 @@
def verifyAPIs(pepperEnv, target) {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
- def out = salt.cmdRun(pepperEnv, target, '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
- if (out.toString().toLowerCase().contains('error')) {
- common.errorMsg(out)
- if (INTERACTIVE.toBoolean()) {
- input message: "APIs are not working as expected. Please fix it manually."
- } else {
- throw new Exception("APIs are not working as expected")
+ def cmds = ["openstack service list",
+ "openstack image list",
+ "openstack flavor list",
+ "openstack compute service list",
+ "openstack server list",
+ "openstack network list",
+ "openstack volume list",
+ "openstack orchestration service list"]
+ def sourcerc = ". /root/keystonercv3;"
+ def cmdOut = ">/dev/null 2>&1;echo \$?"
+ for (c in cmds) {
+ def command = sourcerc + c + cmdOut
+ def out = salt.cmdRun(pepperEnv, target, "${command}")
+ if (!out.toString().toLowerCase().contains('0')) {
+ common.errorMsg(out)
+ if (INTERACTIVE.toBoolean()) {
+ input message: "APIs are not working as expected. Please fix it manually."
+ } else {
+ throw new Exception("APIs are not working as expected")
+ }
}
}
}
@@ -1521,7 +1534,7 @@
if (merges.contains("log")) {
if (salt.testTarget(pepperEnv, LOG_TARGET)) {
- mergeSnapshot(pepperEnv, LOG_TARGET. 'log')
+ mergeSnapshot(pepperEnv, LOG_TARGET, 'log')
}
}
diff --git a/cvp-func.groovy b/cvp-func.groovy
index ef50945..d1fff1a 100644
--- a/cvp-func.groovy
+++ b/cvp-func.groovy
@@ -33,6 +33,7 @@
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
validate.runBasicContainer(saltMaster, TARGET_NODE, TEST_IMAGE)
sh "rm -rf ${artifacts_dir}"
+ salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
validate.configureContainer(saltMaster, TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO, TEMPEST_ENDPOINT_TYPE)
}
diff --git a/cvp-ha.groovy b/cvp-ha.groovy
index 99a49d3..ab5b5d4 100644
--- a/cvp-ha.groovy
+++ b/cvp-ha.groovy
@@ -39,6 +39,7 @@
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
validate.runBasicContainer(saltMaster, TEMPEST_TARGET_NODE, TEST_IMAGE)
sh "rm -rf ${artifacts_dir}"
+ salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
validate.configureContainer(saltMaster, TEMPEST_TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO)
}
diff --git a/cvp-perf.groovy b/cvp-perf.groovy
index d0c7156..fe86197 100644
--- a/cvp-perf.groovy
+++ b/cvp-perf.groovy
@@ -28,6 +28,7 @@
stage('Initialization') {
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
sh "rm -rf ${artifacts_dir}"
+ salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
validate.runBasicContainer(saltMaster, TARGET_NODE, TEST_IMAGE)
validate.configureContainer(saltMaster, TARGET_NODE, PROXY, TOOLS_REPO, "")
diff --git a/cvp-spt.groovy b/cvp-spt.groovy
new file mode 100644
index 0000000..ea4680f
--- /dev/null
+++ b/cvp-spt.groovy
@@ -0,0 +1,63 @@
+/**
+ *
+ * Launch pytest frameworks in Jenkins
+ *
+ * Expected parameters:
+ * SALT_MASTER_URL URL of Salt master
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API
+ *
+ * TESTS_SET Leave empty for full run or choose a file (test)
+ * TESTS_REPO Repo to clone
+ * TESTS_SETTINGS Additional environment varibales to apply
+ * PROXY Proxy to use for cloning repo or for pip
+ *
+ */
+
+validate = new com.mirantis.mcp.Validate()
+
+node() {
+ try{
+ stage('Initialization') {
+ validate.prepareVenv(TESTS_REPO, PROXY)
+ }
+
+ stage('Run Tests') {
+ validate.runTests(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS, TESTS_SET, '', TESTS_SETTINGS)
+ }
+ stage ('Publish results') {
+ archiveArtifacts artifacts: "*"
+ junit "*.xml"
+ plot csvFileName: 'plot-8634d2fe-dc48-4713-99f9-b69a381483aa.csv',
+ group: 'SPT',
+ style: 'line',
+ title: 'SPT Glance results',
+ xmlSeries: [[
+ file: "report.xml",
+ nodeType: 'NODESET',
+ url: '',
+ xpath: '/testsuite/testcase[@name="test_speed_glance"]/properties/property']]
+ plot csvFileName: 'plot-8634d2fe-dc48-4713-99f9-b69a381483bb.csv',
+ group: 'SPT',
+ style: 'line',
+ title: 'SPT HW2HW results',
+ xmlSeries: [[
+ file: "report.xml",
+ nodeType: 'NODESET',
+ url: '',
+ xpath: '/testsuite/testcase[@classname="cvp-spt.cvp_spt.tests.test_hw2hw"]/properties/property']]
+ plot csvFileName: 'plot-8634d2fe-dc48-4713-99f9-b69a381483bc.csv',
+ group: 'SPT',
+ style: 'line',
+ title: 'SPT VM2VM results',
+ xmlSeries: [[
+ file: "report.xml",
+ nodeType: 'NODESET',
+ url: '',
+ xpath: '/testsuite/testcase[@classname="cvp-spt.cvp_spt.tests.test_vm2vm"]/properties/property']]
+ }
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+}
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 0a2888e..6964653 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -169,7 +169,21 @@
stage("Test") {
if (sharedReclassUrl != "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
sh("cp -r ${modelEnv} ${testEnv}")
- saltModelTesting.setupAndTestNode("${saltMaster}.${clusterDomain}", "", testEnv)
+ def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
+ saltModelTesting.setupAndTestNode(
+ "${saltMaster}.${clusterDomain}",
+ "",
+ "",
+ testEnv,
+ 'pkg',
+ 'stable',
+ 'master',
+ 0,
+ false,
+ false,
+ '',
+ '',
+ DockerCName)
}
}
diff --git a/git-mirror-2way-pipeline.groovy b/git-mirror-2way-pipeline.groovy
index 21d72bf..bd9dcb8 100644
--- a/git-mirror-2way-pipeline.groovy
+++ b/git-mirror-2way-pipeline.groovy
@@ -6,7 +6,7 @@
try{
def branches = BRANCHES.tokenize(',')
def pollBranches = []
- for (i=0; i < branches.size; i++) {
+ for (i=0; i < branches.size(); i++) {
pollBranches.add([name:branches[i]])
}
dir("target") {
diff --git a/git-mirror-pipeline.groovy b/git-mirror-pipeline.groovy
index 45536d0..8bfe467 100644
--- a/git-mirror-pipeline.groovy
+++ b/git-mirror-pipeline.groovy
@@ -6,7 +6,7 @@
try{
def branches = BRANCHES.tokenize(',')
def pollBranches = []
- for (i=0; i < branches.size; i++) {
+ for (i=0; i < branches.size(); i++) {
pollBranches.add([name:branches[i]])
}
dir("source") {
diff --git a/openstack-compute-upgrade.groovy b/openstack-compute-upgrade.groovy
index a59fc08..2984b55 100644
--- a/openstack-compute-upgrade.groovy
+++ b/openstack-compute-upgrade.groovy
@@ -7,6 +7,7 @@
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* TARGET_SUBSET_TEST Number of nodes to list package updates, empty string means all targetted nodes.
* TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
*
**/
@@ -96,8 +97,10 @@
salt.runSaltProcessStep(pepperEnv, targetTestSubset, 'pkg.list_upgrades', [], null, true)
}
- stage('Confirm upgrade on sample nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample nodes') {
input message: "Please verify the list of packages that you want to be upgraded. Do you want to continue with upgrade?"
+ }
}
stage("Add new repos on sample nodes") {
@@ -136,8 +139,10 @@
}
}
- stage('Confirm upgrade on sample') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample') {
input message: "Please verify if there are packages that it wants to downgrade. If so, execute apt-cache policy on them and verify if everything is fine. Do you want to continue with upgrade?"
+ }
}
command = "cmd.run"
@@ -180,10 +185,12 @@
}
}
- stage('Confirm upgrade on all targeted nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on all targeted nodes') {
timeout(time: 2, unit: 'HOURS') {
input message: "Verify that the upgraded sample nodes are working correctly. If so, do you want to approve live upgrade on ${targetLiveAll} nodes?"
}
+ }
}
stage("Add new repos on all targeted nodes") {
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index d306e34..89b5e77 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -9,6 +9,7 @@
* STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
* SKIP_VM_RELAUNCH Set to true if vms should not be recreated (bool)
* OPERATING_SYSTEM_RELEASE_UPGRADE Set to true if operating system of vms should be upgraded to newer release (bool)
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
*
**/
@@ -195,10 +196,10 @@
salt.cmdRun(pepperEnv, "${test_upgrade_node}*", '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
- if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
- stage('Ask for manual confirmation') {
- input message: "Do you want to continue with upgrade?"
- }
+ if (INTERACTIVE.toBoolean() && STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
+ stage('Ask for manual confirmation') {
+ input message: "Do you want to continue with upgrade?"
+ }
}
}
@@ -303,8 +304,12 @@
stopServices(pepperEnv, proxy_node, proxy_general_target, 'prx')
stopServices(pepperEnv, control_node, control_general_target, 'ctl')
salt.printSaltCommandResult(out)
- if (out.toString().contains("dpkg returned an error code")) {
- input message: "Apt dist-upgrade failed, please fix it manually and then click on proceed. If unable to fix it, click on abort and run the rollback stage."
+ if (out.toString().contains("dpkg returned an error code")){
+ if (INTERACTIVE.toBoolean()) {
+ input message: "Apt dist-upgrade failed, please fix it manually and then click on proceed. If unable to fix it, click on abort and run the rollback stage."
+ } else {
+ error("Apt dist-upgrade failed. And interactive mode was disabled, failing...")
+ }
}
// run base states
try {
@@ -355,10 +360,14 @@
retryStateRun(pepperEnv, control_general_target, 'heat')
} catch (Exception e) {
errorOccured = true
- if (OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == false) {
- input message: "Some states that require syncdb failed. Please check the reason.Click proceed only if you want to restore database into it's pre-upgrade state. If you want restore production database and also the VMs into its pre-upgrade state please click on abort and run the rollback stage."
+ if (INTERACTIVE.toBoolean()){
+ if (OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == false) {
+ input message: "Some states that require syncdb failed. Please check the reason. Click proceed only if you want to restore database into it's pre-upgrade state. If you want restore production database and also the VMs into its pre-upgrade state please click on abort and run the rollback stage."
+ } else {
+ input message: "Some states that require syncdb failed. Please check the reason and click proceed only if you want to restore database into it's pre-upgrade state. Otherwise, click abort."
+ }
} else {
- input message: "Some states that require syncdb failed. Please check the reason and click proceed only if you want to restore database into it's pre-upgrade state. Otherwise, click abort."
+ error("Stage Real control upgrade failed. And interactive mode was disabled, failing...")
}
openstack.restoreGaleraDb(pepperEnv)
common.errorMsg("Stage Real control upgrade failed")
@@ -420,7 +429,9 @@
/*
if (OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == false) {
- input message: "Please verify if the control upgrade was successful! If so, by clicking proceed the original VMs disk images will be backed up and snapshot will be merged to the upgraded VMs which will finalize the upgrade procedure"
+ if (INTERACTIVE.toBoolean()){
+ input message: "Please verify if the control upgrade was successful! If so, by clicking proceed the original VMs disk images will be backed up and snapshot will be merged to the upgraded VMs which will finalize the upgrade procedure"
+ }
node_count = 1
for (t in proxy_target_hosts) {
def target = salt.stripDomainName(t)
@@ -445,8 +456,10 @@
virsh.liveSnapshotMerge(pepperEnv, nodeProvider, target, snapshotName)
node_count++
}
- input message: "Please scroll up and look for red highlighted messages containing 'virsh blockcommit' string.
- If there are any fix it manually. Otherwise click on proceed."
+ if (INTERACTIVE.toBoolean()){
+ input message: "Please scroll up and look for red highlighted messages containing 'virsh blockcommit' string.
+ If there are any fix it manually. Otherwise click on proceed."
+ }
}
*/
}
@@ -548,7 +561,7 @@
vcpRealUpgrade(pepperEnv)
}
- if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
+ if (INTERACTIVE.toBoolean() && STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
stage('Ask for manual confirmation') {
input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click on proceed to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
}
@@ -557,11 +570,13 @@
if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
stage('Rollback upgrade') {
- stage('Ask for manual confirmation') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Ask for manual confirmation') {
input message: "Before rollback please check the documentation for reclass model changes. Do you really want to continue with the rollback?"
+ }
}
vcpRollback(pepperEnv)
}
}
}
-}
\ No newline at end of file
+}
diff --git a/ovs-gateway-upgrade.groovy b/ovs-gateway-upgrade.groovy
index 8d131c8..87cf828 100644
--- a/ovs-gateway-upgrade.groovy
+++ b/ovs-gateway-upgrade.groovy
@@ -7,6 +7,7 @@
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* TARGET_SUBSET_TEST Number of nodes to list package updates, empty string means all targetted nodes.
* TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
*
**/
@@ -63,8 +64,10 @@
salt.runSaltProcessStep(pepperEnv, targetTestSubset, 'pkg.list_upgrades', [], null, true)
}
- stage('Confirm upgrade on sample nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample nodes') {
input message: "Please verify the list of packages that you want to be upgraded. Do you want to continue with upgrade?"
+ }
}
stage("Add new repos on sample nodes") {
@@ -81,8 +84,10 @@
}
}
- stage('Confirm upgrade on sample') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample') {
input message: "Please verify if there are packages that it wants to downgrade. If so, execute apt-cache policy on them and verify if everything is fine. Do you want to continue with upgrade?"
+ }
}
command = "cmd.run"
@@ -111,10 +116,12 @@
}
}
- stage('Confirm upgrade on all targeted nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on all targeted nodes') {
timeout(time: 2, unit: 'HOURS') {
- input message: "Verify that the upgraded sample nodes are working correctly. If so, do you want to approve live upgrade on ${targetLiveAll} nodes?"
+ input message: "Verify that the upgraded sample nodes are working correctly. If so, do you want to approve live upgrade on ${targetLiveAll} nodes?"
}
+ }
}
stage("Add new repos on all targeted nodes") {
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index ba05406..fedbd14 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -96,7 +96,21 @@
while (nbTry < 5) {
nbTry++
try {
- saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", clusterName, EXTRA_FORMULAS, testEnv, "pkg", DISTRIB_REVISION)
+ def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
+ saltModelTesting.setupAndTestNode(
+ "cfg01.${clusterDomain}",
+ clusterName,
+ EXTRA_FORMULAS,
+ testEnv,
+ 'pkg',
+ DISTRIB_REVISION,
+ 'master',
+ 0,
+ false,
+ false,
+ '',
+ '',
+ DockerCName)
break
} catch (Exception e) {
if (e.getMessage() == "script returned exit code 124") {
@@ -174,11 +188,10 @@
stage("test-nodes") {
def partitions = common.partitionList(contextFileList, PARALLEL_NODE_GROUP_SIZE.toInteger())
def buildSteps = [:]
- for (int i = 0; i < partitions.size(); i++) {
- def partition = partitions[i]
+ partitions.eachWithIndex { partition, i ->
buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
- for(int k = 0; k < partition.size; k++){
- def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
+ for(part in partition){
+ def basename = sh(script: "basename ${part} .yml", returnStdout: true).trim()
def testEnv = "${env.WORKSPACE}/model/${basename}"
buildSteps.get("partition-${i}").put(basename, { testModel(basename, testEnv) })
}
diff --git a/test-salt-formulas-pipeline.groovy b/test-salt-formulas-pipeline.groovy
index 401628b..ca4eb67 100644
--- a/test-salt-formulas-pipeline.groovy
+++ b/test-salt-formulas-pipeline.groovy
@@ -182,9 +182,6 @@
common.errorMsg("----------------KITCHEN LOG:---------------")
println readFile(".kitchen/logs/kitchen.log")
}
- def slack = new com.mirantis.mcp.SlackNotification()
- slack.jobResultNotification("success", "#test_reclass_notify")
- common.sendNotification(currentBuild.result, "", ["slack"])
}
}
}
diff --git a/test-salt-model-node.groovy b/test-salt-model-node.groovy
index 7fc333d..3b6ee09 100644
--- a/test-salt-model-node.groovy
+++ b/test-salt-model-node.groovy
@@ -64,7 +64,22 @@
def workspace = common.getWorkspace()
common.infoMsg("Running salt model test for node ${NODE_TARGET} in cluster ${CLUSTER_NAME}")
try {
- saltModelTesting.setupAndTestNode(NODE_TARGET, CLUSTER_NAME, EXTRA_FORMULAS, workspace, FORMULAS_SOURCE, FORMULAS_REVISION, RECLASS_VERSION, MAX_CPU_PER_JOB.toInteger(), RECLASS_IGNORE_CLASS_NOTFOUND, LEGACY_TEST_MODE, APT_REPOSITORY, APT_REPOSITORY_GPG)
+ def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
+
+ saltModelTesting.setupAndTestNode(
+ NODE_TARGET,
+ CLUSTER_NAME,
+ EXTRA_FORMULAS,
+ workspace,
+ FORMULAS_SOURCE,
+ FORMULAS_REVISION,
+ RECLASS_VERSION,
+ MAX_CPU_PER_JOB.toInteger(),
+ RECLASS_IGNORE_CLASS_NOTFOUND,
+ LEGACY_TEST_MODE,
+ APT_REPOSITORY,
+ APT_REPOSITORY_GPG,
+ DockerCName)
} catch (Exception e) {
if (e.getMessage() == "script returned exit code 124") {
common.errorMsg("Impossible to test node due to timeout of salt-master, ABORTING BUILD")
diff --git a/test-system-reclass-pipeline.groovy b/test-system-reclass-pipeline.groovy
index 7fc1181..fa16739 100644
--- a/test-system-reclass-pipeline.groovy
+++ b/test-system-reclass-pipeline.groovy
@@ -26,6 +26,7 @@
def checkouted = false
def merged = false
def systemRefspec = "HEAD"
+def formulasRevision = 'testing'
timeout(time: 12, unit: 'HOURS') {
node() {
try {
@@ -71,14 +72,17 @@
[$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: clusterGitUrl],
[$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: "HEAD"],
[$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: defaultGitUrl],
- [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec]
+ [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec],
+ [$class: 'StringParameterValue', name: 'FORMULAS_REVISION', value: formulasRevision],
]
}
}
branches["cookiecutter"] = {
build job: "test-mk-cookiecutter-templates", parameters: [
[$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: defaultGitUrl],
- [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec]
+ [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec],
+ [$class: 'StringParameterValue', name: 'DISTRIB_REVISION', value: formulasRevision]
+
]
}
parallel branches
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 0c5c971..08796c9 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -6,19 +6,19 @@
* SALT_MASTER_URL Salt API server location
* SALT_MASTER_CREDENTIALS Credentials to the Salt API
* MCP_VERSION Version of MCP to upgrade to
+ * UPGRADE_SALTSTACK Upgrade SaltStack packages to new version.
* UPDATE_CLUSTER_MODEL Update MCP version parameter in cluster model
* UPDATE_PIPELINES Update pipeline repositories on Gerrit
* UPDATE_LOCAL_REPOS Update local repositories
*/
-import hudson.model.*
-
// Load shared libs
salt = new com.mirantis.mk.Salt()
common = new com.mirantis.mk.Common()
python = new com.mirantis.mk.Python()
jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
venvPepper = "venvPepper"
+workspace = ""
def triggerMirrorJob(jobName){
params = jenkinsUtils.getJobParameters(jobName)
@@ -30,9 +30,44 @@
]
}
+def updateSaltStack(target, pkgs){
+ try{
+ salt.runSaltProcessStep(venvPepper, target, 'pkg.install', ["force_yes=True", "pkgs='$pkgs'"], null, true, 5)
+ }catch(Exception ex){}
+
+ common.retry(10, 30){
+ salt.minionsReachable(venvPepper, 'I@salt:master', '*')
+ def running = salt.runSaltProcessStep(venvPepper, target, 'saltutil.running', [], null, true, 5)
+ for(value in running.get("return")[0].values()){
+ if(value != []){
+ throw new Exception("Not all salt-minions are ready for execution")
+ }
+ }
+ }
+
+ def saltVersion = salt.getPillar(venvPepper, 'I@salt:master', "_param:salt_version").get("return")[0].values()[0]
+ def saltMinionVersions = salt.cmdRun(venvPepper, "*", "apt-cache policy salt-common | awk '/Installed/ && /$saltVersion/'").get("return")
+ def saltMinionVersion = ""
+
+ for(minion in saltMinionVersions[0].keySet()){
+ saltMinionVersion = saltMinionVersions[0].get(minion).replace("Salt command execution success","").trim()
+ if(saltMinionVersion == ""){
+ error("Installed version of Salt on $minion doesn't match specified version in the model.")
+ }
+ }
+}
+
+def archiveReclassInventory(filename){
+ def ret = salt.cmdRun(venvPepper, 'I@salt:master', "reclass -i", true, null, false)
+ def reclassInv = ret.values()[0]
+ writeFile file: filename, text: reclassInv.toString()
+ archiveArtifacts artifacts: "$filename"
+}
+
timeout(time: 12, unit: 'HOURS') {
node("python") {
try {
+ workspace = common.getWorkspace()
python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
if(MCP_VERSION == ""){
@@ -108,6 +143,16 @@
salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_salt.list")
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_salt.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades salt-formula-*")
+
+ def inventoryBeforeFilename = "reclass-inventory-before.out"
+ def inventoryAfterFilename = "reclass-inventory-after.out"
+
+ archiveReclassInventory(inventoryBeforeFilename)
+
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_extra.list")
+ salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_extra.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
+ salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades reclass")
+
salt.fullRefresh(venvPepper, 'I@salt:master')
try{
@@ -126,6 +171,19 @@
error("Reclass fails rendering. Pay attention to your cluster model.")
}
+ archiveReclassInventory(inventoryAfterFilename)
+
+ sh "diff -u $inventoryBeforeFilename $inventoryAfterFilename > reclass-inventory-diff.out || true"
+ archiveArtifacts artifacts: "reclass-inventory-diff.out"
+
+ if(UPGRADE_SALTSTACK.toBoolean()){
+ salt.enforceState(venvPepper, "I@linux:system", 'linux.system.repo', true)
+
+ updateSaltStack("I@salt:master", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
+
+ updateSaltStack("I@salt:minion and not I@salt:master", '["salt-minion"]')
+ }
+
if(UPDATE_PIPELINES.toBoolean()){
triggerMirrorJob("git-mirror-downstream-mk-pipelines")
triggerMirrorJob("git-mirror-downstream-pipeline-library")