Merge "Revert "Set salt test pipeline status to ABORTED in case of skip these tests""
diff --git a/cicd-lab-pipeline.groovy b/cicd-lab-pipeline.groovy
index fe99ecb..3ac17b5 100644
--- a/cicd-lab-pipeline.groovy
+++ b/cicd-lab-pipeline.groovy
@@ -167,7 +167,7 @@
// services are deployed
// XXX: for some weird unknown reason, refresh_pillar is
// required to execute here
- salt.runSaltProcessStep(master, 'I@aptly:publisher', 'saltutil.refresh_pillar', [], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@aptly:publisher', 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(saltMaster, 'I@aptly:publisher', 'aptly.publisher', true)
retry(3) {
sleep(5)
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index c04e5d2..7d331bf 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -209,6 +209,10 @@
print(kubernetes_control_address)
salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'reclass.cluster_meta_set', ['kubernetes_control_address', kubernetes_control_address], null, true)
+ // ensure certificates are generated properly
+ salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, '*', ['salt.minion.cert'], true)
+
orchestrate.installKubernetesInfra(saltMaster)
}
diff --git a/git-mirror-2way-pipeline.groovy b/git-mirror-2way-pipeline.groovy
index 63308ee..c1c808c 100644
--- a/git-mirror-2way-pipeline.groovy
+++ b/git-mirror-2way-pipeline.groovy
@@ -13,7 +13,7 @@
checkout changelog: true, poll: true,
scm: [$class: 'GitSCM', branches: pollBranches, doGenerateSubmoduleConfigurations: false,
extensions: [[$class: 'CleanCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: CREDENTIALS_ID, url: TARGET_URL]]]
- } catch(AbortException e){
+ } catch(hudson.AbortException e){
if(e.message.trim().equals("Couldn't find any revision to build. Verify the repository and branch configuration for this job.")){
common.warningMsg("Warning: Cannot checkout target repo source repo is empty")
} else {
@@ -26,7 +26,7 @@
checkout changelog: true, poll: true,
scm: [$class: 'GitSCM', branches: pollBranches, doGenerateSubmoduleConfigurations: false,
extensions: [[$class: 'CleanCheckout']], submoduleCfg: [], userRemoteConfigs: [[credentialsId: CREDENTIALS_ID, url: SOURCE_URL]]]
- } catch(AbortException e){
+ } catch(hudson.AbortException e){
if(e.message.trim().equals("Couldn't find any revision to build. Verify the repository and branch configuration for this job.")){
common.warningMsg("Warning: Cannot checkout source repo source repo is empty")
} else {
diff --git a/openstack-compute-upgrade.groovy b/openstack-compute-upgrade.groovy
index cde839c..33e0825 100644
--- a/openstack-compute-upgrade.groovy
+++ b/openstack-compute-upgrade.groovy
@@ -5,10 +5,8 @@
* SALT_MASTER_CREDENTIALS Credentials to the Salt API.
* SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
- * TARGET_PACKAGES Space delimited list of packages to be updates [package1=version package2=version], empty string means all updating all packages to the latest version.
* TARGET_SUBSET_TEST Number of nodes to list package updates, empty string means all targetted nodes.
* TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
- * TARGET_BATCH_LIVE Batch size for the complete live package update on all nodes, empty string means apply to all targetted nodes.
*
**/
@@ -22,9 +20,10 @@
def targetLiveAll
def minions
def result
-def packages
+def args
def command
def commandKwargs
+def probe = 1
node() {
try {
@@ -46,6 +45,8 @@
targetTestSubset = minions.join(' or ')
}
targetLiveSubset = minions.subList(0, Integer.valueOf(TARGET_SUBSET_LIVE)).join(' or ')
+ targetTestSubsetProbe = minions.subList(0, probe).join(' or ')
+ targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')
targetLiveAll = minions.join(' or ')
common.infoMsg("Found nodes: ${targetLiveAll}")
@@ -53,60 +54,146 @@
common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
}
- stage("Add new repos on sample") {
- salt.enforceState(saltMaster, targetTestSubset, 'linux.system')
+
+ stage("Add new repos on test nodes") {
+ salt.enforceState(saltMaster, targetTestSubset, 'linux.system.repo')
+ }
+
+
+ opencontrail = null
+
+ try {
+ opencontrail = salt.cmdRun(saltMaster, targetTestSubsetProbe, "salt-call grains.item roles | grep opencontrail.compute")
+ print(opencontrail)
+ } catch (Exception er) {
+ common.infoMsg("opencontrail is not used")
+ }
+
+ if(opencontrail != null) {
+ stage('Remove OC component from repos on test nodes') {
+ salt.cmdRun(saltMaster, targetTestSubset, "find /etc/apt/sources.list* -type f -print0 | xargs -0 sed -i -r -e 's/ oc([0-9]*) / /g'")
+ salt.runSaltProcessStep(saltMaster, targetTestSubset, 'pkg.refresh_db', [], null, true)
+ }
}
stage("List package upgrades") {
salt.runSaltProcessStep(saltMaster, targetTestSubset, 'pkg.list_upgrades', [], null, true)
}
- stage('Confirm live package upgrades on sample') {
- if(TARGET_PACKAGES==""){
- timeout(time: 2, unit: 'HOURS') {
- def userInput = input(
- id: 'userInput', message: 'Insert package names for update', parameters: [
- [$class: 'TextParameterDefinition', defaultValue: '', description: 'Package names (or *)', name: 'packages']
- ])
- if(userInput!= "" && userInput!= "*"){
- TARGET_PACKAGES = userInput
- }
- }
- }else{
- timeout(time: 2, unit: 'HOURS') {
- input message: "Approve live package upgrades on ${targetLiveSubset} nodes?"
- }
+ stage('Confirm upgrade on sample nodes') {
+ input message: "Please verify the list of packages that you want to be upgraded. Do you want to continue with upgrade?"
+ }
+
+ stage("Add new repos on sample nodes") {
+ salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
+ }
+
+ if(opencontrail != null) {
+ stage('Remove OC component from repos on sample nodes') {
+ salt.cmdRun(saltMaster, targetLiveSubset, "find /etc/apt/sources.list* -type f -print0 | xargs -0 sed -i -r -e 's/ oc([0-9]*) / /g'")
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'pkg.refresh_db', [], null, true)
}
}
- if (TARGET_PACKAGES != "") {
- command = "pkg.install"
- packages = TARGET_PACKAGES.tokenize(' ')
- commandKwargs = ['only_upgrade': 'true']
- }else {
- command = "pkg.upgrade"
- packages = null
+ args = "apt-get -y -s -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade"
+
+ stage('Test upgrade on sample') {
+ try {
+ salt.cmdRun(saltMaster, targetLiveSubset, args)
+ } catch (Exception er) {
+ print(er)
+ }
}
+ stage('Confirm upgrade on sample') {
+ input message: "Please verify if there are packages that it wants to downgrade. If so, execute apt-cache policy on them and verify if everything is fine. Do you want to continue with upgrade?"
+ }
+
+ command = "cmd.run"
+ args = "apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade"
+
stage('Apply package upgrades on sample') {
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, packages, commandKwargs)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, commandKwargs)
salt.printSaltCommandResult(out)
}
- stage('Confirm package upgrades on all nodes') {
- timeout(time: 2, unit: 'HOURS') {
- input message: "Approve live package upgrades on ${targetLiveAll} nodes?"
+ openvswitch = null
+
+ try {
+ openvswitch = salt.cmdRun(saltMaster, targetLiveSubsetProbe, "salt-call grains.item roles | grep neutron.compute")
+ } catch (Exception er) {
+ common.infoMsg("openvswitch is not used")
+ }
+
+ if(openvswitch != null) {
+ args = "sudo /usr/share/openvswitch/scripts/ovs-ctl start"
+
+ stage('Start ovs on sample nodes') {
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, commandKwargs)
+ salt.printSaltCommandResult(out)
+ }
+ stage("Run salt states on sample nodes and reboot them") {
+ salt.enforceState(saltMaster, targetLiveSubset, ['nova', 'neutron'])
+ try {
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'system.reboot', null, null, true, 5)
+ } catch (Exception er) {
+ common.infoMsg("The following nodes did not return anything because they were rebooted: ${targetLiveSubset}")
+ }
+ }
+ } else {
+ stage("Run salt states on sample nodes") {
+ salt.enforceState(saltMaster, targetLiveSubset, ['nova'])
+ salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
}
}
+ stage('Confirm upgrade on all nodes') {
+ timeout(time: 2, unit: 'HOURS') {
+ input message: "Verify that the upgraded sample nodes are working correctly. If so, do you want to approve live upgrade on ${targetLiveAll} nodes?"
+ }
+ }
+
+ if(opencontrail != null) {
+ stage('Remove OC component from repos on sample nodes') {
+ salt.cmdRun(saltMaster, targetLiveAll, "find /etc/apt/sources.list* -type f -print0 | xargs -0 sed -i -r -e 's/ oc([0-9]*) / /g'")
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'pkg.refresh_db', [], null, true)
+ }
+ }
+
+ args = "apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade"
+
stage('Apply package upgrades on all nodes') {
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, packages, commandKwargs)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, commandKwargs)
salt.printSaltCommandResult(out)
}
+ if(openvswitch != null) {
+ args = "sudo /usr/share/openvswitch/scripts/ovs-ctl start"
+
+ stage('Start ovs on sample nodes') {
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, commandKwargs)
+ salt.printSaltCommandResult(out)
+ }
+ stage("Run salt states on sample nodes and reboot them") {
+ salt.enforceState(saltMaster, targetLiveAll, ['nova', 'neutron'])
+ try {
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'system.reboot', null, null, true, 5)
+ } catch (Exception er) {
+ common.infoMsg("The following nodes did not return anything because they were rebooted: ${targetLiveAll}")
+ }
+ }
+ } else {
+ stage("Run salt states on sample nodes") {
+ salt.enforceState(saltMaster, targetLiveAll, ['nova'])
+ salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
+ }
+ }
+
} catch (Throwable e) {
// If there was an error or exception thrown, the build failed
currentBuild.result = "FAILURE"
throw e
}
}
+
+
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 8e841b9..bff17f1 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -417,6 +417,24 @@
common.errorMsg("Stage Real control upgrade failed")
}
if(!errorOccured){
+
+ ceph = null
+
+ try {
+ ceph = salt.cmdRun(saltMaster, 'ctl*', "salt-call grains.item roles | grep ceph.client")
+
+ } catch (Exception er) {
+ common.infoMsg("Ceph is not used")
+ }
+
+ if(ceph != null) {
+ try {
+ salt.enforceState(saltMaster, 'ctl*', 'ceph.client')
+ } catch (Exception er) {
+ common.warningMsg("Ceph client state on controllers failed. Please fix it manually")
+ }
+ }
+
// salt 'cmp*' cmd.run 'service nova-compute restart'
salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)