oc and ovs gtw upgrade
Change-Id: I62fb13519b7683ec1ce9681aa150d46306f94c0d
diff --git a/opencontrail-upgrade.groovy b/opencontrail-upgrade.groovy
index f59dd27..02c2bc1 100644
--- a/opencontrail-upgrade.groovy
+++ b/opencontrail-upgrade.groovy
@@ -9,6 +9,9 @@
* STAGE_COMPUTES_UPGRADE Run upgrade on Opencontrail compute nodes (bool)
* COMPUTE_TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* COMPUTE_TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
+ * STAGE_CONTROLLERS_ROLLBACK Run rollback on Opencontrail controllers (bool)
+ * STAGE_ANALYTICS_ROLLBACK Run rollback on Opencontrail analytics (bool)
+ * STAGE_COMPUTES_ROLLBACK Run rollback on Opencontrail compute nodes (bool)
*
**/
@@ -16,15 +19,31 @@
def salt = new com.mirantis.mk.Salt()
def saltMaster
-def targetTestSubset
def targetLiveSubset
def targetLiveAll
def minions
def result
def args
-def command
def commandKwargs
def probe = 1
+def errorOccured = false
+def command = 'cmd.shell'
+
+def CONTROL_PKGS = 'contrail-config contrail-config-openstack contrail-control contrail-dns contrail-lib contrail-nodemgr contrail-utils contrail-web-controller contrail-web-core neutron-plugin-contrail python-contrail'
+def ANALYTIC_PKGS = 'contrail-analytics contrail-lib contrail-nodemgr contrail-utils python-contrail'
+def CMP_PKGS = 'contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms'
+def KERNEL_MODULE_RELOAD = 'service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;'
+
+
+def void runCommonCommands(target, command, args, check, salt, saltMaster, common) {
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': target, 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(60)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': target, 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of \'${check}\' and continue if it is correct."
+}
timestamps {
node() {
@@ -33,25 +52,21 @@
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
- if (STAGE_CONTROLLERS_UPGRADE.toBoolean() == true) {
- // # actual upgrade
+ if (STAGE_CONTROLLERS_UPGRADE.toBoolean() == true && !errorOccured) {
stage('Opencontrail controllers upgrade') {
- oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
- //oc_component_repo = salt.cmdRun(saltMaster, 'I@opencontrail:control and *01*', 'grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\'')
- //print oc_component_repo
oc_component_repo = oc_component_repo['return'][0].values()[0]
- print oc_component_repo
try {
- //salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm ${oc_component_repo}")
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg('Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ errorOccured = true
+ common.errorMsg("Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
return
}
@@ -69,68 +84,26 @@
return
}
- // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
-
- // database // while sleeping nodetool-status
- command = 'cmd.shell'
args = 'apt install contrail-database -y;'
check = 'nodetool status'
// ntw01
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
// ntw02
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
// ntw03
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
+ runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
-
- // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
- args = 'apt install -o Dpkg::Options::=\"--force-confold\" contrail-config contrail-config-openstack contrail-control contrail-dns contrail-lib contrail-nodemgr contrail-utils contrail-web-controller contrail-web-core neutron-plugin-contrail python-contrail -y --force-yes;'
+ args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
check = 'contrail-status'
// ntw01
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
// ntw02
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
// ntw03
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
+ runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
-
- // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
try {
salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
} catch (Exception er) {
@@ -144,7 +117,7 @@
}
}
- if (STAGE_ANALYTICS_UPGRADE.toBoolean() == true) {
+ if (STAGE_ANALYTICS_UPGRADE.toBoolean() == true && !errorOccured) {
stage('Ask for manual confirmation') {
input message: "Do you want to continue with the Opencontrail analytic nodes upgrade?"
@@ -152,86 +125,40 @@
stage('Opencontrail analytics upgrade') {
- oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
- //oc_component_repo = salt.cmdRun(saltMaster, 'I@opencontrail:collector and *01*', 'grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\'')
- //print oc_component_repo
oc_component_repo = oc_component_repo['return'][0].values()[0]
- print oc_component_repo
try {
- //salt.cmdRun(saltMaster, 'I@opencontrail:collector', "rm ${oc_component_repo}")
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg('Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ errorOccured = true
+ common.errorMsg("Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
return
}
- // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
-
- // database // while sleeping nodetool-status
- command = 'cmd.shell'
args = 'apt install contrail-database -y;'
check = 'nodetool status'
- // po apt install contrail-database tak 2 min pockat nez pojedu dalsi nod
// ntw01
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
// ntw02
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
// ntw03
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
+ runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
-
- // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
- args = 'apt install -o Dpkg::Options::="--force-confold" contrail-analytics contrail-lib contrail-nodemgr contrail-utils python-contrail -y --force-yes;'
+ args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
check = 'contrail-status'
// ntw01
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
// ntw02
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
-
+ runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
// ntw03
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, args, null)
- salt.printSaltCommandResult(out)
- sleep(30)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of ${check} and continue if it is correct."
+ runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
-
- // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
try {
salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
} catch (Exception er) {
@@ -241,10 +168,9 @@
out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
salt.printSaltCommandResult(out)
}
-
}
- if (STAGE_COMPUTES_UPGRADE.toBoolean() == true) {
+ if (STAGE_COMPUTES_UPGRADE.toBoolean() == true && !errorOccured) {
try {
@@ -269,85 +195,277 @@
stage("Opencontrail compute upgrade on sample nodes") {
- oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
oc_component_repo = oc_component_repo['return'][0].values()[0]
- print oc_component_repo
try {
salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg('Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ errorOccured = true
+ common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
return
}
- // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
-
- command = 'cmd.shell'
- args = 'export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms -y;'
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
check = 'contrail-status'
- // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
salt.printSaltCommandResult(out)
- sleep(30)
- // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
try {
salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
} catch (Exception er) {
- common.errorMsg('Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.')
+ common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
}
- salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;"], null, true)
- sleep(30)
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+ sleep(10)
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
salt.printSaltCommandResult(out)
}
stage('Confirm upgrade on all targeted nodes') {
- timeout(time: 2, unit: 'HOURS') {
- input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
- }
+ input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
}
stage("Opencontrail compute upgrade on all targeted nodes") {
- oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
oc_component_repo = oc_component_repo['return'][0].values()[0]
- print oc_component_repo
try {
salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
} catch (Exception er) {
- common.errorMsg('Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
return
}
- // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
-
- command = 'cmd.shell'
- args = 'export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms -y;'
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
check = 'contrail-status'
- // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
salt.printSaltCommandResult(out)
- sleep(30)
- // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
try {
salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
} catch (Exception er) {
- common.errorMsg('Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.')
+ common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
}
- sleep(3)
- salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;"], null, true)
- sleep(30)
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+ sleep(10)
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ }
+
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+ }
+
+
+ if (STAGE_CONTROLLERS_ROLLBACK.toBoolean() == true && !errorOccured) {
+
+ stage('Ask for manual confirmation') {
+ input message: "Do you want to continue with the Opencontrail control nodes rollback?"
+ }
+
+ stage('Opencontrail controllers rollback') {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+ try {
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
+ } catch (Exception er) {
+ errorOccured = true
+ common.errorMsg("Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ return
+ }
+
+ args = 'apt install contrail-database -y --force-yes;'
+ check = 'nodetool status'
+
+ // ntw01
+ runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
+ // ntw02
+ runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
+ // ntw03
+ runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
+
+ args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
+ check = 'contrail-status'
+
+ // ntw01
+ runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
+ // ntw02
+ runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
+ // ntw03
+ runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
+
+ try {
+ salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail state was executed on I@opencontrail:control and failed please fix it manually.')
+ }
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+
+ common.warningMsg('Please check \'show bgp summary\' on your bgp router if all bgp peers are in healthy state.')
+ }
+ }
+
+ if (STAGE_ANALYTICS_ROLLBACK.toBoolean() == true && !errorOccured) {
+
+ stage('Ask for manual confirmation') {
+ input message: "Do you want to continue with the Opencontrail analytic nodes rollback?"
+ }
+
+ stage('Opencontrail analytics rollback') {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+ try {
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
+ } catch (Exception er) {
+ errorOccured = true
+ common.errorMsg("Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ return
+ }
+
+ args = 'apt install contrail-database -y --force-yes;'
+ check = 'nodetool status'
+
+ // ntw01
+ runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
+ // ntw02
+ runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
+ // ntw03
+ runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
+
+ args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
+ check = 'contrail-status'
+
+ // ntw01
+ runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
+ // ntw02
+ runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
+ // ntw03
+ runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
+
+ try {
+ salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail state was executed on I@opencontrail:collector and failed please fix it manually.')
+ }
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ }
+ }
+
+ if (STAGE_COMPUTES_ROLLBACK.toBoolean() == true && !errorOccured) {
+
+ try {
+
+ stage('List targeted compute servers') {
+ minions = salt.getMinions(saltMaster, COMPUTE_TARGET_SERVERS)
+
+ if (minions.isEmpty()) {
+ throw new Exception("No minion was targeted")
+ }
+
+ targetLiveSubset = minions.subList(0, Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)).join(' or ')
+ targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')
+
+ targetLiveAll = minions.join(' or ')
+ common.infoMsg("Found nodes: ${targetLiveAll}")
+ common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
+ }
+
+ stage('Confirm rollback on sample nodes') {
+ input message: "Do you want to continue with the Opencontrail compute rollback on the following sample nodes? ${targetLiveSubset}"
+ }
+
+ stage("Opencontrail compute rollback on sample nodes") {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+ try {
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
+ } catch (Exception er) {
+ errorOccured = true
+ common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ return
+ }
+
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
+ check = 'contrail-status'
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+
+ try {
+ salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
+ }
+
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+ sleep(10)
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ }
+
+ stage('Confirm rollback on all targeted nodes') {
+ input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
+ }
+
+ stage("Opencontrail compute upgrade on all targeted nodes") {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+ try {
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
+ } catch (Exception er) {
+ common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+ return
+ }
+
+ args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
+ check = 'contrail-status'
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+
+ try {
+ salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
+ }
+
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+ sleep(10)
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
salt.printSaltCommandResult(out)