contrail upgrade
Change-Id: I038da45a7fe600f1fccb39a323b4ea3667dc17e1
diff --git a/opencontrail-upgrade.groovy b/opencontrail-upgrade.groovy
new file mode 100644
index 0000000..f59dd27
--- /dev/null
+++ b/opencontrail-upgrade.groovy
@@ -0,0 +1,363 @@
+/**
+ * Update packages on given nodes
+ *
+ * Expected parameters:
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
+ * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
+ * STAGE_CONTROLLERS_UPGRADE Run upgrade on Opencontrail controllers (bool)
+ * STAGE_ANALYTICS_UPGRADE Run upgrade on Opencontrail analytics (bool)
+ * STAGE_COMPUTES_UPGRADE Run upgrade on Opencontrail compute nodes (bool)
+ * COMPUTE_TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
+ * COMPUTE_TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
+ *
+**/
+
+def common = new com.mirantis.mk.Common()
+def salt = new com.mirantis.mk.Salt()
+
+def saltMaster
+def targetTestSubset
+def targetLiveSubset
+def targetLiveAll
+def minions
+def result
+def args
+def command
+def commandKwargs
+def probe = 1
+
+timestamps {
+ node() {
+
+ stage('Connect to Salt API') {
+ saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ if (STAGE_CONTROLLERS_UPGRADE.toBoolean() == true) {
+ // # actual upgrade
+
+ stage('Opencontrail controllers upgrade') {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+
+ //oc_component_repo = salt.cmdRun(saltMaster, 'I@opencontrail:control and *01*', 'grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\'')
+ //print oc_component_repo
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+ print oc_component_repo
+
+ try {
+ //salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm ${oc_component_repo}")
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ return
+ }
+
+ try {
+ salt.cmdRun(saltMaster, 'I@opencontrail:control', "su root -c '/usr/local/bin/zookeeper-backup-runner.sh'")
+ } catch (Exception er) {
+ common.errorMsg('Zookeeper failed to backup. Please fix it before continuing.')
+ return
+ }
+
+ try {
+ salt.cmdRun(saltMaster, 'I@cassandra:backup:client', "su root -c '/usr/local/bin/cassandra-backup-runner-call.sh'")
+ } catch (Exception er) {
+ common.errorMsg('Cassandra failed to backup. Please fix it before continuing.')
+ return
+ }
+
+ // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
+
+ // database // while sleeping nodetool-status
+ command = 'cmd.shell'
+ args = 'apt install contrail-database -y;'
+ check = 'nodetool status'
+
+ // ntw01
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw02
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw03
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+
+ // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
+ args = 'apt install -o Dpkg::Options::=\"--force-confold\" contrail-config contrail-config-openstack contrail-control contrail-dns contrail-lib contrail-nodemgr contrail-utils contrail-web-controller contrail-web-core neutron-plugin-contrail python-contrail -y --force-yes;'
+ check = 'contrail-status'
+
+ // ntw01
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw02
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw03
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+
+ // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
+ try {
+ salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail state was executed on I@opencontrail:control and failed please fix it manually.')
+ }
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+
+ common.warningMsg('Please check \'show bgp summary\' on your bgp router if all bgp peers are in healthy state.')
+ }
+ }
+
+ if (STAGE_ANALYTICS_UPGRADE.toBoolean() == true) {
+
+ stage('Ask for manual confirmation') {
+ input message: "Do you want to continue with the Opencontrail analytic nodes upgrade?"
+ }
+
+ stage('Opencontrail analytics upgrade') {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+
+ //oc_component_repo = salt.cmdRun(saltMaster, 'I@opencontrail:collector and *01*', 'grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\'')
+ //print oc_component_repo
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+ print oc_component_repo
+
+ try {
+ //salt.cmdRun(saltMaster, 'I@opencontrail:collector', "rm ${oc_component_repo}")
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ return
+ }
+
+ // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
+
+ // database // while sleeping nodetool-status
+ command = 'cmd.shell'
+ args = 'apt install contrail-database -y;'
+ check = 'nodetool status'
+
+ // po apt install contrail-database tak 2 min pockat nez pojedu dalsi nod
+ // ntw01
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw02
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw03
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+
+ // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
+ args = 'apt install -o Dpkg::Options::="--force-confold" contrail-analytics contrail-lib contrail-nodemgr contrail-utils python-contrail -y --force-yes;'
+ check = 'contrail-status'
+
+ // ntw01
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw02
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+ // ntw03
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ input message: "Please check the output of ${check} and continue if it is correct."
+
+
+ // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
+ try {
+ salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail state was executed on I@opencontrail:collector and failed please fix it manually.')
+ }
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ }
+
+ }
+
+ if (STAGE_COMPUTES_UPGRADE.toBoolean() == true) {
+
+ try {
+
+ stage('List targeted compute servers') {
+ minions = salt.getMinions(saltMaster, COMPUTE_TARGET_SERVERS)
+
+ if (minions.isEmpty()) {
+ throw new Exception("No minion was targeted")
+ }
+
+ targetLiveSubset = minions.subList(0, Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)).join(' or ')
+ targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')
+
+ targetLiveAll = minions.join(' or ')
+ common.infoMsg("Found nodes: ${targetLiveAll}")
+ common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
+ }
+
+ stage('Confirm upgrade on sample nodes') {
+ input message: "Do you want to continue with the Opencontrail compute upgrade on the following sample nodes? ${targetLiveSubset}"
+ }
+
+ stage("Opencontrail compute upgrade on sample nodes") {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+ print oc_component_repo
+
+ try {
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ return
+ }
+
+ // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
+
+ command = 'cmd.shell'
+ args = 'export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms -y;'
+ check = 'contrail-status'
+
+ // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+
+ // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
+ try {
+ salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.')
+ }
+
+ salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;"], null, true)
+ sleep(30)
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ }
+
+ stage('Confirm upgrade on all targeted nodes') {
+ timeout(time: 2, unit: 'HOURS') {
+ input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
+ }
+ }
+ stage("Opencontrail compute upgrade on all targeted nodes") {
+
+ oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = oc_component_repo['return'][0].values()[0]
+ print oc_component_repo
+
+ try {
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
+ return
+ }
+
+ // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'
+
+ command = 'cmd.shell'
+ args = 'export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms -y;'
+ check = 'contrail-status'
+
+ // pkgs // while sleeping contrail-status // pridat jeste 20 sekund sleep
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
+ salt.printSaltCommandResult(out)
+ sleep(30)
+
+ // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
+ try {
+ salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
+ } catch (Exception er) {
+ common.errorMsg('Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.')
+ }
+
+ sleep(3)
+ salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;"], null, true)
+ sleep(30)
+
+ out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
+ salt.printSaltCommandResult(out)
+ }
+
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+ }
+ }
+}