opencontrail-upgrade.groovy

/**
 * Update packages on given nodes
 *
 * Expected parameters:
 *   SALT_MASTER_CREDENTIALS        Credentials to the Salt API.
 *   SALT_MASTER_URL                Full Salt API address [http://10.10.10.1:8000].
 *   STAGE_CONTROLLERS_UPGRADE      Run upgrade on Opencontrail controllers  (bool)
 *   STAGE_ANALYTICS_UPGRADE        Run upgrade on Opencontrail analytics  (bool)
 *   STAGE_COMPUTES_UPGRADE         Run upgrade on Opencontrail compute nodes  (bool)
 *   COMPUTE_TARGET_SERVERS         Salt compound target to match nodes to be updated [*, G@osfamily:debian].
 *   COMPUTE_TARGET_SUBSET_LIVE     Number of selected nodes to live apply selected package update.
 *
**/

def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()

def saltMaster
def targetTestSubset
def targetLiveSubset
def targetLiveAll
def minions
def result
def args
def command
def commandKwargs
def probe = 1

timestamps {
    node() {

        stage('Connect to Salt API') {
            saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
        }

        if (STAGE_CONTROLLERS_UPGRADE.toBoolean() == true) {
            // # actual upgrade

            stage('Opencontrail controllers upgrade') {

                oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)

                //oc_component_repo = salt.cmdRun(saltMaster, 'I@opencontrail:control and *01*', 'grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\'')
                //print oc_component_repo
                oc_component_repo = oc_component_repo['return'][0].values()[0]
                print oc_component_repo

                try {
                    //salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm ${oc_component_repo}")
                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
                    salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
                } catch (Exception er) {
                    common.errorMsg('Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
                    return
                }

                try {
                    salt.cmdRun(saltMaster, 'I@opencontrail:control', "su root -c '/usr/local/bin/zookeeper-backup-runner.sh'")
                } catch (Exception er) {
                    common.errorMsg('Zookeeper failed to backup. Please fix it before continuing.')
                    return
                }

                try {
                    salt.cmdRun(saltMaster, 'I@cassandra:backup:client', "su root -c '/usr/local/bin/cassandra-backup-runner-call.sh'")
                } catch (Exception er) {
                    common.errorMsg('Cassandra failed to backup. Please fix it before continuing.')
                    return
                }

                // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'

                // database // while sleeping nodetool-status
                command = 'cmd.shell'
                args = 'apt install contrail-database -y;'
                check = 'nodetool status'

                // ntw01
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw02
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw03
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."


                // pkgs // while sleeping contrail-status  // pridat jeste 20 sekund sleep
                args = 'apt install -o Dpkg::Options::=\"--force-confold\" contrail-config contrail-config-openstack contrail-control contrail-dns contrail-lib contrail-nodemgr contrail-utils contrail-web-controller contrail-web-core neutron-plugin-contrail python-contrail -y --force-yes;'
                check = 'contrail-status'

                // ntw01
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *01*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw02
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *02*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw03
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control and *03*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."


                // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
                try {
                    salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
                } catch (Exception er) {
                    common.errorMsg('Opencontrail state was executed on I@opencontrail:control and failed please fix it manually.')
                }

                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)

                common.warningMsg('Please check \'show bgp summary\' on your bgp router if all bgp peers are in healthy state.')
            }
        }

        if (STAGE_ANALYTICS_UPGRADE.toBoolean() == true) {

            stage('Ask for manual confirmation') {
                input message: "Do you want to continue with the Opencontrail analytic nodes upgrade?"
            }

            stage('Opencontrail analytics upgrade') {

                oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)

                //oc_component_repo = salt.cmdRun(saltMaster, 'I@opencontrail:collector and *01*', 'grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\'')
                //print oc_component_repo
                oc_component_repo = oc_component_repo['return'][0].values()[0]
                print oc_component_repo

                try {
                    //salt.cmdRun(saltMaster, 'I@opencontrail:collector', "rm ${oc_component_repo}")
                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
                    salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
                } catch (Exception er) {
                    common.errorMsg('Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
                    return
                }

                // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'

                // database // while sleeping nodetool-status
                command = 'cmd.shell'
                args = 'apt install contrail-database -y;'
                check = 'nodetool status'

                // po apt install contrail-database tak 2 min pockat nez pojedu dalsi nod
                // ntw01
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw02
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw03
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."


                // pkgs // while sleeping contrail-status  // pridat jeste 20 sekund sleep
                args = 'apt install -o Dpkg::Options::="--force-confold" contrail-analytics contrail-lib contrail-nodemgr contrail-utils python-contrail -y --force-yes;'
                check = 'contrail-status'

                // ntw01
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *01*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw02
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *02*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."

                // ntw03
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, args, null)
                salt.printSaltCommandResult(out)
                sleep(30)
                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector and *03*', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
                input message: "Please check the output of ${check} and continue if it is correct."


                // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
                try {
                    salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
                } catch (Exception er) {
                    common.errorMsg('Opencontrail state was executed on I@opencontrail:collector and failed please fix it manually.')
                }

                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
                salt.printSaltCommandResult(out)
            }

        }

        if (STAGE_COMPUTES_UPGRADE.toBoolean() == true) {

            try {

                stage('List targeted compute servers') {
                    minions = salt.getMinions(saltMaster, COMPUTE_TARGET_SERVERS)

                    if (minions.isEmpty()) {
                        throw new Exception("No minion was targeted")
                    }

                    targetLiveSubset = minions.subList(0, Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)).join(' or ')
                    targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')

                    targetLiveAll = minions.join(' or ')
                    common.infoMsg("Found nodes: ${targetLiveAll}")
                    common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
                }

                stage('Confirm upgrade on sample nodes') {
                    input message: "Do you want to continue with the Opencontrail compute upgrade on the following sample nodes? ${targetLiveSubset}"
                }

                stage("Opencontrail compute upgrade on sample nodes") {

                    oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
                    oc_component_repo = oc_component_repo['return'][0].values()[0]
                    print oc_component_repo

                    try {
                        salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
                        salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
                        salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
                    } catch (Exception er) {
                        common.errorMsg('Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
                        return
                    }

                    // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'

                    command = 'cmd.shell'
                    args = 'export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms -y;'
                    check = 'contrail-status'

                    // pkgs // while sleeping contrail-status  // pridat jeste 20 sekund sleep
                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
                    salt.printSaltCommandResult(out)
                    sleep(30)

                    // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
                    try {
                        salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
                    } catch (Exception er) {
                        common.errorMsg('Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.')
                    }

                    salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;"], null, true)
                    sleep(30)

                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
                    salt.printSaltCommandResult(out)
                }

                stage('Confirm upgrade on all targeted nodes') {
                    timeout(time: 2, unit: 'HOURS') {
                       input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
                    }
                }
                stage("Opencontrail compute upgrade on all targeted nodes") {

                    oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -R \'oc\\([0-9]*\\)\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
                    oc_component_repo = oc_component_repo['return'][0].values()[0]
                    print oc_component_repo

                    try {
                        salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
                        salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
                        salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
                    } catch (Exception er) {
                        common.errorMsg('Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.')
                        return
                    }

                    // args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" dist-upgrade;'

                    command = 'cmd.shell'
                    args = 'export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms -y;'
                    check = 'contrail-status'

                    // pkgs // while sleeping contrail-status  // pridat jeste 20 sekund sleep
                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
                    salt.printSaltCommandResult(out)
                    sleep(30)

                    // co se stane kdyz tohle neprojede a ma to fixnout manualne? Pokracovat nebo ne nebo co?
                    try {
                        salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
                    } catch (Exception er) {
                        common.errorMsg('Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.')
                    }

                    sleep(3)
                    salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;"], null, true)
                    sleep(30)

                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
                    salt.printSaltCommandResult(out)
                }

            } catch (Throwable e) {
                // If there was an error or exception thrown, the build failed
                currentBuild.result = "FAILURE"
                throw e
            }
        }
    }
}