cloud-update.groovy

/**
 * Update packages on given nodes
 *
 * Expected parameters:
 *   SALT_MASTER_CREDENTIALS    Credentials to the Salt API.
 *   SALT_MASTER_URL            Full Salt API address [https://10.10.10.1:8000].
 *   SNAPSHOT_NAME              Snapshot name
 *   CFG_NODE_PROVIDER          Physical machine name hosting Salt-Master VM (ex. kvm01*)
 *   INTERACTIVE                Ask interactive questions during pipeline run (bool)
 *   PER_NODE                   Target nodes will be managed one by one (bool)
 *   ROLLBACK_BY_REDEPLOY       Omit taking live snapshots. Rollback is planned to be done by redeployment (bool)
 *   STOP_SERVICES              Stop API services before update (bool)
 *   TARGET_UPDATES             Comma separated list of nodes to update (Valid values are cfg,ctl,prx,msg,dbs,log,mon,mtr,ntw,nal,gtw-virtual,cmn,rgw,cid,cmp,kvm,osd,gtw-physical)
 *   TARGET_ROLLBACKS           Comma separated list of nodes to rollback (Valid values are ctl,prx,msg,dbs,log,mon,mtr,ntw,nal,gtw-virtual,cmn,rgw,cmp,kvm,osd,gtw-physical)
 *   TARGET_SNAPSHOT_MERGES     Comma separated list of nodes to merge live snapshot for (Valid values are cfg,ctl,prx,msg,dbs,log,mon,mtr,ntw,nal,gtw-virtual,cmn,rgw,cid)
 *   CTL_TARGET                 Salt targeted CTL nodes (ex. ctl*)
 *   PRX_TARGET                 Salt targeted PRX nodes (ex. prx*)
 *   MSG_TARGET                 Salt targeted MSG nodes (ex. msg*)
 *   DBS_TARGET                 Salt targeted DBS nodes (ex. dbs*)
 *   LOG_TARGET                 Salt targeted LOG nodes (ex. log*)
 *   MON_TARGET                 Salt targeted MON nodes (ex. mon*)
 *   MTR_TARGET                 Salt targeted MTR nodes (ex. mtr*)
 *   NTW_TARGET                 Salt targeted NTW nodes (ex. ntw*)
 *   NAL_TARGET                 Salt targeted NAL nodes (ex. nal*)
 *   CMN_TARGET                 Salt targeted CMN nodes (ex. cmn*)
 *   RGW_TARGET                 Salt targeted RGW nodes (ex. rgw*)
 *   CID_TARGET                 Salt targeted CID nodes (ex. cid*)
 *   CMP_TARGET                 Salt targeted physical compute nodes (ex. cmp001*)
 *   KVM_TARGET                 Salt targeted physical KVM nodes (ex. kvm01*)
 *   CEPH_OSD_TARGET            Salt targeted physical Ceph OSD nodes (ex. osd001*)
 *   GTW_TARGET                 Salt targeted physical or virtual GTW nodes (ex. gtw01*)
 *   REBOOT                     Reboot nodes after update (bool)
 *   ROLLBACK_PKG_VERSIONS      Space separated list of pkgs=versions to rollback to on physical targeted machines (ex. pkg_name1=pkg_version1 pkg_name2=pkg_version2)
 *   PURGE_PKGS                 Space separated list of pkgs=versions to be purged on physical targeted machines (ex. pkg_name1=pkg_version1 pkg_name2=pkg_version2)
 *   REMOVE_PKGS                Space separated list of pkgs=versions to be removed on physical targeted machines (ex. pkg_name1=pkg_version1 pkg_name2=pkg_version2)
 *   RESTORE_GALERA             Restore Galera DB (bool)
 *   RESTORE_CONTRAIL_DB        Restore Cassandra and Zookeeper DBs for OpenContrail (bool)
 *
**/
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
def virsh = new com.mirantis.mk.Virsh()

def updates = TARGET_UPDATES.tokenize(",").collect{it -> it.trim()}
def rollbacks = TARGET_ROLLBACKS.tokenize(",").collect{it -> it.trim()}
def merges = TARGET_SNAPSHOT_MERGES.tokenize(",").collect{it -> it.trim()}

def pepperEnv = "pepperEnv"
def minions
def result
def packages
def command
def commandKwargs

def updatePkgs(pepperEnv, target, targetType="", targetPackages="") {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def commandKwargs
    def distUpgrade
    def pkgs
    def out

    salt.enforceState(pepperEnv, target, 'linux.system.repo')

    stage("List package upgrades") {
        common.infoMsg("Listing all the packages that have a new update available on ${target}")
        pkgs = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, target, 'pkg.list_upgrades', [], null, true))
        if(targetPackages != "" && targetPackages != "*"){
            common.infoMsg("Note that only the ${targetPackages} would be installed from the above list of available updates on the ${target}")
        }
    }

    if (INTERACTIVE.toBoolean()) {
        stage("Confirm live package upgrades on ${target}") {
            if (targetPackages=="") {
                def userInput = input(
                 id: 'userInput', message: 'Insert package names for update', parameters: [
                 [$class: 'TextParameterDefinition', defaultValue: pkgs.keySet().join(",").toString(), description: 'Package names (or *)', name: 'packages']
                ])
                if (userInput!= "" && userInput!= "*") {
                    targetPackages = userInput
                }
            } else {
                input message: "Approve live package upgrades on ${target} nodes?"
            }
        }
    } else {
        targetPackages = pkgs.keySet().join(",").toString()
    }

    if (targetPackages != "") {
        // list installed versions of pkgs that will be upgraded
        def installedPkgs = []
        def newPkgs = []
        def targetPkgList = targetPackages.tokenize(',')
        for (pkg in targetPkgList) {
            def version
            try {
                def pkgsDetails = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, target, 'pkg.info_installed', [pkg], null, true))
                version = pkgsDetails.get(pkg).get('version')
            } catch (Exception er) {
                common.infoMsg("${pkg} not installed yet")
            }
            if (version?.trim()) {
                installedPkgs.add(pkg + '=' + version)
            } else {
                newPkgs.add(pkg)
            }
        }
        common.warningMsg("the following list of pkgs will be upgraded")
        common.warningMsg(installedPkgs.join(" "))
        common.warningMsg("the following list of pkgs will be newly installed")
        common.warningMsg(newPkgs.join(" "))
        // set variables
        command = "pkg.install"
        packages = targetPackages
        commandKwargs = ['only_upgrade': 'true','force_yes': 'true']

    }else {
        command = "pkg.upgrade"
        commandKwargs = ['dist_upgrade': 'true']
        distUpgrade = true
        packages = null
    }

    // todo exception to cfg or cicd
    stage("stop services on ${target}") {
        if ((STOP_SERVICES.toBoolean()) && (targetType != 'cicd')) {
            if (targetType == 'contrail') {
                stopContrailServices(pepperEnv, target)
            } else {
                def probe = salt.getFirstMinion(pepperEnv, "${target}")
                stopServices(pepperEnv, probe, target)
            }
        }
    }

    stage('Apply package upgrades') {
        // salt master pkg
        if (targetType == 'I@salt:master') {
            common.warningMsg('salt-master pkg upgrade, rerun the pipeline if disconnected')
            salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-master'], null, true, 5)
            salt.minionsReachable(pepperEnv, 'I@salt:master', '*')
        }
        // salt minion pkg
        salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-minion'], null, true, 5)
        salt.minionsReachable(pepperEnv, 'I@salt:master', target)
        common.infoMsg('Performing pkg upgrades ... ')
        common.retry(3){
            out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, true, packages, commandKwargs)
            salt.printSaltCommandResult(out)
        }
        def osRelease = salt.getGrain(pepperEnv, target, 'lsb_distrib_codename')
        if (osRelease.toString().toLowerCase().contains('trusty')) {
            args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --force-yes -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" '
        } else {
            args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q -f --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" '
        }
        if (out.toString().contains('errors:')) {
            try {
                if (packages?.trim()) {
                    packages = packages.replaceAll(',', ' ')
                    common.retry(3){
                        out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' install ' + packages])
                    }
                } else {
                    if (distUpgrade) {
                        common.retry(3){
                            out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' dist-upgrade'])
                        }
                    } else {
                        common.retry(3){
                            out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' upgrade'])
                        }
                    }                    }
                if (out.toString().contains('E: ')) {
                    common.errorMsg(out)
                    if (INTERACTIVE.toBoolean()) {
                        input message: "Pkgs update failed to be updated on ${target}. Please fix it manually."
                    } else {
                        salt.printSaltCommandResult(out)
                        throw new Exception("Pkgs update failed")
                    }
                }
            } catch (Exception e) {
                common.errorMsg(out)
                common.errorMsg(e)
                if (INTERACTIVE.toBoolean()) {
                    input message: "Pkgs update failed to be updated on ${target}. Please fix it manually."
                } else {
                    throw new Exception("Pkgs update failed")
                }
            }
        }
    }
}

def rollbackPkgs(pepperEnv, target, targetType = "", targetPackages="") {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def probe = salt.getFirstMinion(pepperEnv, "${target}")
    def distUpgrade
    def pkgs
    def out

    salt.enforceState(pepperEnv, target, 'linux.system.repo')

    if (ROLLBACK_PKG_VERSIONS == "") {
        stage("List package upgrades") {
            common.infoMsg("Listing all the packages that have a new update available on ${target}")
            pkgs = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, target, 'pkg.list_upgrades', [], null, true))
            if(targetPackages != "" && targetPackages != "*"){
                common.infoMsg("Note that only the ${targetPackages} would be installed from the above list of available updates on the ${target}")
            }
        }

        if (INTERACTIVE.toBoolean()) {
            stage("Confirm live package upgrades on ${target}") {
                if(targetPackages==""){
                    timeout(time: 2, unit: 'HOURS') {
                        def userInput = input(
                         id: 'userInput', message: 'Insert package names for update', parameters: [
                         [$class: 'TextParameterDefinition', defaultValue: pkgs.keySet().join(",").toString(), description: 'Package names (or *)', name: 'packages']
                        ])
                        if(userInput!= "" && userInput!= "*"){
                            targetPackages = userInput
                        }
                    }
                }else{
                    timeout(time: 2, unit: 'HOURS') {
                       input message: "Approve live package upgrades on ${target} nodes?"
                    }
                }
            }
        } else {
            targetPackages = pkgs.keySet().join(",").toString()
        }
    } else {
        targetPackages = ROLLBACK_PKG_VERSIONS
    }

    if (targetPackages != "") {
        // set variables
        packages = targetPackages
    } else {
        distUpgrade = true
        packages = null
    }

    stage("stop services on ${target}") {
        try {
            if (INTERACTIVE.toBoolean()) {
                input message: "Click PROCEED to interactively stop services on ${target}. Otherwise click ABORT to skip stopping them and continue."
            }
        } catch (Exception er) {
            common.infoMsg('skipping stopping services')
            return
        }
        if (STOP_SERVICES.toBoolean()) {
            stopServices(pepperEnv, probe, target)
        }
    }

    stage('Apply package downgrades') {
        args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" '
        common.infoMsg('Performing pkgs purge/remove ... ')
        try {
            if (PURGE_PKGS != "") {
                def purgePackages = PURGE_PKGS.replaceAll(',', ' ')
                common.retry(3){
                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' purge ' + purgePackages])
                }
            }
            if (REMOVE_PKGS != "") {
                def removePackages = REMOVE_PKGS.replaceAll(',', ' ')
                common.retry(3){
                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' remove ' + removePackages])
                }
            }
            if (out.toString().contains('E: ')) {
                common.errorMsg(out)
                if (INTERACTIVE.toBoolean()) {
                    input message: "Pkgs ${packages} purge failed on ${target}. Please fix it manually."
                } else {
                    salt.printSaltCommandResult(out)
                    throw new Exception("Pkgs {packages} purge failed")
                }
            }
        } catch (Exception e) {
            common.errorMsg(out)
            common.errorMsg(e)
            if (INTERACTIVE.toBoolean()) {
                input message: "Pkgs {packages} purge on ${target}. Please fix it manually."
            } else {
                throw new Exception("Pkgs {packages} purge failed")
            }
        }

        common.infoMsg('Performing pkg downgrades ... ')
        try {
            packages = packages.replaceAll(',', ' ')
            if (packages?.trim()) {
                packages = packages.replaceAll(',', ' ')
                common.retry(3){
                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' install salt-minion'], null, true, 5)
                }
                salt.minionsReachable(pepperEnv, 'I@salt:master', target)
                common.retry(3){
                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' install ' + packages])
                }
            } else {
                if (distUpgrade) {
                    common.retry(3){
                        out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' dist-upgrade'])
                    }
                } else {
                    common.retry(3){
                        out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' upgrade'])
                    }
                }
            }
            if (out.toString().contains('E: ')) {
                common.errorMsg(out)
                if (INTERACTIVE.toBoolean()) {
                    input message: "Pkgs rollback failed on ${target}. Please fix it manually."
                } else {
                    salt.printSaltCommandResult(out)
                    throw new Exception("Pkgs rollback failed")
                }
            }
        } catch (Exception e) {
            common.errorMsg(out)
            common.errorMsg(e)
            if (INTERACTIVE.toBoolean()) {
                input message: "Pkgs rollback failed on ${target}. Please fix it manually."
            } else {
                throw new Exception("Pkgs rollback failed")
            }
        }
    }
}

def getCfgNodeProvider(pepperEnv, master_name) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    if (!CFG_NODE_PROVIDER?.trim()) {
        kvms = salt.getMinions(pepperEnv, 'I@salt:control')
        for (kvm in kvms) {
            try {
                vms = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, kvm, 'virt.list_active_vms', [], null, true))
                if (vms.toString().contains(master_name)) {
                    CFG_NODE_PROVIDER = kvm
                    //break
                }
            } catch (Exception er) {
                common.infoMsg("${master_name} not present on ${kvm}")
            }
        }
    }
}

/*
def rollbackSaltMaster(pepperEnv, target, path='/var/lib/libvirt/images') {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    try {
        input message: "PART1 - Are you sure to rollback ${target}? To rollback click on PROCEED. To skip rollback PART1 click on ABORT."
    } catch (Exception er) {
        common.infoMsg("skipping rollback of ${target}")
        return
    }
    def domain = salt.getDomainName(pepperEnv)
    def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
    getCfgNodeProvider(pepperEnv, master)
    try {
        try {
            salt.getReturnValues(salt.cmdRun(pepperEnv, CFG_NODE_PROVIDER, "ls -la ${path}/${master}.${domain}.xml"))
            common.errorMsg('Pipeline is about to disconnect from salt-api. You will have to rerun the pipeline with ROLLBACK_CFG checked and skip PART1 to finish rollback.')
            salt.cmdRun(pepperEnv, CFG_NODE_PROVIDER, "virsh destroy ${master}.${domain}; virsh define ${path}/${master}.${domain}.xml; virsh start ${master}.${domain} ")
        } catch (Exception er) {
            common.errorMsg(er)
            input message: "Rollback for ${target} failed. Rollback manually."
        }
    } catch (Exception er) {
        common.errorMsg(er)
        input message: "Rollback for ${target} failed. Rollback manually."
    }
}

def finishSaltMasterRollback(pepperEnv, target, path='/var/lib/libvirt/images') {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def virsh = new com.mirantis.mk.Virsh()
    try {
        input message: "PART2 - Are you sure to finalize ${target} rollback? Click on PROCEED. To skip rollback click on ABORT."
    } catch (Exception er) {
        common.infoMsg("skipping finalize rollback of ${target}")
        return
    }
    salt.minionsReachable(pepperEnv, 'I@salt:master', '*')
    def domain = salt.getDomainName(pepperEnv)
    def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
    getCfgNodeProvider(pepperEnv, master)
    try {
        virsh.liveSnapshotAbsent(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME, path)
        // purge and setup previous repos
        salt.enforceState(pepperEnv, target, 'linux.system.repo')
    } catch (Exception e) {
        common.errorMsg(e)
        input message: "Check what failed after ${target} rollback. Do you want to PROCEED?"
    }
}*/

def stopServices(pepperEnv, probe, target) {
    def openstack = new com.mirantis.mk.Openstack()
    def services = []
    services.add('keepalived')
    services.add('nginx')
    services.add('haproxy')
    services.add('nova-api')
    services.add('cinder')
    services.add('glance')
    services.add('heat')
    services.add('neutron')
    services.add('apache2')
    services.add('rabbitmq-server')
    if (INTERACTIVE.toBoolean()) {
        openstack.stopServices(pepperEnv, probe, target, services, true)
    } else {
        openstack.stopServices(pepperEnv, probe, target, services)
    }
}

// must be stopped separately due to OC on Trusty
def stopContrailServices(pepperEnv, target) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def services = []
    services.add('keepalived')
    services.add('haproxy')
    services.add('supervisor-control')
    services.add('supervisor-config')
    services.add('supervisor-database')
    services.add('zookeeper')
    services.add('ifmap-server')
    for (s in services) {
        try {
            salt.runSaltProcessStep(pepperEnv, target, 'service.stop', [s], null, true)
        } catch (Exception er) {
            common.infoMsg(er)
        }
    }
}

def highstate(pepperEnv, target) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()

    stage("Apply highstate on ${target} nodes") {
        try {
            common.retry(3){
                salt.enforceHighstate(pepperEnv, target)
            }
        } catch (Exception e) {
            common.errorMsg(e)
            if (INTERACTIVE.toBoolean()) {
                input message: "Highstate failed on ${target}. Fix it manually or run rollback on ${target}."
            } else {
                throw new Exception("highstate failed")
            }
        }
    }
    // optionally reboot
    if (REBOOT.toBoolean()) {
        stage("Reboot ${target} nodes") {
            salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
            sleep(10)
            salt.minionsReachable(pepperEnv, 'I@salt:master', target)
        }
    }
}

def rollback(pepperEnv, tgt, generalTarget) {
    def common = new com.mirantis.mk.Common()
    try {
        if (INTERACTIVE.toBoolean()) {
            input message: "Are you sure to rollback ${generalTarget}? To rollback click on PROCEED. To skip rollback click on ABORT."
        }
    } catch (Exception er) {
        common.infoMsg('skipping rollback')
        return
    }
    try {
        rollbackLiveSnapshot(pepperEnv, tgt, generalTarget)
    } catch (Exception err) {
        common.errorMsg(err)
        if (INTERACTIVE.toBoolean()) {
            input message: "Rollback for ${tgt} failed please fix it manually before clicking PROCEED."
        } else {
            throw new Exception("Rollback failed for ${tgt}")
        }
    }
}

def liveSnapshot(pepperEnv, tgt, generalTarget) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def virsh = new com.mirantis.mk.Virsh()
    def domain = salt.getDomainName(pepperEnv)
    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
    common.warningMsg(target_hosts)
    //def nodeCount = 1
    for (t in target_hosts) {
        def target = salt.stripDomainName(t)
        def nodeCount = target[4]
        common.warningMsg(nodeCount)
        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
        virsh.liveSnapshotPresent(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
        //nodeCount++
    }
}

def mergeSnapshot(pepperEnv, tgt, generalTarget='') {
    def salt = new com.mirantis.mk.Salt()
    def virsh = new com.mirantis.mk.Virsh()
    def domain = salt.getDomainName(pepperEnv)
    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
    def nodeCount = 1
    for (t in target_hosts) {
        if (tgt == 'I@salt:master') {
            def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
            getCfgNodeProvider(pepperEnv, master)
            virsh.liveSnapshotMerge(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME)
        } else {
            def target = salt.stripDomainName(t)
            def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
            virsh.liveSnapshotMerge(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
        }
        nodeCount++
    }
    salt.minionsReachable(pepperEnv, 'I@salt:master', tgt)
}



def rollbackLiveSnapshot(pepperEnv, tgt, generalTarget) {
    def salt = new com.mirantis.mk.Salt()
    def virsh = new com.mirantis.mk.Virsh()
    def common = new com.mirantis.mk.Common()
    def domain = salt.getDomainName(pepperEnv)
    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
    // first destroy all vms
    def nodeCount = 1
    for (t in target_hosts) {
        def target = salt.stripDomainName(t)
        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
        salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.destroy', ["${target}.${domain}"], null, true)
        nodeCount++
    }
    nodeCount = 1
    // rollback vms
    for (t in target_hosts) {
        def target = salt.stripDomainName(t)
        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
        virsh.liveSnapshotRollback(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
        nodeCount++
    }
    try {
        salt.minionsReachable(pepperEnv, 'I@salt:master', tgt)
        // purge and setup previous repos
        salt.enforceState(pepperEnv, tgt, 'linux.system.repo')
    } catch (Exception e) {
        common.errorMsg(e)
        if (INTERACTIVE.toBoolean()) {
            input message: "Salt state linux.system.repo on ${tgt} failed. Do you want to PROCEED?."
        } else {
            throw new Exception("Salt state linux.system.repo on ${tgt} failed")
        }
    }
}

def removeNode(pepperEnv, tgt, generalTarget) {
    def salt = new com.mirantis.mk.Salt()
    def virsh = new com.mirantis.mk.Virsh()
    def common = new com.mirantis.mk.Common()
    def domain = salt.getDomainName(pepperEnv)
    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
    // first destroy all vms
    def nodeCount = 1
    for (t in target_hosts) {
        def target = salt.stripDomainName(t)
        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
        salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.destroy', ["${target}.${domain}"], null, true)
        salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.undefine', ["${target}.${domain}"], null, true)
        try {
            salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key -d ${target}.${domain} -y")
        } catch (Exception e) {
            common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
        }
        nodeCount++
    }
}

def saltMasterBackup(pepperEnv) {
    def salt = new com.mirantis.mk.Salt()
    salt.enforceState(pepperEnv, 'I@salt:master', 'backupninja')
    salt.cmdRun(pepperEnv, 'I@salt:master', "su root -c 'backupninja -n --run /etc/backup.d/200.backup.rsync'")
}

def backupCeph(pepperEnv, tgt) {
    def salt = new com.mirantis.mk.Salt()
    salt.enforceState(pepperEnv, 'I@ceph:backup:server', 'ceph.backup')
    salt.enforceState(pepperEnv, "I@ceph:backup:client and ${tgt}", 'ceph.backup')
    salt.cmdRun(pepperEnv, "I@ceph:backup:client and ${tgt}", "su root -c '/usr/local/bin/ceph-backup-runner-call.sh -s'")
}

def backupGalera(pepperEnv) {
    def salt = new com.mirantis.mk.Salt()
    salt.enforceState(pepperEnv, 'I@xtrabackup:server', ['linux.system.repo', 'xtrabackup'])
    salt.enforceState(pepperEnv, 'I@xtrabackup:client', ['linux.system.repo', 'openssh.client'])
    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh -s -f'")
}

// cluster galera - wsrep_cluster_size
def clusterGalera(pepperEnv) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    try {
        salt.runSaltProcessStep(pepperEnv, 'I@galera:slave', 'service.stop', ['mysql'])
    } catch (Exception er) {
        common.warningMsg('Mysql service already stopped')
    }
    try {
        salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'service.stop', ['mysql'])
    } catch (Exception er) {
        common.warningMsg('Mysql service already stopped')
    }
    try {
        salt.cmdRun(pepperEnv, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
    } catch (Exception er) {
        common.warningMsg('Files are not present')
    }
    salt.cmdRun(pepperEnv, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
    salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'service.start', ['mysql'])
    // wait until mysql service on galera master is up
    try {
        salt.commandStatus(pepperEnv, 'I@galera:master', 'service mysql status', 'running')
    } catch (Exception er) {
        if (INTERACTIVE.toBoolean()) {
            input message: "Database is not running please fix it first and only then click on PROCEED."
        } else {
            throw new Exception("Database is not running correctly")
        }
    }
    salt.runSaltProcessStep(pepperEnv, 'I@galera:slave', 'service.start', ['mysql'])
}

def restoreGalera(pepperEnv) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def openstack = new com.mirantis.mk.Openstack()
    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "rm -rf /var/lib/mysql/*")
    openstack.restoreGaleraDb(pepperEnv)
}

def backupZookeeper(pepperEnv) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    salt.enforceState(pepperEnv, 'I@zookeeper:backup:server', 'zookeeper.backup')
    salt.enforceState(pepperEnv, 'I@zookeeper:backup:client', 'zookeeper.backup')
    try {
        salt.cmdRun(pepperEnv, 'I@opencontrail:control', "su root -c '/usr/local/bin/zookeeper-backup-runner.sh -s'")
    } catch (Exception er) {
        throw new Exception('Zookeeper failed to backup. Please fix it before continuing.')
    }
}

def backupCassandra(pepperEnv) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()

    salt.enforceState(pepperEnv, 'I@cassandra:backup:server', 'cassandra.backup')
    salt.enforceState(pepperEnv, 'I@cassandra:backup:client', 'cassandra.backup')
    try {
        salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', "su root -c '/usr/local/bin/cassandra-backup-runner-call.sh -s'")
    } catch (Exception er) {
        throw new Exception('Cassandra failed to backup. Please fix it before continuing.')
    }
}

def backupContrail(pepperEnv) {
    backupZookeeper(pepperEnv)
    backupCassandra(pepperEnv)
}

// cassandra and zookeeper
def restoreContrailDb(pepperEnv) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    build job: "deploy-zookeeper-restore", parameters: [
      [$class: 'StringParameterValue', name: 'SALT_MASTER_CREDENTIALS', value: SALT_MASTER_CREDENTIALS],
      [$class: 'StringParameterValue', name: 'SALT_MASTER_URL', value: SALT_MASTER_URL]
    ]
    build job: "deploy-cassandra-db-restore", parameters: [
      [$class: 'StringParameterValue', name: 'SALT_MASTER_CREDENTIALS', value: SALT_MASTER_CREDENTIALS],
      [$class: 'StringParameterValue', name: 'SALT_MASTER_URL', value: SALT_MASTER_URL]
    ]
}

def verifyAPIs(pepperEnv, target) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def out = salt.cmdRun(pepperEnv, target, '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
    if (out.toString().toLowerCase().contains('error')) {
        common.errorMsg(out)
        if (INTERACTIVE.toBoolean()) {
            input message: "APIs are not working as expected. Please fix it manually."
        } else {
            throw new Exception("APIs are not working as expected")
        }
    }
}

def verifyGalera(pepperEnv, target, count=0, maxRetries=200) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def out
    while(count < maxRetries) {
        try {
            out = salt.getReturnValues(salt.cmdRun(pepperEnv, target, 'salt-call mysql.status | grep -A1 wsrep_cluster_size'))
        } catch (Exception er) {
            common.infoMsg(er)
        }
        if ((!out.toString().contains('wsrep_cluster_size')) || (out.toString().contains('0'))) {
            count++
            if (count == maxRetries) {
                if (INTERACTIVE.toBoolean()) {
                    input message: "Galera is not working as expected. Please check it and fix it first before clicking on PROCEED."
                } else {
                    common.errorMsg(out)
                    throw new Exception("Galera is not working as expected")
                }
            }
            sleep(time: 500, unit: 'MILLISECONDS')
        } else {
            break
        }
    }
}

def verifyContrail(pepperEnv, target) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    salt.commandStatus(pepperEnv, target, "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
}


def verifyService(pepperEnv, target, service) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def targetHosts = salt.getMinionsSorted(pepperEnv, target)
    for (t in targetHosts) {
        try {
            salt.commandStatus(pepperEnv, t, "service ${service} status", 'running')
        } catch (Exception er) {
            common.errorMsg(er)
            if (INTERACTIVE.toBoolean()) {
                input message: "${service} service is not running correctly on ${t}. Please fix it first manually and only then click on PROCEED."
            } else {
                throw new Exception("${service} service is not running correctly on ${t}")
            }
        }
    }
}

def verifyCeph(pepperEnv, target, type) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def targetHosts = salt.getMinionsSorted(pepperEnv, target)
    for (t in targetHosts) {
        def hostname = salt.getReturnValues(salt.getPillar(pepperEnv, t, 'linux:network:hostname'))
        try {
            salt.commandStatus(pepperEnv, t, "systemctl status ceph-${type}${hostname}", 'running')
        } catch (Exception er) {
            common.errorMsg(er)
            if (INTERACTIVE.toBoolean()) {
                input message: "Ceph-${type}${hostname} service is not running correctly on ${t}. Please fix it first manually and only then click on PROCEED."
            } else {
                throw new Exception("Ceph-${type}${hostname} service is not running correctly on ${t}")
            }
        }
    }
}

def verifyCephOsds(pepperEnv, target) {
    def salt = new com.mirantis.mk.Salt()
    def common = new com.mirantis.mk.Common()
    def targetHosts = salt.getMinionsSorted(pepperEnv, target)
    for (t in targetHosts) {
        def osd_ids = []
        // get list of osd disks of the host
        salt.runSaltProcessStep(pepperEnv, t, 'saltutil.sync_grains', [], null, true, 5)
        def cephGrain = salt.getGrain(pepperEnv, t, 'ceph')
        if(cephGrain['return'].isEmpty()){
            throw new Exception("Ceph salt grain cannot be found!")
        }
        common.print(cephGrain)
        def ceph_disks = cephGrain['return'][0].values()[0].values()[0]['ceph_disk']
        for (i in ceph_disks) {
            def osd_id = i.getKey().toString()
            osd_ids.add('osd.' + osd_id)
            print("Will check osd." + osd_id)
        }
        for (i in osd_ids) {
            try {
                salt.commandStatus(pepperEnv, t, "ceph osd tree | grep -w ${i}", 'up')
            } catch (Exception er) {
                common.errorMsg(er)
                if (INTERACTIVE.toBoolean()) {
                    input message: "Ceph ${i} is not running correctly on ${t}. Please fix it first manually and only then click on PROCEED."
                } else {
                    throw new Exception("Ceph ${i} is not running correctly on ${t}")
                }
            }
        }
    }
}


timeout(time: 12, unit: 'HOURS') {
    node() {
        try {

            stage('Setup virtualenv for Pepper') {
                python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
            }

            // TODO, add possibility to update just specific components like kernel, openstack, contrail, ovs, rabbitmq, galera, etc.

            /*
                * Update section
            */
            if (updates.contains("cfg")) {
                def target = 'I@salt:master'
                if (salt.testTarget(pepperEnv, target)) {
                    def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
                    getCfgNodeProvider(pepperEnv, master)
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        virsh.liveSnapshotPresent(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME)
                    } else {
                        saltMasterBackup(pepperEnv)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t, target)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                }
            }

            if (updates.contains("ctl")) {
                def target = CTL_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'ctl'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyAPIs(pepperEnv, target)
                }
            }

            if (updates.contains("prx")) {
                def target = PRX_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'prx'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'nginx')
                }
            }

            if (updates.contains("msg")) {
                def target = MSG_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'msg'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'rabbitmq-server')
                }
            }

            if (updates.contains("dbs")) {
                def target = DBS_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    backupGalera(pepperEnv)
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'dbs'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (REBOOT.toBoolean() || PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                            verifyGalera(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                        verifyGalera(pepperEnv, target)
                    }
                }
            }

            if (updates.contains("ntw")) {
                def target = NTW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    backupContrail(pepperEnv)
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'ntw'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t, 'contrail')
                            highstate(pepperEnv, t)
                            verifyContrail(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target, 'contrail')
                        highstate(pepperEnv, target)
                        verifyContrail(pepperEnv, target)
                    }
                }
            }

            if (updates.contains("nal")) {
                def target = NAL_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'nal'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t, 'contrail')
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target, 'contrail')
                        highstate(pepperEnv, target)
                    }
                    verifyContrail(pepperEnv, target)
                }
            }

            if (updates.contains("gtw-virtual")) {
                def target = GTW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'gtw'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'neutron-dhcp-agent')
                }
            }

            if (updates.contains("cmn")) {
                def target = CMN_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'cmn'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    } else {
                        backupCeph(pepperEnv, target)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyCeph(pepperEnv, target, 'mon@')
                }
            }

            if (updates.contains("rgw")) {
                def target = RGW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'rgw'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyCeph(pepperEnv, target, 'radosgw@rgw.')
                }
            }

            if (updates.contains("log")) {
                def target = LOG_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'log'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                }
            }

            if (updates.contains("mon")) {
                def target = MON_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'mon'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                }
            }

            if (updates.contains("mtr")) {
                def target = MTR_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'mtr'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                }
            }

            if (updates.contains("cid")) {
                def target = CID_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        def generalTarget = 'cid'
                        liveSnapshot(pepperEnv, target, generalTarget)
                    }
                    updatePkgs(pepperEnv, target, 'cicd')
                    highstate(pepperEnv, target)
                    verifyService(pepperEnv, target, 'docker')
                }
            }

            //
            //physical machines update CMP_TARGET
            //
            if (updates.contains("cmp")) {
                def target = CMP_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'nova-compute')
                }
            }

            if (updates.contains("kvm")) {
                def target = KVM_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'libvirt-bin')
                }
            }

            if (updates.contains("osd")) {
                def target = CEPH_OSD_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyCephOsds(pepperEnv, target)
                }
            }

            if (updates.contains("gtw-physical")) {
                def target = GTW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            updatePkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        updatePkgs(pepperEnv, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'neutron-dhcp-agent')
                }
            }

            /*
                * Rollback section
            */
          /*  if (rollbacks.contains("cfg")) {
                if (salt.testTarget(pepperEnv, 'I@salt:master')) {
                    stage('ROLLBACK_CFG') {
                        input message: "To rollback CFG nodes run the following commands on kvm nodes hosting the CFG nodes: virsh destroy cfg0X.domain; virsh define /var/lib/libvirt/images/cfg0X.domain.xml; virsh start cfg0X.domain; virsh snapshot-delete cfg0X.domain --metadata ${SNAPSHOT_NAME}; rm /var/lib/libvirt/images/cfg0X.domain.${SNAPSHOT_NAME}.qcow2; rm /var/lib/libvirt/images/cfg0X.domain.xml; At the end restart 'docker' service on all cicd nodes and run 'linux.system.repo' Salt states on cicd nodes. After running the previous commands current pipeline job will be killed."
                        //rollbackSaltMaster(pepperEnv, 'I@salt:master')
                        //finishSaltMasterRollback(pepperEnv, 'I@salt:master')
                    }
                }
            } */

            if (rollbacks.contains("ctl")) {
                def target = CTL_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'ctl')
                        verifyAPIs(pepperEnv, target)
                    } else {
                        removeNode(pepperEnv, target, 'ctl')
                    }
                }
            }

            if (rollbacks.contains("prx")) {
                def target = PRX_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'prx')
                        verifyService(pepperEnv, target, 'nginx')
                    } else {
                        removeNode(pepperEnv, target, 'prx')
                    }
                }
            }

            if (rollbacks.contains("msg")) {
                def target = MSG_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'msg')
                        salt.enforceState(pepperEnv, target, 'rabbitmq')
                        verifyService(pepperEnv, target, 'rabbitmq-server')
                    } else {
                        removeNode(pepperEnv, target, 'msg')
                    }
                }
            }

            if (rollbacks.contains("dbs")) {
                def target = DBS_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'dbs')
                        clusterGalera(pepperEnv)
                        verifyGalera(pepperEnv, target)
                    } else {
                        removeNode(pepperEnv, target, 'dbs')
                    }
                }
            }

            if (rollbacks.contains("ntw")) {
                def target = NTW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'ntw')
                        verifyContrail(pepperEnv, target)
                    } else {
                        removeNode(pepperEnv, target, 'ntw')
                    }
                }
            }

            if (rollbacks.contains("nal")) {
                def target = NAL_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'nal')
                        verifyContrail(pepperEnv, target)
                    } else {
                        removeNode(pepperEnv, target, 'nal')
                    }
                }
            }

            if (rollbacks.contains("gtw-virtual")) {
                def target = GTW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'gtw')
                        verifyService(pepperEnv, target, 'neutron-dhcp-agent')
                    } else {
                        removeNode(pepperEnv, target, 'gtw')
                    }
                }
            }

            if (rollbacks.contains("cmn")) {
                def target = CMN_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'cmn')
                        verifyCeph(pepperEnv, target, 'mon@')
                    } else {
                        removeNode(pepperEnv, target, 'cmn')
                    }
                }
            }

            if (rollbacks.contains("rgw")) {
                def target = RGW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'rgw')
                        verifyCeph(pepperEnv, target, 'radosgw@rgw.')
                    } else {
                        removeNode(pepperEnv, target, 'rgw')
                    }
                }
            }

            if (rollbacks.contains("log")) {
                def target = LOG_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'log')
                    } else {
                        removeNode(pepperEnv, target, 'log')
                    }
                }
            }

            if (rollbacks.contains("mon")) {
                def target = MON_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'mon')
                    } else {
                        removeNode(pepperEnv, target, 'mon')
                    }
                }
            }

            if (rollbacks.contains("mtr")) {
                def target = MTR_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
                        rollback(pepperEnv, target, 'mtr')
                    } else {
                        removeNode(pepperEnv, target, 'mtr')
                    }
                }
            }
            /*
            if (ROLLBACK_CID.toBoolean()) {
                def target = 'cid*'
                if (salt.testTarget(pepperEnv, target)) {
                    stage('ROLLBACK_CID') {
                        input message: "To rollback CICD nodes run the following commands on kvm nodes hosting the cicd nodes: virsh destroy cid0X.domain; virsh define /var/lib/libvirt/images/cid0X.domain.xml; virsh start cid0X.domain; virsh snapshot-delete cid0X.domain --metadata ${SNAPSHOT_NAME}; rm /var/lib/libvirt/images/cid0X.domain.${SNAPSHOT_NAME}.qcow2; rm /var/lib/libvirt/images/cid0X.domain.xml; At the end restart 'docker' service on all cicd nodes and run 'linux.system.repo' Salt states on cicd nodes. After running the previous commands current pipeline job will be killed."
                    }
                }
            } */

            //
            //physical machines rollback CMP_TARGET
            //
            if (rollbacks.contains("cmp")) {
                def target = CMP_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            rollbackPkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        rollbackPkgs(pepperEnv, target, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'nova-compute')
                }
            }

            if (rollbacks.contains("kvm")) {
                def target = KVM_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            rollbackPkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        rollbackPkgs(pepperEnv, target, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'libvirt-bin')
                }
            }

            if (rollbacks.contains("osd")) {
                def target = CEPH_OSD_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            rollbackPkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        rollbackPkgs(pepperEnv, target, target)
                        highstate(pepperEnv, target)
                    }
                    verifyCephOsds(pepperEnv, target)
                }
            }

            if (rollbacks.contains("gtw-physical")) {
                def target = GTW_TARGET
                if (salt.testTarget(pepperEnv, target)) {
                    if (PER_NODE.toBoolean()) {
                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
                        for (t in targetHosts) {
                            rollbackPkgs(pepperEnv, t)
                            highstate(pepperEnv, t)
                        }
                    } else {
                        rollbackPkgs(pepperEnv, target, target)
                        highstate(pepperEnv, target)
                    }
                    verifyService(pepperEnv, target, 'neutron-dhcp-agent')
                }
            }

            /*
                * Merge snapshots section
            */
            if (merges.contains("cfg")) {
                if (salt.testTarget(pepperEnv, 'I@salt:master')) {
                    mergeSnapshot(pepperEnv, 'I@salt:master')
                }
            }

            if (merges.contains("ctl")) {
                if (salt.testTarget(pepperEnv, CTL_TARGET)) {
                    mergeSnapshot(pepperEnv, CTL_TARGET, 'ctl')
                    verifyService(pepperEnv, CTL_TARGET, 'nova-api')
                }
            }

            if (merges.contains("prx")) {
                if (salt.testTarget(pepperEnv, PRX_TARGET)) {
                    mergeSnapshot(pepperEnv, PRX_TARGET, 'prx')
                    verifyService(pepperEnv, PRX_TARGET, 'nginx')
                }
            }

            if (merges.contains("msg")) {
                if (salt.testTarget(pepperEnv, MSG_TARGET)) {
                    mergeSnapshot(pepperEnv, MSG_TARGET, 'msg')
                    verifyService(pepperEnv, MSG_TARGET, 'rabbitmq-server')
                }
            }

            if (merges.contains("dbs")) {
                if (salt.testTarget(pepperEnv, DBS_TARGET)) {
                    mergeSnapshot(pepperEnv, DBS_TARGET, 'dbs')
                    verifyGalera(pepperEnv, DBS_TARGET)
                    backupGalera(pepperEnv)
                }
            }

            if (merges.contains("ntw")) {
                if (salt.testTarget(pepperEnv, NTW_TARGET)) {
                    mergeSnapshot(pepperEnv, NTW_TARGET, 'ntw')
                    verifyContrail(pepperEnv, NTW_TARGET)
                    backupContrail(pepperEnv)
                }
            }

            if (merges.contains("nal")) {
                if (salt.testTarget(pepperEnv, NAL_TARGET)) {
                    mergeSnapshot(pepperEnv, NAL_TARGET, 'nal')
                    verifyContrail(pepperEnv, NAL_TARGET)
                }
            }

            if (merges.contains("gtw-virtual")) {
                if (salt.testTarget(pepperEnv, GTW_TARGET)) {
                    mergeSnapshot(pepperEnv, GTW_TARGET, 'gtw')
                    verifyService(pepperEnv, GTW_TARGET, 'neutron-dhcp-agent')
                }
            }

            if (merges.contains("cmn")) {
                if (salt.testTarget(pepperEnv, CMN_TARGET)) {
                    mergeSnapshot(pepperEnv, CMN_TARGET, 'cmn')
                    verifyCeph(pepperEnv, CMN_TARGET, 'mon@')
                    backupCeph(pepperEnv, CMN_TARGET)
                }
            }

            if (merges.contains("rgw")) {
                if (salt.testTarget(pepperEnv, RGW_TARGET)) {
                    mergeSnapshot(pepperEnv, RGW_TARGET, 'rgw')
                    verifyCeph(pepperEnv, RGW_TARGET, 'radosgw@rgw.')
                }
            }

            if (merges.contains("log")) {
                if (salt.testTarget(pepperEnv, LOG_TARGET)) {
                    mergeSnapshot(pepperEnv, LOG_TARGET. 'log')
                }
            }

            if (merges.contains("mon")) {
                if (salt.testTarget(pepperEnv, MON_TARGET)) {
                    mergeSnapshot(pepperEnv, MON_TARGET, 'mon')
                }
            }

            if (merges.contains("mtr")) {
                if (salt.testTarget(pepperEnv, MTR_TARGET)) {
                    mergeSnapshot(pepperEnv, MTR_TARGET, 'mtr')
                }
            }

            if (merges.contains("cid")) {
                if (salt.testTarget(pepperEnv, CID_TARGET)) {
                    mergeSnapshot(pepperEnv, CID_TARGET, 'cid')
                    verifyService(pepperEnv, CID_TARGET, 'docker')
                }
            }

            if (RESTORE_GALERA.toBoolean()) {
                restoreGalera(pepperEnv)
                verifyGalera(pepperEnv, DBS_TARGET)
            }

            if (RESTORE_CONTRAIL_DB.toBoolean()) {
                restoreContrailDb(pepperEnv)
                // verification is already present in restore pipelines
            }

        } catch (Throwable e) {
            // If there was an error or exception thrown, the build failed
            currentBuild.result = "FAILURE"
            currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
            throw e
        }
    }
}