diff --git a/aptly-promote-pipeline.groovy b/aptly-promote-pipeline.groovy
index 0f51a90..dfddfcd 100644
--- a/aptly-promote-pipeline.groovy
+++ b/aptly-promote-pipeline.groovy
@@ -26,7 +26,7 @@
 def insufficientPermissions = false
 
 timeout(time: 12, unit: 'HOURS') {
-  node() {
+  node("docker&&hardware") {
     try {
       stage("promote") {
         // promote is restricted to users in aptly-promote-users LDAP group
@@ -36,7 +36,9 @@
               if (storage == "local") {
                 storage = ""
               }
-              aptly.promotePublish(APTLY_URL, SOURCE, TARGET, RECREATE, components, packages, DIFF_ONLY, '-d --timeout 600', DUMP_PUBLISH.toBoolean(), storage)
+              retry(2){
+                aptly.promotePublish(APTLY_URL, SOURCE, TARGET, RECREATE, components, packages, DIFF_ONLY, '-d --timeout 600', DUMP_PUBLISH.toBoolean(), storage)
+              }
             }
           }
         }else{
diff --git a/build-debian-packages-influxdb-relay.groovy b/build-debian-packages-influxdb-relay.groovy
index 3a9bc03..14b1190 100644
--- a/build-debian-packages-influxdb-relay.groovy
+++ b/build-debian-packages-influxdb-relay.groovy
@@ -79,7 +79,9 @@
                         }
                         stage("publish") {
                             aptly.snapshotRepo(APTLY_URL, APTLY_REPO, timestamp)
-                            aptly.publish(APTLY_URL)
+                            retry(2){
+                              aptly.publish(APTLY_URL)
+                            }
                         }
                     }
                 }
diff --git a/build-debian-packages-jmx-exporter.groovy b/build-debian-packages-jmx-exporter.groovy
index d6e7fbd..eaa9f05 100644
--- a/build-debian-packages-jmx-exporter.groovy
+++ b/build-debian-packages-jmx-exporter.groovy
@@ -58,7 +58,9 @@
 
                 stage("publish") {
                     aptly.snapshotRepo(APTLY_URL, APTLY_REPO, timestamp)
-                    aptly.publish(APTLY_URL)
+                    retry(2){
+                      aptly.publish(APTLY_URL)
+                    }
                 }
             }
 
diff --git a/build-debian-packages-libvirt-exporter.groovy b/build-debian-packages-libvirt-exporter.groovy
index 2051504..4c3f732 100644
--- a/build-debian-packages-libvirt-exporter.groovy
+++ b/build-debian-packages-libvirt-exporter.groovy
@@ -60,7 +60,9 @@
                     }
                     stage("publish") {
                         aptly.snapshotRepo(APTLY_URL, APTLY_REPO, timestamp)
-                        aptly.publish(APTLY_URL)
+                        retry(2){
+                          aptly.publish(APTLY_URL)
+                        }
                     }
                 }
             }
diff --git a/build-debian-packages-pipeline.groovy b/build-debian-packages-pipeline.groovy
index 8d46e8e..5d16339 100644
--- a/build-debian-packages-pipeline.groovy
+++ b/build-debian-packages-pipeline.groovy
@@ -135,7 +135,9 @@
 
           stage("publish") {
             aptly.snapshotRepo(APTLY_URL, APTLY_REPO, timestamp)
-            aptly.publish(APTLY_URL)
+            retry(2){
+              aptly.publish(APTLY_URL)
+            }
           }
         }
       }
diff --git a/build-debian-packages-prometheus-relay.groovy b/build-debian-packages-prometheus-relay.groovy
index 45e5b10..f101f57 100644
--- a/build-debian-packages-prometheus-relay.groovy
+++ b/build-debian-packages-prometheus-relay.groovy
@@ -79,7 +79,9 @@
                         }
                         stage("publish") {
                             aptly.snapshotRepo(APTLY_URL, APTLY_REPO, timestamp)
-                            aptly.publish(APTLY_URL)
+                            retry(2){
+                              aptly.publish(APTLY_URL)
+                            }
                         }
 
                         stage("rebuild docker images") {
diff --git a/build-debian-packages-telegraf.groovy b/build-debian-packages-telegraf.groovy
index 16d6259..b48c4fa 100644
--- a/build-debian-packages-telegraf.groovy
+++ b/build-debian-packages-telegraf.groovy
@@ -78,7 +78,9 @@
                         }
                         stage("publish") {
                             aptly.snapshotRepo(APTLY_URL, APTLY_REPO, timestamp)
-                            aptly.publish(APTLY_URL)
+                            retry(2){
+                              aptly.publish(APTLY_URL)
+                            }
                         }
 
                         stage("rebuild docker images") {
diff --git a/build-extra-dpdk-pipeline.groovy b/build-extra-dpdk-pipeline.groovy
index 4d096ae..e9ac981 100644
--- a/build-extra-dpdk-pipeline.groovy
+++ b/build-extra-dpdk-pipeline.groovy
@@ -53,7 +53,9 @@
         }
         stage("publish") {
             aptly.snapshotRepo(APTLY_URL, APTLY_REPO, timestamp)
-            aptly.publish(APTLY_URL)
+            retry(2){
+              aptly.publish(APTLY_URL)
+            }
         }
        }
        } catch (Throwable e) {
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index a37a655..c2f9351 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -579,7 +579,17 @@
 
             stage('Finalize') {
                 if (common.checkContains('STACK_INSTALL', 'finalize')) {
-                    salt.runSaltProcessStep(venvPepper, '*', 'state.apply', [], null, true)
+                    def gluster_compound = 'I@glusterfs:server'
+                    // Enforce highstate asynchronous only on the nodes which are not glusterfs servers
+                    salt.enforceHighstate(venvPepper, '* and not ' + gluster_compound)
+                    // Iterate over nonempty set of gluster servers and apply highstates one by one
+                    // TODO: switch to batch once salt 2017.7+ would be used
+                    def glusterMinions = salt.getMinionsSorted(venvPepper, gluster_compound)
+                    if ( !glusterMinions.isEmpty() ) {
+                        for ( target in glusterMinions ) {
+                            salt.enforceHighstate(venvPepper, target)
+                        }
+                    }
                 }
 
                 outputsPretty = common.prettify(outputs)
diff --git a/cloud-update.groovy b/cloud-update.groovy
new file mode 100644
index 0000000..fd9cbce
--- /dev/null
+++ b/cloud-update.groovy
@@ -0,0 +1,1560 @@
+/**
+ * Update packages on given nodes
+ *
+ * Expected parameters:
+ *   SALT_MASTER_CREDENTIALS    Credentials to the Salt API.
+ *   SALT_MASTER_URL            Full Salt API address [https://10.10.10.1:8000].
+ *   SNAPSHOT_NAME              Snapshot name
+ *   CFG_NODE_PROVIDER          Physical machine name hosting Salt-Master VM (ex. kvm01*)
+ *   INTERACTIVE                Ask interactive questions during pipeline run (bool)
+ *   PER_NODE                   Target nodes will be managed one by one (bool)
+ *   ROLLBACK_BY_REDEPLOY       Omit taking live snapshots. Rollback is planned to be done by redeployment (bool)
+ *   STOP_SERVICES              Stop API services before update (bool)
+ *   TARGET_UPDATES             Comma separated list of nodes to update (Valid values are cfg,ctl,prx,msg,dbs,log,mon,mtr,ntw,nal,gtw-virtual,cmn,rgw,cid,cmp,kvm,osd,gtw-physical)
+ *   TARGET_ROLLBACKS           Comma separated list of nodes to rollback (Valid values are ctl,prx,msg,dbs,log,mon,mtr,ntw,nal,gtw-virtual,cmn,rgw,cmp,kvm,osd,gtw-physical)
+ *   TARGET_SNAPSHOT_MERGES     Comma separated list of nodes to merge live snapshot for (Valid values are cfg,ctl,prx,msg,dbs,log,mon,mtr,ntw,nal,gtw-virtual,cmn,rgw,cid)
+ *   CTL_TARGET                 Salt targeted CTL nodes (ex. ctl*)
+ *   PRX_TARGET                 Salt targeted PRX nodes (ex. prx*)
+ *   MSG_TARGET                 Salt targeted MSG nodes (ex. msg*)
+ *   DBS_TARGET                 Salt targeted DBS nodes (ex. dbs*)
+ *   LOG_TARGET                 Salt targeted LOG nodes (ex. log*)
+ *   MON_TARGET                 Salt targeted MON nodes (ex. mon*)
+ *   MTR_TARGET                 Salt targeted MTR nodes (ex. mtr*)
+ *   NTW_TARGET                 Salt targeted NTW nodes (ex. ntw*)
+ *   NAL_TARGET                 Salt targeted NAL nodes (ex. nal*)
+ *   CMN_TARGET                 Salt targeted CMN nodes (ex. cmn*)
+ *   RGW_TARGET                 Salt targeted RGW nodes (ex. rgw*)
+ *   CID_TARGET                 Salt targeted CID nodes (ex. cid*)
+ *   CMP_TARGET                 Salt targeted physical compute nodes (ex. cmp001*)
+ *   KVM_TARGET                 Salt targeted physical KVM nodes (ex. kvm01*)
+ *   CEPH_OSD_TARGET            Salt targeted physical Ceph OSD nodes (ex. osd001*)
+ *   GTW_TARGET                 Salt targeted physical or virtual GTW nodes (ex. gtw01*)
+ *   REBOOT                     Reboot nodes after update (bool)
+ *   ROLLBACK_PKG_VERSIONS      Space separated list of pkgs=versions to rollback to on physical targeted machines (ex. pkg_name1=pkg_version1 pkg_name2=pkg_version2)
+ *   PURGE_PKGS                 Space separated list of pkgs=versions to be purged on physical targeted machines (ex. pkg_name1=pkg_version1 pkg_name2=pkg_version2)
+ *   REMOVE_PKGS                Space separated list of pkgs=versions to be removed on physical targeted machines (ex. pkg_name1=pkg_version1 pkg_name2=pkg_version2)
+ *   RESTORE_GALERA             Restore Galera DB (bool)
+ *   RESTORE_CONTRAIL_DB        Restore Cassandra and Zookeeper DBs for OpenContrail (bool)
+ *
+**/
+def common = new com.mirantis.mk.Common()
+def salt = new com.mirantis.mk.Salt()
+def python = new com.mirantis.mk.Python()
+def virsh = new com.mirantis.mk.Virsh()
+
+def updates = TARGET_UPDATES.tokenize(",").collect{it -> it.trim()}
+def rollbacks = TARGET_ROLLBACKS.tokenize(",").collect{it -> it.trim()}
+def merges = TARGET_SNAPSHOT_MERGES.tokenize(",").collect{it -> it.trim()}
+
+def pepperEnv = "pepperEnv"
+def minions
+def result
+def packages
+def command
+def commandKwargs
+
+def updatePkgs(pepperEnv, target, targetType="", targetPackages="") {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def commandKwargs
+    def distUpgrade
+    def pkgs
+    def out
+
+    salt.enforceState(pepperEnv, target, 'linux.system.repo')
+
+    stage("List package upgrades") {
+        common.infoMsg("Listing all the packages that have a new update available on ${target}")
+        pkgs = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, target, 'pkg.list_upgrades', [], null, true))
+        if(targetPackages != "" && targetPackages != "*"){
+            common.infoMsg("Note that only the ${targetPackages} would be installed from the above list of available updates on the ${target}")
+        }
+    }
+
+    if (INTERACTIVE.toBoolean()) {
+        stage("Confirm live package upgrades on ${target}") {
+            if (targetPackages=="") {
+                def userInput = input(
+                 id: 'userInput', message: 'Insert package names for update', parameters: [
+                 [$class: 'TextParameterDefinition', defaultValue: pkgs.keySet().join(",").toString(), description: 'Package names (or *)', name: 'packages']
+                ])
+                if (userInput!= "" && userInput!= "*") {
+                    targetPackages = userInput
+                }
+            } else {
+                input message: "Approve live package upgrades on ${target} nodes?"
+            }
+        }
+    } else {
+        targetPackages = pkgs.keySet().join(",").toString()
+    }
+
+    if (targetPackages != "") {
+        // list installed versions of pkgs that will be upgraded
+        def installedPkgs = []
+        def newPkgs = []
+        def targetPkgList = targetPackages.tokenize(',')
+        for (pkg in targetPkgList) {
+            def version
+            try {
+                def pkgsDetails = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, target, 'pkg.info_installed', [pkg], null, true))
+                version = pkgsDetails.get(pkg).get('version')
+            } catch (Exception er) {
+                common.infoMsg("${pkg} not installed yet")
+            }
+            if (version?.trim()) {
+                installedPkgs.add(pkg + '=' + version)
+            } else {
+                newPkgs.add(pkg)
+            }
+        }
+        common.warningMsg("the following list of pkgs will be upgraded")
+        common.warningMsg(installedPkgs.join(" "))
+        common.warningMsg("the following list of pkgs will be newly installed")
+        common.warningMsg(newPkgs.join(" "))
+        // set variables
+        command = "pkg.install"
+        packages = targetPackages
+        commandKwargs = ['only_upgrade': 'true','force_yes': 'true']
+
+    }else {
+        command = "pkg.upgrade"
+        commandKwargs = ['dist_upgrade': 'true']
+        distUpgrade = true
+        packages = null
+    }
+
+    // todo exception to cfg or cicd
+    stage("stop services on ${target}") {
+        if ((STOP_SERVICES.toBoolean()) && (targetType != 'cicd')) {
+            if (targetType == 'contrail') {
+                stopContrailServices(pepperEnv, target)
+            } else {
+                def probe = salt.getFirstMinion(pepperEnv, "${target}")
+                stopServices(pepperEnv, probe, target)
+            }
+        }
+    }
+
+    stage('Apply package upgrades') {
+        // salt master pkg
+        if (targetType == 'I@salt:master') {
+            common.warningMsg('salt-master pkg upgrade, rerun the pipeline if disconnected')
+            salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-master'], null, true, 5)
+            salt.minionsReachable(pepperEnv, 'I@salt:master', '*')
+        }
+        // salt minion pkg
+        salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-minion'], null, true, 5)
+        salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+        common.infoMsg('Performing pkg upgrades ... ')
+        common.retry(3){
+            out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, true, packages, commandKwargs)
+            salt.printSaltCommandResult(out)
+        }
+        def osRelease = salt.getGrain(pepperEnv, target, 'lsb_distrib_codename')
+        if (osRelease.toString().toLowerCase().contains('trusty')) {
+            args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --force-yes -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" '
+        } else {
+            args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q -f --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" '
+        }
+        if (out.toString().contains('errors:')) {
+            try {
+                if (packages?.trim()) {
+                    packages = packages.replaceAll(',', ' ')
+                    common.retry(3){
+                        out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' install ' + packages])
+                    }
+                } else {
+                    if (distUpgrade) {
+                        common.retry(3){
+                            out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' dist-upgrade'])
+                        }
+                    } else {
+                        common.retry(3){
+                            out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' upgrade'])
+                        }
+                    }                    }
+                if (out.toString().contains('E: ')) {
+                    common.errorMsg(out)
+                    if (INTERACTIVE.toBoolean()) {
+                        input message: "Pkgs update failed to be updated on ${target}. Please fix it manually."
+                    } else {
+                        salt.printSaltCommandResult(out)
+                        throw new Exception("Pkgs update failed")
+                    }
+                }
+            } catch (Exception e) {
+                common.errorMsg(out)
+                common.errorMsg(e)
+                if (INTERACTIVE.toBoolean()) {
+                    input message: "Pkgs update failed to be updated on ${target}. Please fix it manually."
+                } else {
+                    throw new Exception("Pkgs update failed")
+                }
+            }
+        }
+    }
+}
+
+def rollbackPkgs(pepperEnv, target, targetType = "", targetPackages="") {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def probe = salt.getFirstMinion(pepperEnv, "${target}")
+    def distUpgrade
+    def pkgs
+    def out
+
+    salt.enforceState(pepperEnv, target, 'linux.system.repo')
+
+    if (ROLLBACK_PKG_VERSIONS == "") {
+        stage("List package upgrades") {
+            common.infoMsg("Listing all the packages that have a new update available on ${target}")
+            pkgs = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, target, 'pkg.list_upgrades', [], null, true))
+            if(targetPackages != "" && targetPackages != "*"){
+                common.infoMsg("Note that only the ${targetPackages} would be installed from the above list of available updates on the ${target}")
+            }
+        }
+
+        if (INTERACTIVE.toBoolean()) {
+            stage("Confirm live package upgrades on ${target}") {
+                if(targetPackages==""){
+                    timeout(time: 2, unit: 'HOURS') {
+                        def userInput = input(
+                         id: 'userInput', message: 'Insert package names for update', parameters: [
+                         [$class: 'TextParameterDefinition', defaultValue: pkgs.keySet().join(",").toString(), description: 'Package names (or *)', name: 'packages']
+                        ])
+                        if(userInput!= "" && userInput!= "*"){
+                            targetPackages = userInput
+                        }
+                    }
+                }else{
+                    timeout(time: 2, unit: 'HOURS') {
+                       input message: "Approve live package upgrades on ${target} nodes?"
+                    }
+                }
+            }
+        } else {
+            targetPackages = pkgs.keySet().join(",").toString()
+        }
+    } else {
+        targetPackages = ROLLBACK_PKG_VERSIONS
+    }
+
+    if (targetPackages != "") {
+        // set variables
+        packages = targetPackages
+    } else {
+        distUpgrade = true
+        packages = null
+    }
+
+    stage("stop services on ${target}") {
+        try {
+            if (INTERACTIVE.toBoolean()) {
+                input message: "Click PROCEED to interactively stop services on ${target}. Otherwise click ABORT to skip stopping them and continue."
+            }
+        } catch (Exception er) {
+            common.infoMsg('skipping stopping services')
+            return
+        }
+        if (STOP_SERVICES.toBoolean()) {
+            stopServices(pepperEnv, probe, target)
+        }
+    }
+
+    stage('Apply package downgrades') {
+        args = 'export DEBIAN_FRONTEND=noninteractive; apt-get -y -q --allow-downgrades -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\" '
+        common.infoMsg('Performing pkgs purge/remove ... ')
+        try {
+            if (PURGE_PKGS != "") {
+                def purgePackages = PURGE_PKGS.replaceAll(',', ' ')
+                common.retry(3){
+                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' purge ' + purgePackages])
+                }
+            }
+            if (REMOVE_PKGS != "") {
+                def removePackages = REMOVE_PKGS.replaceAll(',', ' ')
+                common.retry(3){
+                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' remove ' + removePackages])
+                }
+            }
+            if (out.toString().contains('E: ')) {
+                common.errorMsg(out)
+                if (INTERACTIVE.toBoolean()) {
+                    input message: "Pkgs ${packages} purge failed on ${target}. Please fix it manually."
+                } else {
+                    salt.printSaltCommandResult(out)
+                    throw new Exception("Pkgs {packages} purge failed")
+                }
+            }
+        } catch (Exception e) {
+            common.errorMsg(out)
+            common.errorMsg(e)
+            if (INTERACTIVE.toBoolean()) {
+                input message: "Pkgs {packages} purge on ${target}. Please fix it manually."
+            } else {
+                throw new Exception("Pkgs {packages} purge failed")
+            }
+        }
+
+        common.infoMsg('Performing pkg downgrades ... ')
+        try {
+            packages = packages.replaceAll(',', ' ')
+            if (packages?.trim()) {
+                packages = packages.replaceAll(',', ' ')
+                common.retry(3){
+                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' install salt-minion'], null, true, 5)
+                }
+                salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+                common.retry(3){
+                    out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' install ' + packages])
+                }
+            } else {
+                if (distUpgrade) {
+                    common.retry(3){
+                        out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' dist-upgrade'])
+                    }
+                } else {
+                    common.retry(3){
+                        out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' upgrade'])
+                    }
+                }
+            }
+            if (out.toString().contains('E: ')) {
+                common.errorMsg(out)
+                if (INTERACTIVE.toBoolean()) {
+                    input message: "Pkgs rollback failed on ${target}. Please fix it manually."
+                } else {
+                    salt.printSaltCommandResult(out)
+                    throw new Exception("Pkgs rollback failed")
+                }
+            }
+        } catch (Exception e) {
+            common.errorMsg(out)
+            common.errorMsg(e)
+            if (INTERACTIVE.toBoolean()) {
+                input message: "Pkgs rollback failed on ${target}. Please fix it manually."
+            } else {
+                throw new Exception("Pkgs rollback failed")
+            }
+        }
+    }
+}
+
+def getCfgNodeProvider(pepperEnv, master_name) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    if (!CFG_NODE_PROVIDER?.trim()) {
+        kvms = salt.getMinions(pepperEnv, 'I@salt:control')
+        for (kvm in kvms) {
+            try {
+                vms = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, kvm, 'virt.list_active_vms', [], null, true))
+                if (vms.toString().contains(master_name)) {
+                    CFG_NODE_PROVIDER = kvm
+                    //break
+                }
+            } catch (Exception er) {
+                common.infoMsg("${master_name} not present on ${kvm}")
+            }
+        }
+    }
+}
+
+/*
+def rollbackSaltMaster(pepperEnv, target, path='/var/lib/libvirt/images') {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    try {
+        input message: "PART1 - Are you sure to rollback ${target}? To rollback click on PROCEED. To skip rollback PART1 click on ABORT."
+    } catch (Exception er) {
+        common.infoMsg("skipping rollback of ${target}")
+        return
+    }
+    def domain = salt.getDomainName(pepperEnv)
+    def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
+    getCfgNodeProvider(pepperEnv, master)
+    try {
+        try {
+            salt.getReturnValues(salt.cmdRun(pepperEnv, CFG_NODE_PROVIDER, "ls -la ${path}/${master}.${domain}.xml"))
+            common.errorMsg('Pipeline is about to disconnect from salt-api. You will have to rerun the pipeline with ROLLBACK_CFG checked and skip PART1 to finish rollback.')
+            salt.cmdRun(pepperEnv, CFG_NODE_PROVIDER, "virsh destroy ${master}.${domain}; virsh define ${path}/${master}.${domain}.xml; virsh start ${master}.${domain} ")
+        } catch (Exception er) {
+            common.errorMsg(er)
+            input message: "Rollback for ${target} failed. Rollback manually."
+        }
+    } catch (Exception er) {
+        common.errorMsg(er)
+        input message: "Rollback for ${target} failed. Rollback manually."
+    }
+}
+
+def finishSaltMasterRollback(pepperEnv, target, path='/var/lib/libvirt/images') {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def virsh = new com.mirantis.mk.Virsh()
+    try {
+        input message: "PART2 - Are you sure to finalize ${target} rollback? Click on PROCEED. To skip rollback click on ABORT."
+    } catch (Exception er) {
+        common.infoMsg("skipping finalize rollback of ${target}")
+        return
+    }
+    salt.minionsReachable(pepperEnv, 'I@salt:master', '*')
+    def domain = salt.getDomainName(pepperEnv)
+    def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
+    getCfgNodeProvider(pepperEnv, master)
+    try {
+        virsh.liveSnapshotAbsent(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME, path)
+        // purge and setup previous repos
+        salt.enforceState(pepperEnv, target, 'linux.system.repo')
+    } catch (Exception e) {
+        common.errorMsg(e)
+        input message: "Check what failed after ${target} rollback. Do you want to PROCEED?"
+    }
+}*/
+
+def stopServices(pepperEnv, probe, target) {
+    def openstack = new com.mirantis.mk.Openstack()
+    def services = []
+    services.add('keepalived')
+    services.add('nginx')
+    services.add('haproxy')
+    services.add('nova-api')
+    services.add('cinder')
+    services.add('glance')
+    services.add('heat')
+    services.add('neutron')
+    services.add('apache2')
+    services.add('rabbitmq-server')
+    if (INTERACTIVE.toBoolean()) {
+        openstack.stopServices(pepperEnv, probe, target, services, true)
+    } else {
+        openstack.stopServices(pepperEnv, probe, target, services)
+    }
+}
+
+// must be stopped separately due to OC on Trusty
+def stopContrailServices(pepperEnv, target) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def services = []
+    services.add('keepalived')
+    services.add('haproxy')
+    services.add('supervisor-control')
+    services.add('supervisor-config')
+    services.add('supervisor-database')
+    services.add('zookeeper')
+    services.add('ifmap-server')
+    for (s in services) {
+        try {
+            salt.runSaltProcessStep(pepperEnv, target, 'service.stop', [s], null, true)
+        } catch (Exception er) {
+            common.infoMsg(er)
+        }
+    }
+}
+
+def highstate(pepperEnv, target) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+
+    stage("Apply highstate on ${target} nodes") {
+        try {
+            common.retry(3){
+                salt.enforceHighstate(pepperEnv, target)
+            }
+        } catch (Exception e) {
+            common.errorMsg(e)
+            if (INTERACTIVE.toBoolean()) {
+                input message: "Highstate failed on ${target}. Fix it manually or run rollback on ${target}."
+            } else {
+                throw new Exception("highstate failed")
+            }
+        }
+    }
+    // optionally reboot
+    if (REBOOT.toBoolean()) {
+        stage("Reboot ${target} nodes") {
+            salt.runSaltProcessStep(pepperEnv, target, 'system.reboot', null, null, true, 5)
+            sleep(10)
+            salt.minionsReachable(pepperEnv, 'I@salt:master', target)
+        }
+    }
+}
+
+def rollback(pepperEnv, tgt, generalTarget) {
+    def common = new com.mirantis.mk.Common()
+    try {
+        if (INTERACTIVE.toBoolean()) {
+            input message: "Are you sure to rollback ${generalTarget}? To rollback click on PROCEED. To skip rollback click on ABORT."
+        }
+    } catch (Exception er) {
+        common.infoMsg('skipping rollback')
+        return
+    }
+    try {
+        rollbackLiveSnapshot(pepperEnv, tgt, generalTarget)
+    } catch (Exception err) {
+        common.errorMsg(err)
+        if (INTERACTIVE.toBoolean()) {
+            input message: "Rollback for ${tgt} failed please fix it manually before clicking PROCEED."
+        } else {
+            throw new Exception("Rollback failed for ${tgt}")
+        }
+    }
+}
+
+def liveSnapshot(pepperEnv, tgt, generalTarget) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def virsh = new com.mirantis.mk.Virsh()
+    def domain = salt.getDomainName(pepperEnv)
+    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
+    common.warningMsg(target_hosts)
+    //def nodeCount = 1
+    for (t in target_hosts) {
+        def target = salt.stripDomainName(t)
+        def nodeCount = target[4]
+        common.warningMsg(nodeCount)
+        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+        virsh.liveSnapshotPresent(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
+        //nodeCount++
+    }
+}
+
+def mergeSnapshot(pepperEnv, tgt, generalTarget='') {
+    def salt = new com.mirantis.mk.Salt()
+    def virsh = new com.mirantis.mk.Virsh()
+    def domain = salt.getDomainName(pepperEnv)
+    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
+    def nodeCount = 1
+    for (t in target_hosts) {
+        if (tgt == 'I@salt:master') {
+            def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
+            getCfgNodeProvider(pepperEnv, master)
+            virsh.liveSnapshotMerge(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME)
+        } else {
+            def target = salt.stripDomainName(t)
+            def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+            virsh.liveSnapshotMerge(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
+        }
+        nodeCount++
+    }
+    salt.minionsReachable(pepperEnv, 'I@salt:master', tgt)
+}
+
+
+
+def rollbackLiveSnapshot(pepperEnv, tgt, generalTarget) {
+    def salt = new com.mirantis.mk.Salt()
+    def virsh = new com.mirantis.mk.Virsh()
+    def common = new com.mirantis.mk.Common()
+    def domain = salt.getDomainName(pepperEnv)
+    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
+    // first destroy all vms
+    def nodeCount = 1
+    for (t in target_hosts) {
+        def target = salt.stripDomainName(t)
+        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+        salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.destroy', ["${target}.${domain}"], null, true)
+        nodeCount++
+    }
+    nodeCount = 1
+    // rollback vms
+    for (t in target_hosts) {
+        def target = salt.stripDomainName(t)
+        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+        virsh.liveSnapshotRollback(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
+        nodeCount++
+    }
+    try {
+        salt.minionsReachable(pepperEnv, 'I@salt:master', tgt)
+        // purge and setup previous repos
+        salt.enforceState(pepperEnv, tgt, 'linux.system.repo')
+    } catch (Exception e) {
+        common.errorMsg(e)
+        if (INTERACTIVE.toBoolean()) {
+            input message: "Salt state linux.system.repo on ${tgt} failed. Do you want to PROCEED?."
+        } else {
+            throw new Exception("Salt state linux.system.repo on ${tgt} failed")
+        }
+    }
+}
+
+def removeNode(pepperEnv, tgt, generalTarget) {
+    def salt = new com.mirantis.mk.Salt()
+    def virsh = new com.mirantis.mk.Virsh()
+    def common = new com.mirantis.mk.Common()
+    def domain = salt.getDomainName(pepperEnv)
+    def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
+    // first destroy all vms
+    def nodeCount = 1
+    for (t in target_hosts) {
+        def target = salt.stripDomainName(t)
+        def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+        salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.destroy', ["${target}.${domain}"], null, true)
+        salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.undefine', ["${target}.${domain}"], null, true)
+        try {
+            salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key -d ${target}.${domain} -y")
+        } catch (Exception e) {
+            common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
+        }
+        nodeCount++
+    }
+}
+
+def saltMasterBackup(pepperEnv) {
+    def salt = new com.mirantis.mk.Salt()
+    salt.enforceState(pepperEnv, 'I@salt:master', 'backupninja')
+    salt.cmdRun(pepperEnv, 'I@salt:master', "su root -c 'backupninja -n --run /etc/backup.d/200.backup.rsync'")
+}
+
+def backupCeph(pepperEnv, tgt) {
+    def salt = new com.mirantis.mk.Salt()
+    salt.enforceState(pepperEnv, 'I@ceph:backup:server', 'ceph.backup')
+    salt.enforceState(pepperEnv, "I@ceph:backup:client and ${tgt}", 'ceph.backup')
+    salt.cmdRun(pepperEnv, "I@ceph:backup:client and ${tgt}", "su root -c '/usr/local/bin/ceph-backup-runner-call.sh -s'")
+}
+
+def backupGalera(pepperEnv) {
+    def salt = new com.mirantis.mk.Salt()
+    salt.enforceState(pepperEnv, 'I@xtrabackup:server', ['linux.system.repo', 'xtrabackup'])
+    salt.enforceState(pepperEnv, 'I@xtrabackup:client', ['linux.system.repo', 'openssh.client'])
+    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
+    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh -s -f'")
+}
+
+// cluster galera - wsrep_cluster_size
+def clusterGalera(pepperEnv) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    try {
+        salt.runSaltProcessStep(pepperEnv, 'I@galera:slave', 'service.stop', ['mysql'])
+    } catch (Exception er) {
+        common.warningMsg('Mysql service already stopped')
+    }
+    try {
+        salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'service.stop', ['mysql'])
+    } catch (Exception er) {
+        common.warningMsg('Mysql service already stopped')
+    }
+    try {
+        salt.cmdRun(pepperEnv, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
+    } catch (Exception er) {
+        common.warningMsg('Files are not present')
+    }
+    salt.cmdRun(pepperEnv, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
+    salt.runSaltProcessStep(pepperEnv, 'I@galera:master', 'service.start', ['mysql'])
+    // wait until mysql service on galera master is up
+    try {
+        salt.commandStatus(pepperEnv, 'I@galera:master', 'service mysql status', 'running')
+    } catch (Exception er) {
+        if (INTERACTIVE.toBoolean()) {
+            input message: "Database is not running please fix it first and only then click on PROCEED."
+        } else {
+            throw new Exception("Database is not running correctly")
+        }
+    }
+    salt.runSaltProcessStep(pepperEnv, 'I@galera:slave', 'service.start', ['mysql'])
+}
+
+def restoreGalera(pepperEnv) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def openstack = new com.mirantis.mk.Openstack()
+    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "rm -rf /var/lib/mysql/*")
+    openstack.restoreGaleraDb(pepperEnv)
+}
+
+def backupZookeeper(pepperEnv) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    salt.enforceState(pepperEnv, 'I@zookeeper:backup:server', 'zookeeper.backup')
+    salt.enforceState(pepperEnv, 'I@zookeeper:backup:client', 'zookeeper.backup')
+    try {
+        salt.cmdRun(pepperEnv, 'I@opencontrail:control', "su root -c '/usr/local/bin/zookeeper-backup-runner.sh -s'")
+    } catch (Exception er) {
+        throw new Exception('Zookeeper failed to backup. Please fix it before continuing.')
+    }
+}
+
+def backupCassandra(pepperEnv) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+
+    salt.enforceState(pepperEnv, 'I@cassandra:backup:server', 'cassandra.backup')
+    salt.enforceState(pepperEnv, 'I@cassandra:backup:client', 'cassandra.backup')
+    try {
+        salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', "su root -c '/usr/local/bin/cassandra-backup-runner-call.sh -s'")
+    } catch (Exception er) {
+        throw new Exception('Cassandra failed to backup. Please fix it before continuing.')
+    }
+}
+
+def backupContrail(pepperEnv) {
+    backupZookeeper(pepperEnv)
+    backupCassandra(pepperEnv)
+}
+
+// cassandra and zookeeper
+def restoreContrailDb(pepperEnv) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    build job: "deploy-zookeeper-restore", parameters: [
+      [$class: 'StringParameterValue', name: 'SALT_MASTER_CREDENTIALS', value: SALT_MASTER_CREDENTIALS],
+      [$class: 'StringParameterValue', name: 'SALT_MASTER_URL', value: SALT_MASTER_URL]
+    ]
+    build job: "deploy-cassandra-db-restore", parameters: [
+      [$class: 'StringParameterValue', name: 'SALT_MASTER_CREDENTIALS', value: SALT_MASTER_CREDENTIALS],
+      [$class: 'StringParameterValue', name: 'SALT_MASTER_URL', value: SALT_MASTER_URL]
+    ]
+}
+
+def verifyAPIs(pepperEnv, target) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def out = salt.cmdRun(pepperEnv, target, '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
+    if (out.toString().toLowerCase().contains('error')) {
+        common.errorMsg(out)
+        if (INTERACTIVE.toBoolean()) {
+            input message: "APIs are not working as expected. Please fix it manually."
+        } else {
+            throw new Exception("APIs are not working as expected")
+        }
+    }
+}
+
+def verifyGalera(pepperEnv, target, count=0, maxRetries=200) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def out
+    while(count < maxRetries) {
+        try {
+            out = salt.getReturnValues(salt.cmdRun(pepperEnv, target, 'salt-call mysql.status | grep -A1 wsrep_cluster_size'))
+        } catch (Exception er) {
+            common.infoMsg(er)
+        }
+        if ((!out.toString().contains('wsrep_cluster_size')) || (out.toString().contains('0'))) {
+            count++
+            if (count == maxRetries) {
+                if (INTERACTIVE.toBoolean()) {
+                    input message: "Galera is not working as expected. Please check it and fix it first before clicking on PROCEED."
+                } else {
+                    common.errorMsg(out)
+                    throw new Exception("Galera is not working as expected")
+                }
+            }
+            sleep(time: 500, unit: 'MILLISECONDS')
+        } else {
+            break
+        }
+    }
+}
+
+def verifyContrail(pepperEnv, target) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    salt.commandStatus(pepperEnv, target, "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
+}
+
+
+def verifyService(pepperEnv, target, service) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+    for (t in targetHosts) {
+        try {
+            salt.commandStatus(pepperEnv, t, "service ${service} status", 'running')
+        } catch (Exception er) {
+            common.errorMsg(er)
+            if (INTERACTIVE.toBoolean()) {
+                input message: "${service} service is not running correctly on ${t}. Please fix it first manually and only then click on PROCEED."
+            } else {
+                throw new Exception("${service} service is not running correctly on ${t}")
+            }
+        }
+    }
+}
+
+def verifyCeph(pepperEnv, target, type) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+    for (t in targetHosts) {
+        def hostname = salt.getReturnValues(salt.getPillar(pepperEnv, t, 'linux:network:hostname'))
+        try {
+            salt.commandStatus(pepperEnv, t, "systemctl status ceph-${type}${hostname}", 'running')
+        } catch (Exception er) {
+            common.errorMsg(er)
+            if (INTERACTIVE.toBoolean()) {
+                input message: "Ceph-${type}${hostname} service is not running correctly on ${t}. Please fix it first manually and only then click on PROCEED."
+            } else {
+                throw new Exception("Ceph-${type}${hostname} service is not running correctly on ${t}")
+            }
+        }
+    }
+}
+
+def verifyCephOsds(pepperEnv, target) {
+    def salt = new com.mirantis.mk.Salt()
+    def common = new com.mirantis.mk.Common()
+    def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+    for (t in targetHosts) {
+        def osd_ids = []
+        // get list of osd disks of the host
+        salt.runSaltProcessStep(pepperEnv, t, 'saltutil.sync_grains', [], null, true, 5)
+        def cephGrain = salt.getGrain(pepperEnv, t, 'ceph')
+        if(cephGrain['return'].isEmpty()){
+            throw new Exception("Ceph salt grain cannot be found!")
+        }
+        common.print(cephGrain)
+        def ceph_disks = cephGrain['return'][0].values()[0].values()[0]['ceph_disk']
+        for (i in ceph_disks) {
+            def osd_id = i.getKey().toString()
+            osd_ids.add('osd.' + osd_id)
+            print("Will check osd." + osd_id)
+        }
+        for (i in osd_ids) {
+            try {
+                salt.commandStatus(pepperEnv, t, "ceph osd tree | grep -w ${i}", 'up')
+            } catch (Exception er) {
+                common.errorMsg(er)
+                if (INTERACTIVE.toBoolean()) {
+                    input message: "Ceph ${i} is not running correctly on ${t}. Please fix it first manually and only then click on PROCEED."
+                } else {
+                    throw new Exception("Ceph ${i} is not running correctly on ${t}")
+                }
+            }
+        }
+    }
+}
+
+
+timeout(time: 12, unit: 'HOURS') {
+    node() {
+        try {
+
+            stage('Setup virtualenv for Pepper') {
+                python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+            }
+
+            // TODO, add possibility to update just specific components like kernel, openstack, contrail, ovs, rabbitmq, galera, etc.
+
+            /*
+                * Update section
+            */
+            if (updates.contains("cfg")) {
+                def target = 'I@salt:master'
+                if (salt.testTarget(pepperEnv, target)) {
+                    def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
+                    getCfgNodeProvider(pepperEnv, master)
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        virsh.liveSnapshotPresent(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME)
+                    } else {
+                        saltMasterBackup(pepperEnv)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t, target)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                }
+            }
+
+            if (updates.contains("ctl")) {
+                def target = CTL_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'ctl'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyAPIs(pepperEnv, target)
+                }
+            }
+
+            if (updates.contains("prx")) {
+                def target = PRX_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'prx'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'nginx')
+                }
+            }
+
+            if (updates.contains("msg")) {
+                def target = MSG_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'msg'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'rabbitmq-server')
+                }
+            }
+
+            if (updates.contains("dbs")) {
+                def target = DBS_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    backupGalera(pepperEnv)
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'dbs'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (REBOOT.toBoolean() || PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                            verifyGalera(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                        verifyGalera(pepperEnv, target)
+                    }
+                }
+            }
+
+            if (updates.contains("ntw")) {
+                def target = NTW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    backupContrail(pepperEnv)
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'ntw'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t, 'contrail')
+                            highstate(pepperEnv, t)
+                            verifyContrail(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target, 'contrail')
+                        highstate(pepperEnv, target)
+                        verifyContrail(pepperEnv, target)
+                    }
+                }
+            }
+
+            if (updates.contains("nal")) {
+                def target = NAL_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'nal'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t, 'contrail')
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target, 'contrail')
+                        highstate(pepperEnv, target)
+                    }
+                    verifyContrail(pepperEnv, target)
+                }
+            }
+
+            if (updates.contains("gtw-virtual")) {
+                def target = GTW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'gtw'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'neutron-dhcp-agent')
+                }
+            }
+
+            if (updates.contains("cmn")) {
+                def target = CMN_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'cmn'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    } else {
+                        backupCeph(pepperEnv, target)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyCeph(pepperEnv, target, 'mon@')
+                }
+            }
+
+            if (updates.contains("rgw")) {
+                def target = RGW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'rgw'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyCeph(pepperEnv, target, 'radosgw@rgw.')
+                }
+            }
+
+            if (updates.contains("log")) {
+                def target = LOG_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'log'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                }
+            }
+
+            if (updates.contains("mon")) {
+                def target = MON_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'mon'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                }
+            }
+
+            if (updates.contains("mtr")) {
+                def target = MTR_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'mtr'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                }
+            }
+
+            if (updates.contains("cid")) {
+                def target = CID_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        def generalTarget = 'cid'
+                        liveSnapshot(pepperEnv, target, generalTarget)
+                    }
+                    updatePkgs(pepperEnv, target, 'cicd')
+                    highstate(pepperEnv, target)
+                    verifyService(pepperEnv, target, 'docker')
+                }
+            }
+
+            //
+            //physical machines update CMP_TARGET
+            //
+            if (updates.contains("cmp")) {
+                def target = CMP_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'nova-compute')
+                }
+            }
+
+            if (updates.contains("kvm")) {
+                def target = KVM_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'libvirt-bin')
+                }
+            }
+
+            if (updates.contains("osd")) {
+                def target = CEPH_OSD_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyCephOsds(pepperEnv, target)
+                }
+            }
+
+            if (updates.contains("gtw-physical")) {
+                def target = GTW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            updatePkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        updatePkgs(pepperEnv, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'neutron-dhcp-agent')
+                }
+            }
+
+            /*
+                * Rollback section
+            */
+          /*  if (rollbacks.contains("cfg")) {
+                if (salt.testTarget(pepperEnv, 'I@salt:master')) {
+                    stage('ROLLBACK_CFG') {
+                        input message: "To rollback CFG nodes run the following commands on kvm nodes hosting the CFG nodes: virsh destroy cfg0X.domain; virsh define /var/lib/libvirt/images/cfg0X.domain.xml; virsh start cfg0X.domain; virsh snapshot-delete cfg0X.domain --metadata ${SNAPSHOT_NAME}; rm /var/lib/libvirt/images/cfg0X.domain.${SNAPSHOT_NAME}.qcow2; rm /var/lib/libvirt/images/cfg0X.domain.xml; At the end restart 'docker' service on all cicd nodes and run 'linux.system.repo' Salt states on cicd nodes. After running the previous commands current pipeline job will be killed."
+                        //rollbackSaltMaster(pepperEnv, 'I@salt:master')
+                        //finishSaltMasterRollback(pepperEnv, 'I@salt:master')
+                    }
+                }
+            } */
+
+            if (rollbacks.contains("ctl")) {
+                def target = CTL_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'ctl')
+                        verifyAPIs(pepperEnv, target)
+                    } else {
+                        removeNode(pepperEnv, target, 'ctl')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("prx")) {
+                def target = PRX_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'prx')
+                        verifyService(pepperEnv, target, 'nginx')
+                    } else {
+                        removeNode(pepperEnv, target, 'prx')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("msg")) {
+                def target = MSG_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'msg')
+                        salt.enforceState(pepperEnv, target, 'rabbitmq')
+                        verifyService(pepperEnv, target, 'rabbitmq-server')
+                    } else {
+                        removeNode(pepperEnv, target, 'msg')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("dbs")) {
+                def target = DBS_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'dbs')
+                        clusterGalera(pepperEnv)
+                        verifyGalera(pepperEnv, target)
+                    } else {
+                        removeNode(pepperEnv, target, 'dbs')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("ntw")) {
+                def target = NTW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'ntw')
+                        verifyContrail(pepperEnv, target)
+                    } else {
+                        removeNode(pepperEnv, target, 'ntw')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("nal")) {
+                def target = NAL_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'nal')
+                        verifyContrail(pepperEnv, target)
+                    } else {
+                        removeNode(pepperEnv, target, 'nal')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("gtw-virtual")) {
+                def target = GTW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'gtw')
+                        verifyService(pepperEnv, target, 'neutron-dhcp-agent')
+                    } else {
+                        removeNode(pepperEnv, target, 'gtw')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("cmn")) {
+                def target = CMN_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'cmn')
+                        verifyCeph(pepperEnv, target, 'mon@')
+                    } else {
+                        removeNode(pepperEnv, target, 'cmn')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("rgw")) {
+                def target = RGW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'rgw')
+                        verifyCeph(pepperEnv, target, 'radosgw@rgw.')
+                    } else {
+                        removeNode(pepperEnv, target, 'rgw')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("log")) {
+                def target = LOG_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'log')
+                    } else {
+                        removeNode(pepperEnv, target, 'log')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("mon")) {
+                def target = MON_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'mon')
+                    } else {
+                        removeNode(pepperEnv, target, 'mon')
+                    }
+                }
+            }
+
+            if (rollbacks.contains("mtr")) {
+                def target = MTR_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
+                        rollback(pepperEnv, target, 'mtr')
+                    } else {
+                        removeNode(pepperEnv, target, 'mtr')
+                    }
+                }
+            }
+            /*
+            if (ROLLBACK_CID.toBoolean()) {
+                def target = 'cid*'
+                if (salt.testTarget(pepperEnv, target)) {
+                    stage('ROLLBACK_CID') {
+                        input message: "To rollback CICD nodes run the following commands on kvm nodes hosting the cicd nodes: virsh destroy cid0X.domain; virsh define /var/lib/libvirt/images/cid0X.domain.xml; virsh start cid0X.domain; virsh snapshot-delete cid0X.domain --metadata ${SNAPSHOT_NAME}; rm /var/lib/libvirt/images/cid0X.domain.${SNAPSHOT_NAME}.qcow2; rm /var/lib/libvirt/images/cid0X.domain.xml; At the end restart 'docker' service on all cicd nodes and run 'linux.system.repo' Salt states on cicd nodes. After running the previous commands current pipeline job will be killed."
+                    }
+                }
+            } */
+
+            //
+            //physical machines rollback CMP_TARGET
+            //
+            if (rollbacks.contains("cmp")) {
+                def target = CMP_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            rollbackPkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        rollbackPkgs(pepperEnv, target, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'nova-compute')
+                }
+            }
+
+            if (rollbacks.contains("kvm")) {
+                def target = KVM_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            rollbackPkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        rollbackPkgs(pepperEnv, target, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'libvirt-bin')
+                }
+            }
+
+            if (rollbacks.contains("osd")) {
+                def target = CEPH_OSD_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            rollbackPkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        rollbackPkgs(pepperEnv, target, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyCephOsds(pepperEnv, target)
+                }
+            }
+
+            if (rollbacks.contains("gtw-physical")) {
+                def target = GTW_TARGET
+                if (salt.testTarget(pepperEnv, target)) {
+                    if (PER_NODE.toBoolean()) {
+                        def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+                        for (t in targetHosts) {
+                            rollbackPkgs(pepperEnv, t)
+                            highstate(pepperEnv, t)
+                        }
+                    } else {
+                        rollbackPkgs(pepperEnv, target, target)
+                        highstate(pepperEnv, target)
+                    }
+                    verifyService(pepperEnv, target, 'neutron-dhcp-agent')
+                }
+            }
+
+            /*
+                * Merge snapshots section
+            */
+            if (merges.contains("cfg")) {
+                if (salt.testTarget(pepperEnv, 'I@salt:master')) {
+                    mergeSnapshot(pepperEnv, 'I@salt:master')
+                }
+            }
+
+            if (merges.contains("ctl")) {
+                if (salt.testTarget(pepperEnv, CTL_TARGET)) {
+                    mergeSnapshot(pepperEnv, CTL_TARGET, 'ctl')
+                    verifyService(pepperEnv, CTL_TARGET, 'nova-api')
+                }
+            }
+
+            if (merges.contains("prx")) {
+                if (salt.testTarget(pepperEnv, PRX_TARGET)) {
+                    mergeSnapshot(pepperEnv, PRX_TARGET, 'prx')
+                    verifyService(pepperEnv, PRX_TARGET, 'nginx')
+                }
+            }
+
+            if (merges.contains("msg")) {
+                if (salt.testTarget(pepperEnv, MSG_TARGET)) {
+                    mergeSnapshot(pepperEnv, MSG_TARGET, 'msg')
+                    verifyService(pepperEnv, MSG_TARGET, 'rabbitmq-server')
+                }
+            }
+
+            if (merges.contains("dbs")) {
+                if (salt.testTarget(pepperEnv, DBS_TARGET)) {
+                    mergeSnapshot(pepperEnv, DBS_TARGET, 'dbs')
+                    verifyGalera(pepperEnv, DBS_TARGET)
+                    backupGalera(pepperEnv)
+                }
+            }
+
+            if (merges.contains("ntw")) {
+                if (salt.testTarget(pepperEnv, NTW_TARGET)) {
+                    mergeSnapshot(pepperEnv, NTW_TARGET, 'ntw')
+                    verifyContrail(pepperEnv, NTW_TARGET)
+                    backupContrail(pepperEnv)
+                }
+            }
+
+            if (merges.contains("nal")) {
+                if (salt.testTarget(pepperEnv, NAL_TARGET)) {
+                    mergeSnapshot(pepperEnv, NAL_TARGET, 'nal')
+                    verifyContrail(pepperEnv, NAL_TARGET)
+                }
+            }
+
+            if (merges.contains("gtw-virtual")) {
+                if (salt.testTarget(pepperEnv, GTW_TARGET)) {
+                    mergeSnapshot(pepperEnv, GTW_TARGET, 'gtw')
+                    verifyService(pepperEnv, GTW_TARGET, 'neutron-dhcp-agent')
+                }
+            }
+
+            if (merges.contains("cmn")) {
+                if (salt.testTarget(pepperEnv, CMN_TARGET)) {
+                    mergeSnapshot(pepperEnv, CMN_TARGET, 'cmn')
+                    verifyCeph(pepperEnv, CMN_TARGET, 'mon@')
+                    backupCeph(pepperEnv, CMN_TARGET)
+                }
+            }
+
+            if (merges.contains("rgw")) {
+                if (salt.testTarget(pepperEnv, RGW_TARGET)) {
+                    mergeSnapshot(pepperEnv, RGW_TARGET, 'rgw')
+                    verifyCeph(pepperEnv, RGW_TARGET, 'radosgw@rgw.')
+                }
+            }
+
+            if (merges.contains("log")) {
+                if (salt.testTarget(pepperEnv, LOG_TARGET)) {
+                    mergeSnapshot(pepperEnv, LOG_TARGET. 'log')
+                }
+            }
+
+            if (merges.contains("mon")) {
+                if (salt.testTarget(pepperEnv, MON_TARGET)) {
+                    mergeSnapshot(pepperEnv, MON_TARGET, 'mon')
+                }
+            }
+
+            if (merges.contains("mtr")) {
+                if (salt.testTarget(pepperEnv, MTR_TARGET)) {
+                    mergeSnapshot(pepperEnv, MTR_TARGET, 'mtr')
+                }
+            }
+
+            if (merges.contains("cid")) {
+                if (salt.testTarget(pepperEnv, CID_TARGET)) {
+                    mergeSnapshot(pepperEnv, CID_TARGET, 'cid')
+                    verifyService(pepperEnv, CID_TARGET, 'docker')
+                }
+            }
+
+            if (RESTORE_GALERA.toBoolean()) {
+                restoreGalera(pepperEnv)
+                verifyGalera(pepperEnv, DBS_TARGET)
+            }
+
+            if (RESTORE_CONTRAIL_DB.toBoolean()) {
+                restoreContrailDb(pepperEnv)
+                // verification is already present in restore pipelines
+            }
+
+        } catch (Throwable e) {
+            // If there was an error or exception thrown, the build failed
+            currentBuild.result = "FAILURE"
+            currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+            throw e
+        }
+    }
+}
diff --git a/docker-cleanup-pipeline.groovy b/docker-cleanup-pipeline.groovy
index 710b1bb..388793a 100644
--- a/docker-cleanup-pipeline.groovy
+++ b/docker-cleanup-pipeline.groovy
@@ -28,7 +28,7 @@
           """, false)
     }
     stage("Run docker system prune"){
-      salt.cmdRun(pepperEnv, 'I@jenkins:slave', "docker system prune -f")
+      salt.cmdRun(pepperEnv, 'I@jenkins:slave', "docker system prune -f", false) // dont verify the result
     }
   }
 }
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 39590e5..c2d4943 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -54,6 +54,8 @@
             def clusterDomain = templateContext.default_context.cluster_domain
             def clusterName = templateContext.default_context.cluster_name
             def saltMaster = templateContext.default_context.salt_master_hostname
+            def localRepositories = templateContext.default_context.local_repositories.toBoolean()
+            def offlineDeployment = templateContext.default_context.offline_deployment.toBoolean()
             def cutterEnv = "${env.WORKSPACE}/cutter"
             def jinjaEnv = "${env.WORKSPACE}/jinja"
             def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
@@ -98,6 +100,7 @@
                     sh "git init"
                     ssh.agentSh("git submodule add ${sharedReclassUrl} 'classes/system'")
                 }
+
                 def sharedReclassBranch = templateContext.default_context.shared_reclass_branch
                 // Use refspec if exists first of all
                 if (sharedReclassBranch.toString().startsWith('refs/')) {
@@ -161,6 +164,16 @@
                 }
             }
 
+            if(localRepositories && !offlineDeployment){
+                def aptlyModelUrl = templateContext.default_context.local_model_url
+                dir(path: modelEnv) {
+                    ssh.agentSh "git submodule add \"${aptlyModelUrl}\" \"classes/cluster/${clusterName}/cicd/aptly\""
+                        if(!(mcpVersion in ["nightly", "testing", "stable"])){
+                        ssh.agentSh "cd \"classes/cluster/${clusterName}/cicd/aptly\";git fetch --tags;git checkout ${mcpVersion}"
+                    }
+                }
+            }
+
             stage('Generate new SaltMaster node') {
                 def nodeFile = "${modelEnv}/nodes/${saltMaster}.${clusterDomain}.yml"
                 def nodeString = """classes:
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 8f1b612..21134a2 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -145,7 +145,7 @@
     salt.enforceState(pepperEnv, 'I@xtrabackup:server', 'xtrabackup')
     salt.enforceState(pepperEnv, 'I@xtrabackup:client', 'openssh.client')
     salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
-    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
+    salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh -f -s'")
 
     def databases = salt.cmdRun(pepperEnv, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
     if(databases && databases != ""){
@@ -249,7 +249,7 @@
     }
 
     if ((OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == true) && (SKIP_VM_RELAUNCH.toBoolean() == false)) {
-        salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
+        salt.cmdRun(pepperEnv, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh -f -s'")
 
         salt.enforceState(pepperEnv, 'I@salt:control', 'salt.control')
 
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index ffc035a..10ec378 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -25,6 +25,16 @@
             stage('Ask for manual confirmation') {
                 input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore Cassandra?"
             }
+            try {
+                salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.stop', ['neutron-server'], null, true)
+            } catch (Exception er) {
+                common.warningMsg('neutron-server service already stopped')
+            }
+            try {
+                salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ['supervisor-config'], null, true)
+            } catch (Exception er) {
+                common.warningMsg('Supervisor-config service already stopped')
+            }
             // Cassandra restore section
             try {
                 salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ['supervisor-database'], null, true)
@@ -70,6 +80,7 @@
             sleep(5)
 
             salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.restart', ['supervisor-database'], null, true)
+            salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.start', ['neutron-server'], null, true)
 
             // wait until contrail-status is up
             salt.commandStatus(pepperEnv, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
diff --git a/test-salt-model-node.groovy b/test-salt-model-node.groovy
index 3893066..7fc333d 100644
--- a/test-salt-model-node.groovy
+++ b/test-salt-model-node.groovy
@@ -10,6 +10,7 @@
  *  SYSTEM_GIT_URL
  *  SYSTEM_GIT_REF
  *  FORMULAS_SOURCE
+ *  RECLASS_VERSION
  *  MAX_CPU_PER_JOB
  *  LEGACY_TEST_MODE
  *  RECLASS_IGNORE_CLASS_NOTFOUND
@@ -63,7 +64,7 @@
             def workspace = common.getWorkspace()
             common.infoMsg("Running salt model test for node ${NODE_TARGET} in cluster ${CLUSTER_NAME}")
             try {
-              saltModelTesting.setupAndTestNode(NODE_TARGET, CLUSTER_NAME, EXTRA_FORMULAS, workspace, FORMULAS_SOURCE, FORMULAS_REVISION, MAX_CPU_PER_JOB.toInteger(), RECLASS_IGNORE_CLASS_NOTFOUND, LEGACY_TEST_MODE, APT_REPOSITORY, APT_REPOSITORY_GPG)
+              saltModelTesting.setupAndTestNode(NODE_TARGET, CLUSTER_NAME, EXTRA_FORMULAS, workspace, FORMULAS_SOURCE, FORMULAS_REVISION, RECLASS_VERSION, MAX_CPU_PER_JOB.toInteger(), RECLASS_IGNORE_CLASS_NOTFOUND, LEGACY_TEST_MODE, APT_REPOSITORY, APT_REPOSITORY_GPG)
             } catch (Exception e) {
               if (e.getMessage() == "script returned exit code 124") {
                 common.errorMsg("Impossible to test node due to timeout of salt-master, ABORTING BUILD")
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index ca07fc1..d6279f9 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -105,6 +105,7 @@
     [$class: 'StringParameterValue', name: 'FORMULAS_REVISION', value: FORMULAS_REVISION],
     [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: CREDENTIALS_ID],
     [$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: SYSTEM_GIT_URL],
+    [$class: 'StringParameterValue', name: 'RECLASS_VERSION', value: RECLASS_VERSION],
     [$class: 'StringParameterValue', name: 'MAX_CPU_PER_JOB', value: MAX_CPU_PER_JOB],
     [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: SYSTEM_GIT_REF],
     [$class: 'BooleanParameterValue', name: 'LEGACY_TEST_MODE', value: LEGACY_TEST_MODE.toBoolean()],