diff --git a/cicd-lab-pipeline.groovy b/cicd-lab-pipeline.groovy
index 37643cf..3f45233 100644
--- a/cicd-lab-pipeline.groovy
+++ b/cicd-lab-pipeline.groovy
@@ -32,317 +32,316 @@
 salt = new com.mirantis.mk.Salt()
 orchestrate = new com.mirantis.mk.Orchestrate()
 _MAX_PERMITTED_STACKS = 2
-timestamps {
-    node {
+
+node {
+    try {
+        // connection objects
+        def openstackCloud
+        def saltMaster
+
+        // value defaults
+        def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
+        def openstackEnv = "${env.WORKSPACE}/venv"
+
         try {
-            // connection objects
-            def openstackCloud
-            def saltMaster
+            sshPubKey = SSH_PUBLIC_KEY
+        } catch (MissingPropertyException e) {
+            sshPubKey = false
+        }
 
-            // value defaults
-            def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
-            def openstackEnv = "${env.WORKSPACE}/venv"
+        if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
+            error("If you want to reuse existing stack you need to provide it's name")
+        }
 
-            try {
-                sshPubKey = SSH_PUBLIC_KEY
-            } catch (MissingPropertyException e) {
-                sshPubKey = false
-            }
-
-            if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
-                error("If you want to reuse existing stack you need to provide it's name")
-            }
-
-            if (HEAT_STACK_REUSE.toBoolean() == false) {
-                // Don't allow to set custom heat stack name
-                wrap([$class: 'BuildUser']) {
-                    if (env.BUILD_USER_ID) {
-                        HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
-                    } else {
-                        HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
-                    }
-                    currentBuild.description = HEAT_STACK_NAME
-                }
-            }
-
-            //
-            // Bootstrap
-            //
-
-            stage ('Download Heat templates') {
-                git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
-            }
-
-            stage('Install OpenStack CLI') {
-                openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
-            }
-
-            stage('Connect to OpenStack cloud') {
-                openstackCloud = openstack.createOpenstackEnv(
-                    OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS,
-                    OPENSTACK_API_PROJECT, OPENSTACK_API_PROJECT_DOMAIN,
-                    OPENSTACK_API_PROJECT_ID, OPENSTACK_API_USER_DOMAIN,
-                    OPENSTACK_API_VERSION)
-                openstack.getKeystoneToken(openstackCloud, openstackEnv)
-                wrap([$class: 'BuildUser']) {
-                    if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !HEAT_STACK_REUSE.toBoolean()) {
-                        def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
-                        if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
-                            HEAT_STACK_DELETE = "false"
-                            throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
-                        }
-                    }
-                }
-            }
-
-            if (HEAT_STACK_REUSE.toBoolean() == false) {
-                stage('Launch new Heat stack') {
-                    envParams = [
-                        'instance_zone': HEAT_STACK_ZONE,
-                        'public_net': HEAT_STACK_PUBLIC_NET
-                    ]
-                    openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
-                }
-            }
-
-            stage('Connect to Salt master') {
-                def saltMasterPort
-                try {
-                    saltMasterPort = SALT_MASTER_PORT
-                } catch (MissingPropertyException e) {
-                    saltMasterPort = 6969
-                }
-                saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
-                currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
-                saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}"
-                saltMaster = salt.connection(saltMasterUrl, SALT_MASTER_CREDENTIALS)
-            }
-
-            //
-            // Install
-            //
-
-            stage('Install core infra') {
-                // salt.master, reclass
-                // refresh_pillar
-                // sync_all
-                // linux,openssh,salt.minion.ntp
-
-                orchestrate.installFoundationInfra(saltMaster)
-                orchestrate.validateFoundationInfra(saltMaster)
-            }
-
-            stage("Deploy GlusterFS") {
-                salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
-                retry(2) {
-                    salt.enforceState(saltMaster, 'ci01*', 'glusterfs.server.setup', true)
-                }
-                sleep(5)
-                salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
-
-                timeout(5) {
-                    println "Waiting for GlusterFS volumes to get mounted.."
-                    salt.cmdRun(saltMaster, 'I@glusterfs:client', 'while true; do systemctl -a|grep "GlusterFS File System"|grep -v mounted >/dev/null || break; done')
-                }
-                print common.prettyPrint(salt.cmdRun(saltMaster, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"'))
-            }
-
-            stage("Deploy GlusterFS") {
-                salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy,keepalived')
-            }
-
-            stage("Setup Docker Swarm") {
-                salt.enforceState(saltMaster, 'I@docker:host', 'docker.host', true)
-                salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.swarm', true)
-                salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'salt', true)
-                salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.flush')
-                salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.update')
-                salt.enforceState(saltMaster, 'I@docker:swarm', 'docker.swarm', true)
-                print common.prettyPrint(salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'docker node ls'))
-            }
-
-            stage("Configure OSS services") {
-                salt.enforceState(saltMaster, 'I@devops_portal:config', 'devops_portal.config')
-                salt.enforceState(saltMaster, 'I@rundeck:server', 'rundeck.server')
-            }
-
-            stage("Deploy Docker services") {
-                // We need /etc/aptly-publisher.yaml to be present before
-                // services are deployed
-                // XXX: for some weird unknown reason, refresh_pillar is
-                // required to execute here
-                salt.runSaltProcessStep(saltMaster, 'I@aptly:publisher', 'saltutil.refresh_pillar', [], null, true)
-                salt.enforceState(saltMaster, 'I@aptly:publisher', 'aptly.publisher', true)
-                retry(3) {
-                    sleep(5)
-                    salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
-                }
-                // XXX: Workaround to have `/var/lib/jenkins` on all
-                // nodes where are jenkins_slave services are created.
-                salt.runSaltProcessStep(saltMaster, 'I@docker:swarm', 'cmd.run', ['mkdir -p /var/lib/jenkins'])
-            }
-
-            stage("Configure CI/CD services") {
-                salt.syncAll(saltMaster, '*')
-
-                // Aptly
-                timeout(10) {
-                    println "Waiting for Aptly to come up.."
-                    retry(2) {
-                        // XXX: retry to workaround magical VALUE_TRIMMED
-                        // response from salt master + to give slow cloud some
-                        // more time to settle down
-                        salt.cmdRun(saltMaster, 'I@aptly:server', 'while true; do curl -sf http://172.16.10.254:8084/api/version >/dev/null && break; done')
-                    }
-                }
-                salt.enforceState(saltMaster, 'I@aptly:server', 'aptly', true)
-
-                // OpenLDAP
-                timeout(10) {
-                    println "Waiting for OpenLDAP to come up.."
-                    salt.cmdRun(saltMaster, 'I@openldap:client', 'while true; do curl -sf ldap://172.16.10.254 >/dev/null && break; done')
-                }
-                salt.enforceState(saltMaster, 'I@openldap:client', 'openldap', true)
-
-                // Gerrit
-                timeout(10) {
-                    println "Waiting for Gerrit to come up.."
-                    salt.cmdRun(saltMaster, 'I@gerrit:client', 'while true; do curl -sf 172.16.10.254:8080 >/dev/null && break; done')
-                }
-                salt.enforceState(saltMaster, 'I@gerrit:client', 'gerrit', true)
-
-                // Jenkins
-                timeout(10) {
-                    println "Waiting for Jenkins to come up.."
-                    salt.cmdRun(saltMaster, 'I@jenkins:client', 'while true; do curl -sf 172.16.10.254:8081 >/dev/null && break; done')
-                }
-                retry(2) {
-                    // XXX: needs retry as first run installs python-jenkins
-                    // thus make jenkins modules available for second run
-                    salt.enforceState(saltMaster, 'I@jenkins:client', 'jenkins', true)
-                }
-
-                // Postgres client - initialize OSS services databases
-                timeout(300){
-                    println "Waiting for postgresql database to come up.."
-                    salt.cmdRun(saltMaster, 'I@postgresql:client', 'while true; do if docker service logs postgresql_db | grep "ready to accept"; then break; else sleep 5; fi; done')
-                }
-                salt.enforceState(saltMaster, 'I@postgresql:client', 'postgresql.client', true, false)
-
-                // Setup postgres database with integration between
-                // Pushkin notification service and Security Monkey security audit service
-                timeout(10) {
-                    println "Waiting for Pushkin to come up.."
-                    salt.cmdRun(saltMaster, 'I@postgresql:client', 'while true; do curl -sf 172.16.10.254:8887/apps >/dev/null && break; done')
-                }
-                salt.enforceState(saltMaster, 'I@postgresql:client', 'postgresql.client', true)
-
-                // Rundeck
-                timeout(10) {
-                    println "Waiting for Rundeck to come up.."
-                    salt.cmdRun(saltMaster, 'I@rundeck:client', 'while true; do curl -sf 172.16.10.254:4440 >/dev/null && break; done')
-                }
-                salt.enforceState(saltMaster, 'I@rundeck:client', 'rundeck.client', true)
-
-                // Elasticsearch
-                timeout(10) {
-                    println 'Waiting for Elasticsearch to come up..'
-                    salt.cmdRun(saltMaster, 'I@elasticsearch:client', 'while true; do curl -sf 172.16.10.254:9200 >/dev/null && break; done')
-                }
-                salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client', true)
-            }
-
-            stage("Finalize") {
-                //
-                // Deploy user's ssh key
-                //
-                def adminUser
-                def authorizedKeysFile
-                def adminUserCmdOut = salt.cmdRun(saltMaster, 'I@salt:master', "[ ! -d /home/ubuntu ] || echo 'ubuntu user exists'")
-                if (adminUserCmdOut =~ /ubuntu user exists/) {
-                    adminUser = "ubuntu"
-                    authorizedKeysFile = "/home/ubuntu/.ssh/authorized_keys"
+        if (HEAT_STACK_REUSE.toBoolean() == false) {
+            // Don't allow to set custom heat stack name
+            wrap([$class: 'BuildUser']) {
+                if (env.BUILD_USER_ID) {
+                    HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
                 } else {
-                    adminUser = "root"
-                    authorizedKeysFile = "/root/.ssh/authorized_keys"
+                    HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
                 }
+                currentBuild.description = HEAT_STACK_NAME
+            }
+        }
 
-                if (sshPubKey) {
-                    println "Deploying provided ssh key at ${authorizedKeysFile}"
-                    salt.cmdRun(saltMaster, '*', "echo '${sshPubKey}' | tee -a ${authorizedKeysFile}")
+        //
+        // Bootstrap
+        //
+
+        stage ('Download Heat templates') {
+            git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
+        }
+
+        stage('Install OpenStack CLI') {
+            openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
+        }
+
+        stage('Connect to OpenStack cloud') {
+            openstackCloud = openstack.createOpenstackEnv(
+                OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS,
+                OPENSTACK_API_PROJECT, OPENSTACK_API_PROJECT_DOMAIN,
+                OPENSTACK_API_PROJECT_ID, OPENSTACK_API_USER_DOMAIN,
+                OPENSTACK_API_VERSION)
+            openstack.getKeystoneToken(openstackCloud, openstackEnv)
+            wrap([$class: 'BuildUser']) {
+                if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !HEAT_STACK_REUSE.toBoolean()) {
+                    def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
+                    if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
+                        HEAT_STACK_DELETE = "false"
+                        throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
+                    }
                 }
+            }
+        }
 
-                //
-                // Generate docs
-                //
+        if (HEAT_STACK_REUSE.toBoolean() == false) {
+            stage('Launch new Heat stack') {
+                envParams = [
+                    'instance_zone': HEAT_STACK_ZONE,
+                    'public_net': HEAT_STACK_PUBLIC_NET
+                ]
+                openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+            }
+        }
+
+        stage('Connect to Salt master') {
+            def saltMasterPort
+            try {
+                saltMasterPort = SALT_MASTER_PORT
+            } catch (MissingPropertyException e) {
+                saltMasterPort = 6969
+            }
+            saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
+            currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
+            saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}"
+            saltMaster = salt.connection(saltMasterUrl, SALT_MASTER_CREDENTIALS)
+        }
+
+        //
+        // Install
+        //
+
+        stage('Install core infra') {
+            // salt.master, reclass
+            // refresh_pillar
+            // sync_all
+            // linux,openssh,salt.minion.ntp
+
+            orchestrate.installFoundationInfra(saltMaster)
+            orchestrate.validateFoundationInfra(saltMaster)
+        }
+
+        stage("Deploy GlusterFS") {
+            salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
+            retry(2) {
+                salt.enforceState(saltMaster, 'ci01*', 'glusterfs.server.setup', true)
+            }
+            sleep(5)
+            salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
+
+            timeout(5) {
+                println "Waiting for GlusterFS volumes to get mounted.."
+                salt.cmdRun(saltMaster, 'I@glusterfs:client', 'while true; do systemctl -a|grep "GlusterFS File System"|grep -v mounted >/dev/null || break; done')
+            }
+            print common.prettyPrint(salt.cmdRun(saltMaster, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"'))
+        }
+
+        stage("Deploy GlusterFS") {
+            salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy,keepalived')
+        }
+
+        stage("Setup Docker Swarm") {
+            salt.enforceState(saltMaster, 'I@docker:host', 'docker.host', true)
+            salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.swarm', true)
+            salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'salt', true)
+            salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.flush')
+            salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.update')
+            salt.enforceState(saltMaster, 'I@docker:swarm', 'docker.swarm', true)
+            print common.prettyPrint(salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'docker node ls'))
+        }
+
+        stage("Configure OSS services") {
+            salt.enforceState(saltMaster, 'I@devops_portal:config', 'devops_portal.config')
+            salt.enforceState(saltMaster, 'I@rundeck:server', 'rundeck.server')
+        }
+
+        stage("Deploy Docker services") {
+            // We need /etc/aptly-publisher.yaml to be present before
+            // services are deployed
+            // XXX: for some weird unknown reason, refresh_pillar is
+            // required to execute here
+            salt.runSaltProcessStep(saltMaster, 'I@aptly:publisher', 'saltutil.refresh_pillar', [], null, true)
+            salt.enforceState(saltMaster, 'I@aptly:publisher', 'aptly.publisher', true)
+            retry(3) {
+                sleep(5)
+                salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
+            }
+            // XXX: Workaround to have `/var/lib/jenkins` on all
+            // nodes where are jenkins_slave services are created.
+            salt.runSaltProcessStep(saltMaster, 'I@docker:swarm', 'cmd.run', ['mkdir -p /var/lib/jenkins'])
+        }
+
+        stage("Configure CI/CD services") {
+            salt.syncAll(saltMaster, '*')
+
+            // Aptly
+            timeout(10) {
+                println "Waiting for Aptly to come up.."
+                retry(2) {
+                    // XXX: retry to workaround magical VALUE_TRIMMED
+                    // response from salt master + to give slow cloud some
+                    // more time to settle down
+                    salt.cmdRun(saltMaster, 'I@aptly:server', 'while true; do curl -sf http://172.16.10.254:8084/api/version >/dev/null && break; done')
+                }
+            }
+            salt.enforceState(saltMaster, 'I@aptly:server', 'aptly', true)
+
+            // OpenLDAP
+            timeout(10) {
+                println "Waiting for OpenLDAP to come up.."
+                salt.cmdRun(saltMaster, 'I@openldap:client', 'while true; do curl -sf ldap://172.16.10.254 >/dev/null && break; done')
+            }
+            salt.enforceState(saltMaster, 'I@openldap:client', 'openldap', true)
+
+            // Gerrit
+            timeout(10) {
+                println "Waiting for Gerrit to come up.."
+                salt.cmdRun(saltMaster, 'I@gerrit:client', 'while true; do curl -sf 172.16.10.254:8080 >/dev/null && break; done')
+            }
+            salt.enforceState(saltMaster, 'I@gerrit:client', 'gerrit', true)
+
+            // Jenkins
+            timeout(10) {
+                println "Waiting for Jenkins to come up.."
+                salt.cmdRun(saltMaster, 'I@jenkins:client', 'while true; do curl -sf 172.16.10.254:8081 >/dev/null && break; done')
+            }
+            retry(2) {
+                // XXX: needs retry as first run installs python-jenkins
+                // thus make jenkins modules available for second run
+                salt.enforceState(saltMaster, 'I@jenkins:client', 'jenkins', true)
+            }
+
+            // Postgres client - initialize OSS services databases
+            timeout(300){
+                println "Waiting for postgresql database to come up.."
+                salt.cmdRun(saltMaster, 'I@postgresql:client', 'while true; do if docker service logs postgresql_db | grep "ready to accept"; then break; else sleep 5; fi; done')
+            }
+            salt.enforceState(saltMaster, 'I@postgresql:client', 'postgresql.client', true, false)
+
+            // Setup postgres database with integration between
+            // Pushkin notification service and Security Monkey security audit service
+            timeout(10) {
+                println "Waiting for Pushkin to come up.."
+                salt.cmdRun(saltMaster, 'I@postgresql:client', 'while true; do curl -sf 172.16.10.254:8887/apps >/dev/null && break; done')
+            }
+            salt.enforceState(saltMaster, 'I@postgresql:client', 'postgresql.client', true)
+
+            // Rundeck
+            timeout(10) {
+                println "Waiting for Rundeck to come up.."
+                salt.cmdRun(saltMaster, 'I@rundeck:client', 'while true; do curl -sf 172.16.10.254:4440 >/dev/null && break; done')
+            }
+            salt.enforceState(saltMaster, 'I@rundeck:client', 'rundeck.client', true)
+
+            // Elasticsearch
+            timeout(10) {
+                println 'Waiting for Elasticsearch to come up..'
+                salt.cmdRun(saltMaster, 'I@elasticsearch:client', 'while true; do curl -sf 172.16.10.254:9200 >/dev/null && break; done')
+            }
+            salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client', true)
+        }
+
+        stage("Finalize") {
+            //
+            // Deploy user's ssh key
+            //
+            def adminUser
+            def authorizedKeysFile
+            def adminUserCmdOut = salt.cmdRun(saltMaster, 'I@salt:master', "[ ! -d /home/ubuntu ] || echo 'ubuntu user exists'")
+            if (adminUserCmdOut =~ /ubuntu user exists/) {
+                adminUser = "ubuntu"
+                authorizedKeysFile = "/home/ubuntu/.ssh/authorized_keys"
+            } else {
+                adminUser = "root"
+                authorizedKeysFile = "/root/.ssh/authorized_keys"
+            }
+
+            if (sshPubKey) {
+                println "Deploying provided ssh key at ${authorizedKeysFile}"
+                salt.cmdRun(saltMaster, '*', "echo '${sshPubKey}' | tee -a ${authorizedKeysFile}")
+            }
+
+            //
+            // Generate docs
+            //
+            try {
                 try {
-                    try {
-                        // Run sphinx state to install sphinx-build needed in
-                        // upcomming orchestrate
-                        salt.enforceState(saltMaster, 'I@sphinx:server', 'sphinx')
-                    } catch (Throwable e) {
-                        true
-                    }
-                    retry(3) {
-                        // TODO: fix salt.orchestrateSystem
-                        // print salt.orchestrateSystem(saltMaster, ['expression': '*', 'type': 'compound'], 'sphinx.orch.generate_doc')
-                        def out = salt.cmdRun(saltMaster, 'I@salt:master', 'salt-run state.orchestrate sphinx.orch.generate_doc || echo "Command execution failed"')
-                        print common.prettyPrint(out)
-                        if (out =~ /Command execution failed/) {
-                            throw new Exception("Command execution failed")
-                        }
-                    }
+                    // Run sphinx state to install sphinx-build needed in
+                    // upcomming orchestrate
+                    salt.enforceState(saltMaster, 'I@sphinx:server', 'sphinx')
                 } catch (Throwable e) {
-                    // We don't want sphinx docs to ruin whole build, so possible
-                    // errors are just ignored here
                     true
                 }
-                salt.enforceState(saltMaster, 'I@nginx:server', 'nginx')
-
-                def failedSvc = salt.cmdRun(saltMaster, '*', """systemctl --failed | grep -E 'loaded[ \t]+failed' && echo 'Command execution failed' || true""")
-                print common.prettyPrint(failedSvc)
-                if (failedSvc =~ /Command execution failed/) {
-                    common.errorMsg("Some services are not running. Environment may not be fully functional!")
+                retry(3) {
+                    // TODO: fix salt.orchestrateSystem
+                    // print salt.orchestrateSystem(saltMaster, ['expression': '*', 'type': 'compound'], 'sphinx.orch.generate_doc')
+                    def out = salt.cmdRun(saltMaster, 'I@salt:master', 'salt-run state.orchestrate sphinx.orch.generate_doc || echo "Command execution failed"')
+                    print common.prettyPrint(out)
+                    if (out =~ /Command execution failed/) {
+                        throw new Exception("Command execution failed")
+                    }
                 }
-
-                common.successMsg("""
-    ============================================================
-    Your CI/CD lab has been deployed and you can enjoy it:
-    Use sshuttle to connect to your private subnet:
-
-        sshuttle -r ${adminUser}@${saltMasterHost} 172.16.10.0/24
-
-    And visit services running at 172.16.10.254 (vip address):
-
-        9600    HAProxy statistics
-        8080    Gerrit
-        8081    Jenkins
-        8089    LDAP administration
-        4440    Rundeck
-        8084    DevOps Portal
-        8091    Docker swarm visualizer
-        8090    Reclass-generated documentation
-
-    If you provided SSH_PUBLIC_KEY, you can use it to login,
-    otherwise you need to get private key connected to this
-    heat template.
-
-    DON'T FORGET TO TERMINATE YOUR STACK WHEN YOU DON'T NEED IT!
-    ============================================================""")
+            } catch (Throwable e) {
+                // We don't want sphinx docs to ruin whole build, so possible
+                // errors are just ignored here
+                true
             }
-        } catch (Throwable e) {
-            // If there was an error or exception thrown, the build failed
-            currentBuild.result = "FAILURE"
-            throw e
-        } finally {
-            // Cleanup
-            if (HEAT_STACK_DELETE.toBoolean() == true) {
-                stage('Trigger cleanup job') {
-                    build(job: 'deploy-stack-cleanup', parameters: [
-                        [$class: 'StringParameterValue', name: 'STACK_NAME', value: HEAT_STACK_NAME],
-                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
-                    ])
-                }
+            salt.enforceState(saltMaster, 'I@nginx:server', 'nginx')
+
+            def failedSvc = salt.cmdRun(saltMaster, '*', """systemctl --failed | grep -E 'loaded[ \t]+failed' && echo 'Command execution failed' || true""")
+            print common.prettyPrint(failedSvc)
+            if (failedSvc =~ /Command execution failed/) {
+                common.errorMsg("Some services are not running. Environment may not be fully functional!")
+            }
+
+            common.successMsg("""
+============================================================
+Your CI/CD lab has been deployed and you can enjoy it:
+Use sshuttle to connect to your private subnet:
+
+    sshuttle -r ${adminUser}@${saltMasterHost} 172.16.10.0/24
+
+And visit services running at 172.16.10.254 (vip address):
+
+    9600    HAProxy statistics
+    8080    Gerrit
+    8081    Jenkins
+    8089    LDAP administration
+    4440    Rundeck
+    8084    DevOps Portal
+    8091    Docker swarm visualizer
+    8090    Reclass-generated documentation
+
+If you provided SSH_PUBLIC_KEY, you can use it to login,
+otherwise you need to get private key connected to this
+heat template.
+
+DON'T FORGET TO TERMINATE YOUR STACK WHEN YOU DON'T NEED IT!
+============================================================""")
+        }
+    } catch (Throwable e) {
+        // If there was an error or exception thrown, the build failed
+        currentBuild.result = "FAILURE"
+        throw e
+    } finally {
+        // Cleanup
+        if (HEAT_STACK_DELETE.toBoolean() == true) {
+            stage('Trigger cleanup job') {
+                build(job: 'deploy-stack-cleanup', parameters: [
+                    [$class: 'StringParameterValue', name: 'STACK_NAME', value: HEAT_STACK_NAME],
+                    [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
+                ])
             }
         }
     }
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 2ec5e14..2583fb1 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -17,84 +17,84 @@
 saltModelTesting = new com.mirantis.mk.SaltModelTesting()
 ssh = new com.mirantis.mk.Ssh()
 
-timestamps {
-    node("python&&docker") {
-        def templateEnv = "${env.WORKSPACE}/template"
-        def modelEnv = "${env.WORKSPACE}/model"
-        def testEnv = "${env.WORKSPACE}/test"
 
-        try {
-            def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
-            def clusterDomain = templateContext.default_context.cluster_domain
-            def clusterName = templateContext.default_context.cluster_name
-            def cutterEnv = "${env.WORKSPACE}/cutter"
-            def jinjaEnv = "${env.WORKSPACE}/jinja"
-            def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
-            def targetBranch = "feature/${clusterName}"
-            def templateBaseDir = "${env.WORKSPACE}/template"
-            def templateDir = "${templateEnv}/template/dir"
-            def templateOutputDir = templateBaseDir
-            def user
-            wrap([$class: 'BuildUser']) {
-                user = env.BUILD_USER_ID
-            }
+node("python&&docker") {
+    def templateEnv = "${env.WORKSPACE}/template"
+    def modelEnv = "${env.WORKSPACE}/model"
+    def testEnv = "${env.WORKSPACE}/test"
 
-            currentBuild.description = clusterName
-            print("Using context:\n" + COOKIECUTTER_TEMPLATE_CONTEXT)
+    try {
+        def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
+        def clusterDomain = templateContext.default_context.cluster_domain
+        def clusterName = templateContext.default_context.cluster_name
+        def cutterEnv = "${env.WORKSPACE}/cutter"
+        def jinjaEnv = "${env.WORKSPACE}/jinja"
+        def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
+        def targetBranch = "feature/${clusterName}"
+        def templateBaseDir = "${env.WORKSPACE}/template"
+        def templateDir = "${templateEnv}/template/dir"
+        def templateOutputDir = templateBaseDir
+        def user
+        wrap([$class: 'BuildUser']) {
+            user = env.BUILD_USER_ID
+        }
 
-            stage ('Download Cookiecutter template') {
-                git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, COOKIECUTTER_TEMPLATE_CREDENTIALS)
-            }
+        currentBuild.description = clusterName
+        print("Using context:\n" + COOKIECUTTER_TEMPLATE_CONTEXT)
 
-            stage ('Create empty reclass model') {
-                dir(path: modelEnv) {
-                    sh "rm -rfv .git"
-                    sh "git init"
+        stage ('Download Cookiecutter template') {
+            git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, COOKIECUTTER_TEMPLATE_CREDENTIALS)
+        }
 
-                    if (SHARED_RECLASS_URL != '') {
-                        ssh.agentSh "git submodule add \"${SHARED_RECLASS_URL}\" \"classes/system\""
-                        git.commitGitChanges(modelEnv, "Added new shared reclass submodule", "${user}@localhost", "${user}")
-                    }
+        stage ('Create empty reclass model') {
+            dir(path: modelEnv) {
+                sh "rm -rfv .git"
+                sh "git init"
+
+                if (SHARED_RECLASS_URL != '') {
+                    ssh.agentSh "git submodule add \"${SHARED_RECLASS_URL}\" \"classes/system\""
+                    git.commitGitChanges(modelEnv, "Added new shared reclass submodule", "${user}@localhost", "${user}")
                 }
             }
+        }
 
-            def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
-            for (product in productList) {
+        def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "stacklight"]
+        for (product in productList) {
 
-                // get templateOutputDir and productDir
-                if (product.startsWith("stacklight")) {
-                    templateOutputDir = "${env.WORKSPACE}/output/stacklight"
-                    try {
-                        productDir = "stacklight" + templateContext.default_context['stacklight_version']
-                    } catch (Throwable e) {
-                        productDir = "stacklight1"
-                    }
-                } else {
-                    templateOutputDir = "${env.WORKSPACE}/output/${product}"
-                    productDir = product
+            // get templateOutputDir and productDir
+            if (product.startsWith("stacklight")) {
+                templateOutputDir = "${env.WORKSPACE}/output/stacklight"
+                try {
+                    productDir = "stacklight" + templateContext.default_context['stacklight_version']
+                } catch (Throwable e) {
+                    productDir = "stacklight1"
                 }
-
-                if (product == "infra" || (templateContext.default_context["${product}_enabled"]
-                    && templateContext.default_context["${product}_enabled"].toBoolean())) {
-
-                    templateDir = "${templateEnv}/cluster_product/${productDir}"
-                    common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
-
-                    sh "rm -rf ${templateOutputDir} || true"
-                    sh "mkdir -p ${templateOutputDir}"
-                    sh "mkdir -p ${outputDestination}"
-
-                    python.setupCookiecutterVirtualenv(cutterEnv)
-                    python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
-                    sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
-                } else {
-                    common.warningMsg("Product " + product + " is disabled")
-                }
+            } else {
+                templateOutputDir = "${env.WORKSPACE}/output/${product}"
+                productDir = product
             }
 
-            stage('Generate new SaltMaster node') {
-                def nodeFile = "${modelEnv}/nodes/cfg01.${clusterDomain}.yml"
-                def nodeString = """classes:
+            if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+                && templateContext.default_context["${product}_enabled"].toBoolean())) {
+
+                templateDir = "${templateEnv}/cluster_product/${productDir}"
+                common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
+
+                sh "rm -rf ${templateOutputDir} || true"
+                sh "mkdir -p ${templateOutputDir}"
+                sh "mkdir -p ${outputDestination}"
+
+                python.setupCookiecutterVirtualenv(cutterEnv)
+                python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
+                sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+            } else {
+                common.warningMsg("Product " + product + " is disabled")
+            }
+        }
+
+        stage('Generate new SaltMaster node') {
+            def nodeFile = "${modelEnv}/nodes/cfg01.${clusterDomain}.yml"
+            def nodeString = """classes:
 - cluster.${clusterName}.infra.config
 parameters:
   _param:
@@ -105,73 +105,72 @@
       name: cfg01
       domain: ${clusterDomain}
 """
-                sh "mkdir -p ${modelEnv}/nodes/"
-                writeFile(file: nodeFile, text: nodeString)
+            sh "mkdir -p ${modelEnv}/nodes/"
+            writeFile(file: nodeFile, text: nodeString)
 
-                git.commitGitChanges(modelEnv, "Create model ${clusterName}", "${user}@localhost", "${user}")
-            }
-
-            stage("Test") {
-                if (SHARED_RECLASS_URL != "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
-                    sh("cp -r ${modelEnv} ${testEnv}")
-                    saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", "", testEnv)
-                }
-            }
-
-            stage("Generate config drive") {
-                // apt package genisoimage is required for this stage
-
-                // download create-config-drive
-                def config_drive_script_url = "https://raw.githubusercontent.com/pupapaik/virt-utils/master/create-config-drive"
-                def user_data_script_url = "https://raw.githubusercontent.com/mceloud/scripts/master/master_config.sh"
-
-                sh "wget -O create-config-drive ${config_drive_script_url} && chmod +x create-config-drive"
-                sh "wget -O user_data.sh ${user_data_script_url}"
-
-
-                // load data from model
-                def smc = [:]
-                smc['SALT_MASTER_MINION_ID'] = "cfg.${clusterDomain}"
-                smc['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
-                smc['DEPLOY_NETWORK_GW'] = templateContext['default_context']['deploy_network_gateway']
-                smc['DEPLOY_NETWORK_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
-                smc['DNS_SERVERS'] = templateContext['default_context']['dns_server01']
-
-                for (i in common.entries(smc)) {
-                    sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" user_data.sh"
-                }
-
-                // create config-drive
-                sh "./create-config-drive --user-data user_data.sh --hostname cfg --model ${modelEnv} cfg.${clusterDomain}-config.iso"
-
-                // save iso to artifacts
-                archiveArtifacts artifacts: "cfg.${clusterDomain}-config.iso"
-            }
-
-            stage ('Save changes reclass model') {
-
-                sh(returnStatus: true, script: "tar -zcf ${clusterName}.tar.gz -C ${modelEnv} .")
-                archiveArtifacts artifacts: "${clusterName}.tar.gz"
-
-
-                if (EMAIL_ADDRESS != null && EMAIL_ADDRESS != "") {
-                     emailext(to: EMAIL_ADDRESS,
-                              attachmentsPattern: "${clusterName}.tar.gz",
-                              body: "Mirantis Jenkins\n\nRequested reclass model ${clusterName} has been created and attached to this email.\nEnjoy!\n\nMirantis",
-                              subject: "Your Salt model ${clusterName}")
-                }
-            }
-
-        } catch (Throwable e) {
-             // If there was an error or exception thrown, the build failed
-             currentBuild.result = "FAILURE"
-             throw e
-        } finally {
-            stage ('Clean workspace directories') {
-                sh(returnStatus: true, script: "rm -rf ${templateEnv}")
-                sh(returnStatus: true, script: "rm -rf ${modelEnv}")
-            }
-             // common.sendNotification(currentBuild.result,"",["slack"])
+            git.commitGitChanges(modelEnv, "Create model ${clusterName}", "${user}@localhost", "${user}")
         }
+
+        stage("Test") {
+            if (SHARED_RECLASS_URL != "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
+                sh("cp -r ${modelEnv} ${testEnv}")
+                saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", "", testEnv)
+            }
+        }
+
+        stage("Generate config drive") {
+            // apt package genisoimage is required for this stage
+
+            // download create-config-drive
+            def config_drive_script_url = "https://raw.githubusercontent.com/pupapaik/virt-utils/master/create-config-drive"
+            def user_data_script_url = "https://raw.githubusercontent.com/mceloud/scripts/master/master_config.sh"
+
+            sh "wget -O create-config-drive ${config_drive_script_url} && chmod +x create-config-drive"
+            sh "wget -O user_data.sh ${user_data_script_url}"
+
+
+            // load data from model
+            def smc = [:]
+            smc['SALT_MASTER_MINION_ID'] = "cfg.${clusterDomain}"
+            smc['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
+            smc['DEPLOY_NETWORK_GW'] = templateContext['default_context']['deploy_network_gateway']
+            smc['DEPLOY_NETWORK_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
+            smc['DNS_SERVERS'] = templateContext['default_context']['dns_server01']
+
+            for (i in common.entries(smc)) {
+                sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" user_data.sh"
+            }
+
+            // create config-drive
+            sh "./create-config-drive --user-data user_data.sh --hostname cfg --model ${modelEnv} cfg.${clusterDomain}-config.iso"
+
+            // save iso to artifacts
+            archiveArtifacts artifacts: "cfg.${clusterDomain}-config.iso"
+        }
+
+        stage ('Save changes reclass model') {
+
+            sh(returnStatus: true, script: "tar -zcf ${clusterName}.tar.gz -C ${modelEnv} .")
+            archiveArtifacts artifacts: "${clusterName}.tar.gz"
+
+
+            if (EMAIL_ADDRESS != null && EMAIL_ADDRESS != "") {
+                 emailext(to: EMAIL_ADDRESS,
+                          attachmentsPattern: "${clusterName}.tar.gz",
+                          body: "Mirantis Jenkins\n\nRequested reclass model ${clusterName} has been created and attached to this email.\nEnjoy!\n\nMirantis",
+                          subject: "Your Salt model ${clusterName}")
+            }
+        }
+
+    } catch (Throwable e) {
+         // If there was an error or exception thrown, the build failed
+         currentBuild.result = "FAILURE"
+         throw e
+    } finally {
+        stage ('Clean workspace directories') {
+            sh(returnStatus: true, script: "rm -rf ${templateEnv}")
+            sh(returnStatus: true, script: "rm -rf ${modelEnv}")
+        }
+         // common.sendNotification(currentBuild.result,"",["slack"])
     }
 }
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index 9f4571c..c605421 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -81,311 +81,309 @@
 
 _MAX_PERMITTED_STACKS = 2
 
-timestamps {
-    node {
-        // try to get STACK_INSTALL or fallback to INSTALL if exists
-        try {
-          def temporary = STACK_INSTALL
-        } catch (MissingPropertyException e) {
-          try {
-            STACK_INSTALL = INSTALL
-            env['STACK_INSTALL'] = INSTALL
-          } catch (MissingPropertyException e2) {
-            common.errorMsg("Property STACK_INSTALL or INSTALL not found!")
-          }
-        }
-        try {
-            //
-            // Prepare machines
-            //
-            stage ('Create infrastructure') {
-
-                if (STACK_TYPE == 'heat') {
-                    // value defaults
-                    def openstackCloud
-                    def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
-                    def openstackEnv = "${env.WORKSPACE}/venv"
-
-                    if (STACK_REUSE.toBoolean() == true && STACK_NAME == '') {
-                        error("If you want to reuse existing stack you need to provide it's name")
-                    }
-
-                    if (STACK_REUSE.toBoolean() == false) {
-                        // Don't allow to set custom heat stack name
-                        wrap([$class: 'BuildUser']) {
-                            if (env.BUILD_USER_ID) {
-                                STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
-                            } else {
-                                STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
-                            }
-                            currentBuild.description = STACK_NAME
-                        }
-                    }
-
-                    // set description
-                    currentBuild.description = "${STACK_NAME}"
-
-                    // get templates
-                    git.checkoutGitRepository('template', STACK_TEMPLATE_URL, STACK_TEMPLATE_BRANCH, STACK_TEMPLATE_CREDENTIALS)
-
-                    // create openstack env
-                    openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
-                    openstackCloud = openstack.createOpenstackEnv(
-                        OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS,
-                        OPENSTACK_API_PROJECT, OPENSTACK_API_PROJECT_DOMAIN,
-                        OPENSTACK_API_PROJECT_ID, OPENSTACK_API_USER_DOMAIN,
-                        OPENSTACK_API_VERSION)
-                    openstack.getKeystoneToken(openstackCloud, openstackEnv)
-                    //
-                    // Verify possibility of create stack for given user and stack type
-                    //
-                    wrap([$class: 'BuildUser']) {
-                        if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !env.BUILD_USER_ID.equals("mceloud") && !STACK_REUSE.toBoolean()) {
-                            def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
-                            if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
-                                STACK_DELETE = "false"
-                                throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
-                            }
-                        }
-                    }
-                    // launch stack
-                    if (STACK_REUSE.toBoolean() == false) {
-                        stage('Launch new Heat stack') {
-                            // create stack
-                            envParams = [
-                                'instance_zone': HEAT_STACK_ZONE,
-                                'public_net': HEAT_STACK_PUBLIC_NET
-                            ]
-                            try {
-                                envParams.put('cfg_reclass_branch', STACK_RECLASS_BRANCH)
-                                envParams.put('cfg_reclass_address', STACK_RECLASS_ADDRESS)
-                            } catch (MissingPropertyException e) {
-                                common.infoMsg("Property STACK_RECLASS_BRANCH or STACK_RECLASS_ADDRESS not found! Using default values from template.")
-                            }
-                            openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
-                        }
-                    }
-
-                    // get SALT_MASTER_URL
-                    saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, STACK_NAME, 'salt_master_ip', openstackEnv)
-                    currentBuild.description = "${STACK_NAME}: ${saltMasterHost}"
-
-                    SALT_MASTER_URL = "http://${saltMasterHost}:6969"
-                }
-            }
-
-            //
-            // Connect to Salt master
-            //
-
-            def saltMaster
-            stage('Connect to Salt API') {
-                saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
-            }
-
-            // Set up override params
-            if (env.getEnvironment().containsKey('SALT_OVERRIDES')) {
-                stage('Set Salt overrides') {
-                    salt.setSaltOverrides(saltMaster,  SALT_OVERRIDES)
-                }
-            }
-
-            //
-            // Install
-            //
-
-            if (common.checkContains('STACK_INSTALL', 'core')) {
-                stage('Install core infrastructure') {
-                    orchestrate.installFoundationInfra(saltMaster)
-
-                    if (common.checkContains('STACK_INSTALL', 'kvm')) {
-                        orchestrate.installInfraKvm(saltMaster)
-                        orchestrate.installFoundationInfra(saltMaster)
-                    }
-
-                    orchestrate.validateFoundationInfra(saltMaster)
-                }
-            }
-
-            // install k8s
-            if (common.checkContains('STACK_INSTALL', 'k8s')) {
-
-                // install infra libs for k8s
-                stage('Install Kubernetes infra') {
-                    orchestrate.installKubernetesInfra(saltMaster)
-                }
-
-                // If k8s install with contrail network manager then contrail need to be install first
-                if (common.checkContains('STACK_INSTALL', 'contrail')) {
-                    stage('Install Contrail for Kubernetes') {
-                        orchestrate.installContrailNetwork(saltMaster)
-                        orchestrate.installContrailCompute(saltMaster)
-                        orchestrate.installKubernetesContrailCompute(saltMaster)
-                    }
-                }
-
-                stage('Install Kubernetes control') {
-                    orchestrate.installKubernetesControl(saltMaster)
-                }
-            }
-
-            // install openstack
-            if (common.checkContains('STACK_INSTALL', 'openstack')) {
-                // install Infra and control, tests, ...
-
-                stage('Install OpenStack infra') {
-                    orchestrate.installOpenstackInfra(saltMaster)
-                }
-
-                stage('Install OpenStack control') {
-                    orchestrate.installOpenstackControl(saltMaster)
-                }
-
-                stage('Install OpenStack network') {
-
-                    if (common.checkContains('STACK_INSTALL', 'contrail')) {
-                        orchestrate.installContrailNetwork(saltMaster)
-                    } else if (common.checkContains('STACK_INSTALL', 'ovs')) {
-                        orchestrate.installOpenstackNetwork(saltMaster)
-                    }
-
-                    salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
-                    salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
-                }
-
-                if (salt.testTarget(saltMaster, 'I@ironic:conductor')){
-                    stage('Install OpenStack Ironic conductor') {
-                        orchestrate.installIronicConductor(saltMaster)
-                    }
-                }
-
-
-                stage('Install OpenStack compute') {
-                    orchestrate.installOpenstackCompute(saltMaster)
-
-                    if (common.checkContains('STACK_INSTALL', 'contrail')) {
-                        orchestrate.installContrailCompute(saltMaster)
-                    }
-                }
-
-            }
-
-
-            if (common.checkContains('STACK_INSTALL', 'sl-legacy')) {
-                stage('Install StackLight v1') {
-                    orchestrate.installStacklightv1Control(saltMaster)
-                    orchestrate.installStacklightv1Client(saltMaster)
-                }
-            }
-
-            if (common.checkContains('STACK_INSTALL', 'stacklight')) {
-                stage('Install StackLight') {
-                    orchestrate.installDockerSwarm(saltMaster)
-                    orchestrate.installStacklight(saltMaster)
-                }
-            }
-
-            //
-            // Test
-            //
-            def artifacts_dir = '_artifacts/'
-
-            if (common.checkContains('STACK_TEST', 'k8s')) {
-                stage('Run k8s bootstrap tests') {
-                    def image = 'tomkukral/k8s-scripts'
-                    def output_file = image.replaceAll('/', '-') + '.output'
-
-                    // run image
-                    test.runConformanceTests(saltMaster, TEST_K8S_API_SERVER, image)
-
-                    // collect output
-                    sh "mkdir -p ${artifacts_dir}"
-                    file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
-                    writeFile file: "${artifacts_dir}${output_file}", text: file_content
-                    sh "cat ${artifacts_dir}${output_file}"
-
-                    // collect artifacts
-                    archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
-                }
-
-                stage('Run k8s conformance e2e tests') {
-                    //test.runConformanceTests(saltMaster, TEST_K8S_API_SERVER, TEST_K8S_CONFORMANCE_IMAGE)
-
-                    def image = TEST_K8S_CONFORMANCE_IMAGE
-                    def output_file = image.replaceAll('/', '-') + '.output'
-
-                    // run image
-                    test.runConformanceTests(saltMaster, TEST_K8S_API_SERVER, image)
-
-                    // collect output
-                    sh "mkdir -p ${artifacts_dir}"
-                    file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
-                    writeFile file: "${artifacts_dir}${output_file}", text: file_content
-                    sh "cat ${artifacts_dir}${output_file}"
-
-                    // collect artifacts
-                    archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
-                }
-            }
-
-            if (common.checkContains('STACK_TEST', 'openstack')) {
-                if (common.checkContains('TEST_DOCKER_INSTALL', 'true')) {
-                    test.install_docker(saltMaster, TEST_TEMPEST_TARGET)
-                }
-                stage('Run OpenStack tests') {
-                    test.runTempestTests(saltMaster, TEST_TEMPEST_IMAGE, TEST_TEMPEST_TARGET, TEST_TEMPEST_PATTERN)
-                }
-
-                stage('Copy Tempest results to config node') {
-                    test.copyTempestResults(saltMaster, TEST_TEMPEST_TARGET)
-                }
-            }
-
-            if (common.checkContains('STACK_INSTALL', 'finalize')) {
-                stage('Finalize') {
-                    salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
-                }
-            }
-        } catch (Throwable e) {
-            currentBuild.result = 'FAILURE'
-            throw e
-        } finally {
-
-
-            //
-            // Clean
-            //
+node {
+    // try to get STACK_INSTALL or fallback to INSTALL if exists
+    try {
+      def temporary = STACK_INSTALL
+    } catch (MissingPropertyException e) {
+      try {
+        STACK_INSTALL = INSTALL
+        env['STACK_INSTALL'] = INSTALL
+      } catch (MissingPropertyException e2) {
+        common.errorMsg("Property STACK_INSTALL or INSTALL not found!")
+      }
+    }
+    try {
+        //
+        // Prepare machines
+        //
+        stage ('Create infrastructure') {
 
             if (STACK_TYPE == 'heat') {
-                // send notification
-                common.sendNotification(currentBuild.result, STACK_NAME, ["slack"])
+                // value defaults
+                def openstackCloud
+                def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
+                def openstackEnv = "${env.WORKSPACE}/venv"
 
-                if (STACK_DELETE.toBoolean() == true) {
-                    common.errorMsg('Heat job cleanup triggered')
-                    stage('Trigger cleanup job') {
-                        build(job: STACK_CLEANUP_JOB, parameters: [
-                            [$class: 'StringParameterValue', name: 'STACK_NAME', value: STACK_NAME],
-                            [$class: 'StringParameterValue', name: 'STACK_TYPE', value: STACK_TYPE],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_URL', value: OPENSTACK_API_URL],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_CREDENTIALS', value: OPENSTACK_API_CREDENTIALS],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT_DOMAIN', value: OPENSTACK_API_PROJECT_DOMAIN],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT_ID', value: OPENSTACK_API_PROJECT_ID],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_USER_DOMAIN', value: OPENSTACK_API_USER_DOMAIN],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_CLIENT', value: OPENSTACK_API_CLIENT],
-                            [$class: 'StringParameterValue', name: 'OPENSTACK_API_VERSION', value: OPENSTACK_API_VERSION]
-                        ])
+                if (STACK_REUSE.toBoolean() == true && STACK_NAME == '') {
+                    error("If you want to reuse existing stack you need to provide it's name")
+                }
+
+                if (STACK_REUSE.toBoolean() == false) {
+                    // Don't allow to set custom heat stack name
+                    wrap([$class: 'BuildUser']) {
+                        if (env.BUILD_USER_ID) {
+                            STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
+                        } else {
+                            STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
+                        }
+                        currentBuild.description = STACK_NAME
                     }
-                } else {
-                    if (currentBuild.result == 'FAILURE') {
-                        common.errorMsg("Deploy job FAILED and was not deleted. Please fix the problem and delete stack on you own.")
+                }
 
-                        if (SALT_MASTER_URL) {
-                            common.errorMsg("Salt master URL: ${SALT_MASTER_URL}")
+                // set description
+                currentBuild.description = "${STACK_NAME}"
+
+                // get templates
+                git.checkoutGitRepository('template', STACK_TEMPLATE_URL, STACK_TEMPLATE_BRANCH, STACK_TEMPLATE_CREDENTIALS)
+
+                // create openstack env
+                openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
+                openstackCloud = openstack.createOpenstackEnv(
+                    OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS,
+                    OPENSTACK_API_PROJECT, OPENSTACK_API_PROJECT_DOMAIN,
+                    OPENSTACK_API_PROJECT_ID, OPENSTACK_API_USER_DOMAIN,
+                    OPENSTACK_API_VERSION)
+                openstack.getKeystoneToken(openstackCloud, openstackEnv)
+                //
+                // Verify possibility of create stack for given user and stack type
+                //
+                wrap([$class: 'BuildUser']) {
+                    if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !env.BUILD_USER_ID.equals("mceloud") && !STACK_REUSE.toBoolean()) {
+                        def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
+                        if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
+                            STACK_DELETE = "false"
+                            throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
                         }
                     }
-
                 }
+                // launch stack
+                if (STACK_REUSE.toBoolean() == false) {
+                    stage('Launch new Heat stack') {
+                        // create stack
+                        envParams = [
+                            'instance_zone': HEAT_STACK_ZONE,
+                            'public_net': HEAT_STACK_PUBLIC_NET
+                        ]
+                        try {
+                            envParams.put('cfg_reclass_branch', STACK_RECLASS_BRANCH)
+                            envParams.put('cfg_reclass_address', STACK_RECLASS_ADDRESS)
+                        } catch (MissingPropertyException e) {
+                            common.infoMsg("Property STACK_RECLASS_BRANCH or STACK_RECLASS_ADDRESS not found! Using default values from template.")
+                        }
+                        openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
+                    }
+                }
+
+                // get SALT_MASTER_URL
+                saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, STACK_NAME, 'salt_master_ip', openstackEnv)
+                currentBuild.description = "${STACK_NAME}: ${saltMasterHost}"
+
+                SALT_MASTER_URL = "http://${saltMasterHost}:6969"
+            }
+        }
+
+        //
+        // Connect to Salt master
+        //
+
+        def saltMaster
+        stage('Connect to Salt API') {
+            saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+        }
+
+        // Set up override params
+        if (env.getEnvironment().containsKey('SALT_OVERRIDES')) {
+            stage('Set Salt overrides') {
+                salt.setSaltOverrides(saltMaster,  SALT_OVERRIDES)
+            }
+        }
+
+        //
+        // Install
+        //
+
+        if (common.checkContains('STACK_INSTALL', 'core')) {
+            stage('Install core infrastructure') {
+                orchestrate.installFoundationInfra(saltMaster)
+
+                if (common.checkContains('STACK_INSTALL', 'kvm')) {
+                    orchestrate.installInfraKvm(saltMaster)
+                    orchestrate.installFoundationInfra(saltMaster)
+                }
+
+                orchestrate.validateFoundationInfra(saltMaster)
+            }
+        }
+
+        // install k8s
+        if (common.checkContains('STACK_INSTALL', 'k8s')) {
+
+            // install infra libs for k8s
+            stage('Install Kubernetes infra') {
+                orchestrate.installKubernetesInfra(saltMaster)
+            }
+
+            // If k8s install with contrail network manager then contrail need to be install first
+            if (common.checkContains('STACK_INSTALL', 'contrail')) {
+                stage('Install Contrail for Kubernetes') {
+                    orchestrate.installContrailNetwork(saltMaster)
+                    orchestrate.installContrailCompute(saltMaster)
+                    orchestrate.installKubernetesContrailCompute(saltMaster)
+                }
+            }
+
+            stage('Install Kubernetes control') {
+                orchestrate.installKubernetesControl(saltMaster)
+            }
+        }
+
+        // install openstack
+        if (common.checkContains('STACK_INSTALL', 'openstack')) {
+            // install Infra and control, tests, ...
+
+            stage('Install OpenStack infra') {
+                orchestrate.installOpenstackInfra(saltMaster)
+            }
+
+            stage('Install OpenStack control') {
+                orchestrate.installOpenstackControl(saltMaster)
+            }
+
+            stage('Install OpenStack network') {
+
+                if (common.checkContains('STACK_INSTALL', 'contrail')) {
+                    orchestrate.installContrailNetwork(saltMaster)
+                } else if (common.checkContains('STACK_INSTALL', 'ovs')) {
+                    orchestrate.installOpenstackNetwork(saltMaster)
+                }
+
+                salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
+                salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
+            }
+
+            if (salt.testTarget(saltMaster, 'I@ironic:conductor')){
+                stage('Install OpenStack Ironic conductor') {
+                    orchestrate.installIronicConductor(saltMaster)
+                }
+            }
+
+
+            stage('Install OpenStack compute') {
+                orchestrate.installOpenstackCompute(saltMaster)
+
+                if (common.checkContains('STACK_INSTALL', 'contrail')) {
+                    orchestrate.installContrailCompute(saltMaster)
+                }
+            }
+
+        }
+
+
+        if (common.checkContains('STACK_INSTALL', 'sl-legacy')) {
+            stage('Install StackLight v1') {
+                orchestrate.installStacklightv1Control(saltMaster)
+                orchestrate.installStacklightv1Client(saltMaster)
+            }
+        }
+
+        if (common.checkContains('STACK_INSTALL', 'stacklight')) {
+            stage('Install StackLight') {
+                orchestrate.installDockerSwarm(saltMaster)
+                orchestrate.installStacklight(saltMaster)
+            }
+        }
+
+        //
+        // Test
+        //
+        def artifacts_dir = '_artifacts/'
+
+        if (common.checkContains('STACK_TEST', 'k8s')) {
+            stage('Run k8s bootstrap tests') {
+                def image = 'tomkukral/k8s-scripts'
+                def output_file = image.replaceAll('/', '-') + '.output'
+
+                // run image
+                test.runConformanceTests(saltMaster, TEST_K8S_API_SERVER, image)
+
+                // collect output
+                sh "mkdir -p ${artifacts_dir}"
+                file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
+                writeFile file: "${artifacts_dir}${output_file}", text: file_content
+                sh "cat ${artifacts_dir}${output_file}"
+
+                // collect artifacts
+                archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
+            }
+
+            stage('Run k8s conformance e2e tests') {
+                //test.runConformanceTests(saltMaster, TEST_K8S_API_SERVER, TEST_K8S_CONFORMANCE_IMAGE)
+
+                def image = TEST_K8S_CONFORMANCE_IMAGE
+                def output_file = image.replaceAll('/', '-') + '.output'
+
+                // run image
+                test.runConformanceTests(saltMaster, TEST_K8S_API_SERVER, image)
+
+                // collect output
+                sh "mkdir -p ${artifacts_dir}"
+                file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
+                writeFile file: "${artifacts_dir}${output_file}", text: file_content
+                sh "cat ${artifacts_dir}${output_file}"
+
+                // collect artifacts
+                archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
+            }
+        }
+
+        if (common.checkContains('STACK_TEST', 'openstack')) {
+            if (common.checkContains('TEST_DOCKER_INSTALL', 'true')) {
+                test.install_docker(saltMaster, TEST_TEMPEST_TARGET)
+            }
+            stage('Run OpenStack tests') {
+                test.runTempestTests(saltMaster, TEST_TEMPEST_IMAGE, TEST_TEMPEST_TARGET, TEST_TEMPEST_PATTERN)
+            }
+
+            stage('Copy Tempest results to config node') {
+                test.copyTempestResults(saltMaster, TEST_TEMPEST_TARGET)
+            }
+        }
+
+        if (common.checkContains('STACK_INSTALL', 'finalize')) {
+            stage('Finalize') {
+                salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
+            }
+        }
+    } catch (Throwable e) {
+        currentBuild.result = 'FAILURE'
+        throw e
+    } finally {
+
+
+        //
+        // Clean
+        //
+
+        if (STACK_TYPE == 'heat') {
+            // send notification
+            common.sendNotification(currentBuild.result, STACK_NAME, ["slack"])
+
+            if (STACK_DELETE.toBoolean() == true) {
+                common.errorMsg('Heat job cleanup triggered')
+                stage('Trigger cleanup job') {
+                    build(job: STACK_CLEANUP_JOB, parameters: [
+                        [$class: 'StringParameterValue', name: 'STACK_NAME', value: STACK_NAME],
+                        [$class: 'StringParameterValue', name: 'STACK_TYPE', value: STACK_TYPE],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_URL', value: OPENSTACK_API_URL],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_CREDENTIALS', value: OPENSTACK_API_CREDENTIALS],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT_DOMAIN', value: OPENSTACK_API_PROJECT_DOMAIN],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT_ID', value: OPENSTACK_API_PROJECT_ID],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_USER_DOMAIN', value: OPENSTACK_API_USER_DOMAIN],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_CLIENT', value: OPENSTACK_API_CLIENT],
+                        [$class: 'StringParameterValue', name: 'OPENSTACK_API_VERSION', value: OPENSTACK_API_VERSION]
+                    ])
+                }
+            } else {
+                if (currentBuild.result == 'FAILURE') {
+                    common.errorMsg("Deploy job FAILED and was not deleted. Please fix the problem and delete stack on you own.")
+
+                    if (SALT_MASTER_URL) {
+                        common.errorMsg("Salt master URL: ${SALT_MASTER_URL}")
+                    }
+                }
+
             }
         }
     }
diff --git a/opencontrail-upgrade.groovy b/opencontrail-upgrade.groovy
index a01affe..6bcb788 100644
--- a/opencontrail-upgrade.groovy
+++ b/opencontrail-upgrade.groovy
@@ -40,9 +40,9 @@
     salt.printSaltCommandResult(out)
     // wait until $check is in correct state
     if ( check == "nodetool status" ) {
-        salt.commandStatus(saltMaster, target, check, 'Status=Up')  
+        salt.commandStatus(saltMaster, target, check, 'Status=Up')
     } else if ( check == "contrail-status" ) {
-        salt.commandStatus(saltMaster, target, "${check} | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)  
+        salt.commandStatus(saltMaster, target, "${check} | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
     }
 
     //out = salt.runSaltCommand(saltMaster, 'local', ['expression': target, 'type': 'compound'], command, null, check, null)
@@ -50,443 +50,441 @@
     //input message: "Please check the output of \'${check}\' and continue if it is correct."
 }
 
-timestamps {
-    node() {
+node() {
 
-        stage('Connect to Salt API') {
-            saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
-        }
+    stage('Connect to Salt API') {
+        saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    }
 
-        if (STAGE_CONTROLLERS_UPGRADE.toBoolean() == true && !errorOccured) {
+    if (STAGE_CONTROLLERS_UPGRADE.toBoolean() == true && !errorOccured) {
 
-            stage('Opencontrail controllers upgrade') {
+        stage('Opencontrail controllers upgrade') {
 
-                oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+            oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
 
-                oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                try {
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
-                    salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
-                } catch (Exception er) {
-                    errorOccured = true
-                    common.errorMsg("Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                    return
-                }
-
-                try {
-                    salt.cmdRun(saltMaster, 'I@opencontrail:control', "su root -c '/usr/local/bin/zookeeper-backup-runner.sh'")
-                } catch (Exception er) {
-                    common.errorMsg('Zookeeper failed to backup. Please fix it before continuing.')
-                    return
-                }
-
-                try {
-                    salt.cmdRun(saltMaster, 'I@cassandra:backup:client', "su root -c '/usr/local/bin/cassandra-backup-runner-call.sh'")
-                } catch (Exception er) {
-                    common.errorMsg('Cassandra failed to backup. Please fix it before continuing.')
-                    return
-                }
-
-                args = 'apt install contrail-database -y;'
-                check = 'nodetool status'
-
-                // ntw01
-                runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
-                // ntw02
-                runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
-                // ntw03
-                runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
-
-                args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
-                check = 'contrail-status'
-
-                // ntw01
-                runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
-                // ntw02
-                runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
-                // ntw03
-                runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
-
-                try {
-                    salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
-                } catch (Exception er) {
-                    common.errorMsg('Opencontrail state was executed on I@opencontrail:control and failed please fix it manually.')
-                }
-
-                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control', 'type': 'compound'], command, null, check, null)
-                salt.printSaltCommandResult(out)
-
-                common.warningMsg('Please check \'show bgp summary\' on your bgp router if all bgp peers are in healthy state.')
-            }
-        }
-
-        if (STAGE_ANALYTICS_UPGRADE.toBoolean() == true && !errorOccured) {
-
-            stage('Ask for manual confirmation') {
-                input message: "Do you want to continue with the Opencontrail analytic nodes upgrade?"
-            }
-
-            stage('Opencontrail analytics upgrade') {
-
-                oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
-
-                oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                try {
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
-                    salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
-                } catch (Exception er) {
-                    errorOccured = true
-                    common.errorMsg("Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                    return
-                }
-
-                args = 'apt install contrail-database -y;'
-                check = 'nodetool status'
-
-                // nal01
-                runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
-                // nal02
-                runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
-                // nal03
-                runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
-
-                args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
-                check = 'contrail-status'
-
-                // nal01
-                runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
-                // nal02
-                runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
-                // nal03
-                runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
-
-                try {
-                    salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
-                } catch (Exception er) {
-                    common.errorMsg('Opencontrail state was executed on I@opencontrail:collector and failed please fix it manually.')
-                }
-
-                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
-                salt.printSaltCommandResult(out)
-            }
-        }
-
-        if (STAGE_COMPUTES_UPGRADE.toBoolean() == true && !errorOccured) {
+            oc_component_repo = oc_component_repo['return'][0].values()[0]
 
             try {
-
-                stage('List targeted compute servers') {
-                    minions = salt.getMinions(saltMaster, COMPUTE_TARGET_SERVERS)
-
-                    if (minions.isEmpty()) {
-                        throw new Exception("No minion was targeted")
-                    }
-
-                    targetLiveSubset = minions.subList(0, Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)).join(' or ')
-                    targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')
-
-                    targetLiveAll = minions.join(' or ')
-                    common.infoMsg("Found nodes: ${targetLiveAll}")
-                    common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
-                }
-
-                stage('Confirm upgrade on sample nodes') {
-                    input message: "Do you want to continue with the Opencontrail compute upgrade on the following sample nodes? ${targetLiveSubset}"
-                }
-
-                stage("Opencontrail compute upgrade on sample nodes") {
-
-                    oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
-                    oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                    try {
-                        salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                        salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
-                        salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
-                    } catch (Exception er) {
-                        errorOccured = true
-                        common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                        return
-                    }
-
-                    args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS}  -y;"
-                    check = 'contrail-status'
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
-                    salt.printSaltCommandResult(out)
-
-                    try {
-                        salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
-                    } catch (Exception er) {
-                        common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
-                    }
-
-                    salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
-
-                    //sleep(10)
-                    salt.commandStatus(saltMaster, targetLiveSubset, "${check} | grep -v == | grep -v active", null, false)
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
-                    salt.printSaltCommandResult(out)
-                }
-
-                stage('Confirm upgrade on all targeted nodes') {
-                    input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
-                }
-                stage("Opencontrail compute upgrade on all targeted nodes") {
-
-                    oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
-                    oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                    try {
-                        salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                        salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
-                        salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
-                    } catch (Exception er) {
-                        common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                        return
-                    }
-
-                    args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS}  -y;"
-                    check = 'contrail-status'
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
-                    salt.printSaltCommandResult(out)
-
-                    try {
-                        salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
-                    } catch (Exception er) {
-                        common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
-                    }
-
-                    salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
-                    //sleep(10)
-                    salt.commandStatus(saltMaster, targetLiveAll, "${check} | grep -v == | grep -v active", null, false)
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
-                    salt.printSaltCommandResult(out)
-                }
-
-            } catch (Throwable e) {
-                // If there was an error or exception thrown, the build failed
-                currentBuild.result = "FAILURE"
-                throw e
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
+                salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
+            } catch (Exception er) {
+                errorOccured = true
+                common.errorMsg("Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                return
             }
-        }
-
-
-        if (STAGE_CONTROLLERS_ROLLBACK.toBoolean() == true && !errorOccured) {
-
-            stage('Ask for manual confirmation') {
-                input message: "Do you want to continue with the Opencontrail control nodes rollback?"
-            }
-
-           stage('Opencontrail controllers rollback') {
-
-                oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
-                oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                try {
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
-                    salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
-                } catch (Exception er) {
-                    errorOccured = true
-                    common.errorMsg("Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                    return
-                }
-
-                args = 'apt install contrail-database -y --force-yes;'
-                check = 'nodetool status'
-
-                // ntw01
-                runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
-                // ntw02
-                runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
-                // ntw03
-                runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
-
-                args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
-                check = 'contrail-status'
-
-                // ntw01
-                runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
-                // ntw02
-                runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
-                // ntw03
-                runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
-
-                try {
-                    salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
-                } catch (Exception er) {
-                    common.errorMsg('Opencontrail state was executed on I@opencontrail:control and failed please fix it manually.')
-                }
-
-                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control', 'type': 'compound'], command, null, check, null)
-                salt.printSaltCommandResult(out)
-
-                common.warningMsg('Please check \'show bgp summary\' on your bgp router if all bgp peers are in healthy state.')
-            }
-        }
-
-        if (STAGE_ANALYTICS_ROLLBACK.toBoolean() == true && !errorOccured) {
-
-            stage('Ask for manual confirmation') {
-                input message: "Do you want to continue with the Opencontrail analytic nodes rollback?"
-            }
-
-            stage('Opencontrail analytics rollback') {
-
-                oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
-                oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                try {
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                    salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
-                    salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
-                } catch (Exception er) {
-                    errorOccured = true
-                    common.errorMsg("Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                    return
-                }
-
-                args = 'apt install contrail-database -y --force-yes;'
-                check = 'nodetool status'
-
-                // nal01
-                runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
-                // nal02
-                runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
-                // nal03
-                runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
-
-                args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
-                check = 'contrail-status'
-
-                // nal01
-                runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
-                // nal02
-                runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
-                // nal03
-                runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
-
-                try {
-                    salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
-                } catch (Exception er) {
-                    common.errorMsg('Opencontrail state was executed on I@opencontrail:collector and failed please fix it manually.')
-                }
-
-                out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
-                salt.printSaltCommandResult(out)
-            }
-        }
-
-        if (STAGE_COMPUTES_ROLLBACK.toBoolean() == true && !errorOccured) {
 
             try {
-
-                stage('List targeted compute servers') {
-                    minions = salt.getMinions(saltMaster, COMPUTE_TARGET_SERVERS)
-
-                    if (minions.isEmpty()) {
-                        throw new Exception("No minion was targeted")
-                    }
-
-                    targetLiveSubset = minions.subList(0, Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)).join(' or ')
-                    targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')
-
-                    targetLiveAll = minions.join(' or ')
-                    common.infoMsg("Found nodes: ${targetLiveAll}")
-                    common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
-                }
-
-                stage('Confirm rollback on sample nodes') {
-                    input message: "Do you want to continue with the Opencontrail compute rollback on the following sample nodes? ${targetLiveSubset}"
-                }
-
-                stage("Opencontrail compute rollback on sample nodes") {
-
-                    oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
-                    oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                    try {
-                        salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                        salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
-                        salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
-                    } catch (Exception er) {
-                        errorOccured = true
-                        common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                        return
-                    }
-
-                    args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS}  -y;"
-                    check = 'contrail-status'
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
-                    salt.printSaltCommandResult(out)
-
-                    try {
-                        salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
-                    } catch (Exception er) {
-                        common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
-                    }
-
-                    salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
-                    //sleep(10)
-                    salt.commandStatus(saltMaster, targetLiveSubset, "${check} | grep -v == | grep -v active", null, false)
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
-                    salt.printSaltCommandResult(out)
-                }
-
-                stage('Confirm rollback on all targeted nodes') {
-                    input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
-                }
-
-                stage("Opencontrail compute upgrade on all targeted nodes") {
-
-                    oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
-                    oc_component_repo = oc_component_repo['return'][0].values()[0]
-
-                    try {
-                        salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
-                        salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
-                        salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
-                    } catch (Exception er) {
-                        common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
-                        return
-                    }
-
-                    args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
-                    check = 'contrail-status'
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
-                    salt.printSaltCommandResult(out)
-
-                    try {
-                        salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
-                    } catch (Exception er) {
-                        common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
-                    }
-
-                    salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
-
-                    //sleep(10)
-                    salt.commandStatus(saltMaster, targetLiveAll, "${check} | grep -v == | grep -v active", null, false)
-
-                    out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
-                    salt.printSaltCommandResult(out)
-                }
-
-            } catch (Throwable e) {
-                // If there was an error or exception thrown, the build failed
-                currentBuild.result = "FAILURE"
-                throw e
+                salt.cmdRun(saltMaster, 'I@opencontrail:control', "su root -c '/usr/local/bin/zookeeper-backup-runner.sh'")
+            } catch (Exception er) {
+                common.errorMsg('Zookeeper failed to backup. Please fix it before continuing.')
+                return
             }
+
+            try {
+                salt.cmdRun(saltMaster, 'I@cassandra:backup:client', "su root -c '/usr/local/bin/cassandra-backup-runner-call.sh'")
+            } catch (Exception er) {
+                common.errorMsg('Cassandra failed to backup. Please fix it before continuing.')
+                return
+            }
+
+            args = 'apt install contrail-database -y;'
+            check = 'nodetool status'
+
+            // ntw01
+            runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
+            // ntw02
+            runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
+            // ntw03
+            runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
+
+            args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
+            check = 'contrail-status'
+
+            // ntw01
+            runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
+            // ntw02
+            runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
+            // ntw03
+            runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
+
+            try {
+                salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
+            } catch (Exception er) {
+                common.errorMsg('Opencontrail state was executed on I@opencontrail:control and failed please fix it manually.')
+            }
+
+            out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control', 'type': 'compound'], command, null, check, null)
+            salt.printSaltCommandResult(out)
+
+            common.warningMsg('Please check \'show bgp summary\' on your bgp router if all bgp peers are in healthy state.')
+        }
+    }
+
+    if (STAGE_ANALYTICS_UPGRADE.toBoolean() == true && !errorOccured) {
+
+        stage('Ask for manual confirmation') {
+            input message: "Do you want to continue with the Opencontrail analytic nodes upgrade?"
+        }
+
+        stage('Opencontrail analytics upgrade') {
+
+            oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+
+            oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+            try {
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
+                salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
+            } catch (Exception er) {
+                errorOccured = true
+                common.errorMsg("Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                return
+            }
+
+            args = 'apt install contrail-database -y;'
+            check = 'nodetool status'
+
+            // nal01
+            runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
+            // nal02
+            runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
+            // nal03
+            runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
+
+            args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
+            check = 'contrail-status'
+
+            // nal01
+            runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
+            // nal02
+            runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
+            // nal03
+            runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
+
+            try {
+                salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
+            } catch (Exception er) {
+                common.errorMsg('Opencontrail state was executed on I@opencontrail:collector and failed please fix it manually.')
+            }
+
+            out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
+            salt.printSaltCommandResult(out)
+        }
+    }
+
+    if (STAGE_COMPUTES_UPGRADE.toBoolean() == true && !errorOccured) {
+
+        try {
+
+            stage('List targeted compute servers') {
+                minions = salt.getMinions(saltMaster, COMPUTE_TARGET_SERVERS)
+
+                if (minions.isEmpty()) {
+                    throw new Exception("No minion was targeted")
+                }
+
+                targetLiveSubset = minions.subList(0, Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)).join(' or ')
+                targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')
+
+                targetLiveAll = minions.join(' or ')
+                common.infoMsg("Found nodes: ${targetLiveAll}")
+                common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
+            }
+
+            stage('Confirm upgrade on sample nodes') {
+                input message: "Do you want to continue with the Opencontrail compute upgrade on the following sample nodes? ${targetLiveSubset}"
+            }
+
+            stage("Opencontrail compute upgrade on sample nodes") {
+
+                oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+                oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+                try {
+                    salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                    salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
+                    salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
+                } catch (Exception er) {
+                    errorOccured = true
+                    common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                    return
+                }
+
+                args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS}  -y;"
+                check = 'contrail-status'
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
+                salt.printSaltCommandResult(out)
+
+                try {
+                    salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
+                } catch (Exception er) {
+                    common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
+                }
+
+                salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+
+                //sleep(10)
+                salt.commandStatus(saltMaster, targetLiveSubset, "${check} | grep -v == | grep -v active", null, false)
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
+                salt.printSaltCommandResult(out)
+            }
+
+            stage('Confirm upgrade on all targeted nodes') {
+                input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
+            }
+            stage("Opencontrail compute upgrade on all targeted nodes") {
+
+                oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+                oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+                try {
+                    salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                    salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
+                    salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
+                } catch (Exception er) {
+                    common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                    return
+                }
+
+                args = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS}  -y;"
+                check = 'contrail-status'
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
+                salt.printSaltCommandResult(out)
+
+                try {
+                    salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
+                } catch (Exception er) {
+                    common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
+                }
+
+                salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+                //sleep(10)
+                salt.commandStatus(saltMaster, targetLiveAll, "${check} | grep -v == | grep -v active", null, false)
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
+                salt.printSaltCommandResult(out)
+            }
+
+        } catch (Throwable e) {
+            // If there was an error or exception thrown, the build failed
+            currentBuild.result = "FAILURE"
+            throw e
+        }
+    }
+
+
+    if (STAGE_CONTROLLERS_ROLLBACK.toBoolean() == true && !errorOccured) {
+
+        stage('Ask for manual confirmation') {
+            input message: "Do you want to continue with the Opencontrail control nodes rollback?"
+        }
+
+       stage('Opencontrail controllers rollback') {
+
+            oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+            oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+            try {
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'saltutil.refresh_pillar', [], null, true)
+                salt.enforceState(saltMaster, 'I@opencontrail:control', 'linux.system.repo')
+            } catch (Exception er) {
+                errorOccured = true
+                common.errorMsg("Opencontrail component on I@opencontrail:control probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                return
+            }
+
+            args = 'apt install contrail-database -y --force-yes;'
+            check = 'nodetool status'
+
+            // ntw01
+            runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
+            // ntw02
+            runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
+            // ntw03
+            runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
+
+            args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
+            check = 'contrail-status'
+
+            // ntw01
+            runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, saltMaster, common)
+            // ntw02
+            runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, saltMaster, common)
+            // ntw03
+            runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, saltMaster, common)
+
+            try {
+                salt.enforceState(saltMaster, 'I@opencontrail:control', 'opencontrail')
+            } catch (Exception er) {
+                common.errorMsg('Opencontrail state was executed on I@opencontrail:control and failed please fix it manually.')
+            }
+
+            out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:control', 'type': 'compound'], command, null, check, null)
+            salt.printSaltCommandResult(out)
+
+            common.warningMsg('Please check \'show bgp summary\' on your bgp router if all bgp peers are in healthy state.')
+        }
+    }
+
+    if (STAGE_ANALYTICS_ROLLBACK.toBoolean() == true && !errorOccured) {
+
+        stage('Ask for manual confirmation') {
+            input message: "Do you want to continue with the Opencontrail analytic nodes rollback?"
+        }
+
+        stage('Opencontrail analytics rollback') {
+
+            oc_component_repo = salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+            oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+            try {
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'saltutil.refresh_pillar', [], null, true)
+                salt.enforceState(saltMaster, 'I@opencontrail:collector', 'linux.system.repo')
+            } catch (Exception er) {
+                errorOccured = true
+                common.errorMsg("Opencontrail component on I@opencontrail:collector probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                return
+            }
+
+            args = 'apt install contrail-database -y --force-yes;'
+            check = 'nodetool status'
+
+            // nal01
+            runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
+            // nal02
+            runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
+            // nal03
+            runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
+
+            args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
+            check = 'contrail-status'
+
+            // nal01
+            runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
+            // nal02
+            runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
+            // nal03
+            runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
+
+            try {
+                salt.enforceState(saltMaster, 'I@opencontrail:collector', 'opencontrail')
+            } catch (Exception er) {
+                common.errorMsg('Opencontrail state was executed on I@opencontrail:collector and failed please fix it manually.')
+            }
+
+            out = salt.runSaltCommand(saltMaster, 'local', ['expression': 'I@opencontrail:collector', 'type': 'compound'], command, null, check, null)
+            salt.printSaltCommandResult(out)
+        }
+    }
+
+    if (STAGE_COMPUTES_ROLLBACK.toBoolean() == true && !errorOccured) {
+
+        try {
+
+            stage('List targeted compute servers') {
+                minions = salt.getMinions(saltMaster, COMPUTE_TARGET_SERVERS)
+
+                if (minions.isEmpty()) {
+                    throw new Exception("No minion was targeted")
+                }
+
+                targetLiveSubset = minions.subList(0, Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)).join(' or ')
+                targetLiveSubsetProbe = minions.subList(0, probe).join(' or ')
+
+                targetLiveAll = minions.join(' or ')
+                common.infoMsg("Found nodes: ${targetLiveAll}")
+                common.infoMsg("Selected sample nodes: ${targetLiveSubset}")
+            }
+
+            stage('Confirm rollback on sample nodes') {
+                input message: "Do you want to continue with the Opencontrail compute rollback on the following sample nodes? ${targetLiveSubset}"
+            }
+
+            stage("Opencontrail compute rollback on sample nodes") {
+
+                oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+                oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+                try {
+                    salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                    salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'saltutil.refresh_pillar', [], null, true)
+                    salt.enforceState(saltMaster, targetLiveSubset, 'linux.system.repo')
+                } catch (Exception er) {
+                    errorOccured = true
+                    common.errorMsg("Opencontrail component on ${targetLiveSubset} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                    return
+                }
+
+                args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS}  -y;"
+                check = 'contrail-status'
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, args, null)
+                salt.printSaltCommandResult(out)
+
+                try {
+                    salt.enforceState(saltMaster, targetLiveSubset, 'opencontrail')
+                } catch (Exception er) {
+                    common.errorMsg("Opencontrail state was executed on ${targetLiveSubset} and failed please fix it manually.")
+                }
+
+                salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+                //sleep(10)
+                salt.commandStatus(saltMaster, targetLiveSubset, "${check} | grep -v == | grep -v active", null, false)
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
+                salt.printSaltCommandResult(out)
+            }
+
+            stage('Confirm rollback on all targeted nodes') {
+                input message: "Do you want to continue with the Opencontrail compute upgrade on all the targeted nodes? ${targetLiveAll} nodes?"
+            }
+
+            stage("Opencontrail compute upgrade on all targeted nodes") {
+
+                oc_component_repo = salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+                oc_component_repo = oc_component_repo['return'][0].values()[0]
+
+                try {
+                    salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["rm ${oc_component_repo}"], null, true)
+                    salt.runSaltProcessStep(saltMaster, targetLiveAll, 'saltutil.refresh_pillar', [], null, true)
+                    salt.enforceState(saltMaster, targetLiveAll, 'linux.system.repo')
+                } catch (Exception er) {
+                    common.errorMsg("Opencontrail component on ${targetLiveAll} probably failed to be replaced. Please check it in ${oc_component_repo} before continuing.")
+                    return
+                }
+
+                args = "export DEBIAN_FRONTEND=noninteractive; apt install --allow-downgrades -o Dpkg::Options::=\"--force-confold\" -o Dpkg::Options::=\"--force-confdef\" ${CMP_PKGS} -y;"
+                check = 'contrail-status'
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, args, null)
+                salt.printSaltCommandResult(out)
+
+                try {
+                    salt.enforceState(saltMaster, targetLiveAll, 'opencontrail')
+                } catch (Exception er) {
+                    common.errorMsg("Opencontrail state was executed on ${targetLiveAll} and failed please fix it manually.")
+                }
+
+                salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
+
+                //sleep(10)
+                salt.commandStatus(saltMaster, targetLiveAll, "${check} | grep -v == | grep -v active", null, false)
+
+                out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
+                salt.printSaltCommandResult(out)
+            }
+
+        } catch (Throwable e) {
+            // If there was an error or exception thrown, the build failed
+            currentBuild.result = "FAILURE"
+            throw e
         }
     }
 }
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 7bf646c..9680f24 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -16,549 +16,396 @@
 
 def saltMaster
 
-timestamps {
-    node() {
+node() {
 
-        stage('Connect to Salt API') {
-            saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
-        }
+    stage('Connect to Salt API') {
+        saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    }
 
-        if (STAGE_TEST_UPGRADE.toBoolean() == true) {
-            stage('Test upgrade') {
+    if (STAGE_TEST_UPGRADE.toBoolean() == true) {
+        stage('Test upgrade') {
 
 
-                try {
-                    salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
-                } catch (Exception e) {
-                    common.warningMsg("Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
-                }
+            try {
+                salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
+            } catch (Exception e) {
+                common.warningMsg("Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
+            }
 
-                try {
-                    salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
-                } catch (Exception e) {
-                    common.warningMsg("No response from some minions. We should continue to run")
-                }
-                
-                try {
-                    salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
-                } catch (Exception e) {
-                    common.warningMsg("No response from some minions. We should continue to run")
-                }
+            try {
+                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
+            } catch (Exception e) {
+                common.warningMsg("No response from some minions. We should continue to run")
+            }
 
-                def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
-                def domain = _pillar['return'][0].values()[0].values()[0]
-                print(_pillar)
-                print(domain)
+            try {
+                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
+            } catch (Exception e) {
+                common.warningMsg("No response from some minions. We should continue to run")
+            }
 
-                // read backupninja variable
-                _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
-                def backupninja_backup_host = _pillar['return'][0].values()[0]
-                print(_pillar)
-                print(backupninja_backup_host)
+            def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
+            def domain = _pillar['return'][0].values()[0].values()[0]
+            print(_pillar)
+            print(domain)
 
-                _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
-                def kvm01 = _pillar['return'][0].values()[0].values()[0]
-                print(_pillar)
-                print(kvm01)
+            // read backupninja variable
+            _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
+            def backupninja_backup_host = _pillar['return'][0].values()[0]
+            print(_pillar)
+            print(backupninja_backup_host)
 
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
-                def upgNodeProvider = _pillar['return'][0].values()[0]
-                print(_pillar)
-                print(upgNodeProvider)
+            _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
+            def kvm01 = _pillar['return'][0].values()[0].values()[0]
+            print(_pillar)
+            print(kvm01)
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
+            def upgNodeProvider = _pillar['return'][0].values()[0]
+            print(_pillar)
+            print(upgNodeProvider)
 
 
-                salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
-
-                
-                try {
-                    salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
-                } catch (Exception e) {
-                    common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
-                }
+            salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
 
 
-                // salt 'kvm02*' state.sls salt.control
-                salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
+            try {
+                salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
+            } catch (Exception e) {
+                common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
+            }
 
-                // wait until upg node is registered in salt-key
-                salt.minionPresent(saltMaster, 'I@salt:master', 'upg01')
 
-                // salt '*' saltutil.refresh_pillar
-                salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
-                // salt '*' saltutil.sync_all
-                salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
+            // salt 'kvm02*' state.sls salt.control
+            salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
 
-                // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
-                try {
-                    salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
-                } catch (Exception e) {
-                    common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
-                }
+            // wait until upg node is registered in salt-key
+            salt.minionPresent(saltMaster, 'I@salt:master', 'upg01')
+
+            // salt '*' saltutil.refresh_pillar
+            salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
+            // salt '*' saltutil.sync_all
+            salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
+
+            // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
+            try {
                 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
+            } catch (Exception e) {
+                common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
+            }
+            salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
 
-                // salt "upg*" state.sls rabbitmq
-                salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
-                try {
-                    salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
-                } catch (Exception e) {
-                    common.warningMsg('salt-minion was restarted. We should continue to run')
-                }
-                try {
-                    salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
-                } catch (Exception e) {
-                    common.warningMsg('salt-minion was restarted. We should continue to run')
-                }
-                // salt '*' state.apply salt.minion.grains
-                //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
-                // salt -C 'I@backupninja:server' state.sls backupninja
-                salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
-                // salt -C 'I@backupninja:client' state.sls backupninja
-                salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
-                salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
-                try {
-                    salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
-                } catch (Exception e) {
-                    common.warningMsg('The ARP entry does not exist. We should continue to run.')
-                }
-                salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
-                salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
-                salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
+            // salt "upg*" state.sls rabbitmq
+            salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
+            try {
+                salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
+            } catch (Exception e) {
+                common.warningMsg('salt-minion was restarted. We should continue to run')
+            }
+            try {
+                salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
+            } catch (Exception e) {
+                common.warningMsg('salt-minion was restarted. We should continue to run')
+            }
+            // salt '*' state.apply salt.minion.grains
+            //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
+            // salt -C 'I@backupninja:server' state.sls backupninja
+            salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
+            // salt -C 'I@backupninja:client' state.sls backupninja
+            salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
+            salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
+            try {
+                salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
+            } catch (Exception e) {
+                common.warningMsg('The ARP entry does not exist. We should continue to run.')
+            }
+            salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
+            salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
+            salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
 
-                salt.enforceState(saltMaster, 'I@xtrabackup:server', 'xtrabackup')
-                salt.enforceState(saltMaster, 'I@xtrabackup:client', 'openssh.client')
-                salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
-                salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
+            salt.enforceState(saltMaster, 'I@xtrabackup:server', 'xtrabackup')
+            salt.enforceState(saltMaster, 'I@xtrabackup:client', 'openssh.client')
+            salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
+            salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
 
-                def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
-                if(databases && databases != ""){
-                    def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
-                    for( i = 0; i < databasesList.size(); i++){
-                        if(databasesList[i].toLowerCase().contains('upgrade')){
-                            salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
-                            common.warningMsg("removing database ${databasesList[i]}")
-                            salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
-                        }
+            def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
+            if(databases && databases != ""){
+                def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
+                for( i = 0; i < databasesList.size(); i++){
+                    if(databasesList[i].toLowerCase().contains('upgrade')){
+                        salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
+                        common.warningMsg("removing database ${databasesList[i]}")
+                        salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
                     }
-                    salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
-                }else{
-                    common.errorMsg("No _upgrade databases were returned")
                 }
+                salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
+            }else{
+                common.errorMsg("No _upgrade databases were returned")
+            }
 
-                try {
-                    salt.enforceState(saltMaster, 'upg*', 'keystone.server')
-                    salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
-                } catch (Exception e) {
-                    common.warningMsg('Restarting Apache2')
-                    salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'keystone.server')
+                salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
+            } catch (Exception e) {
+                common.warningMsg('Restarting Apache2')
+                salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
+            }
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'keystone.client')
+            } catch (Exception e) {
+                common.warningMsg('running keystone.client state again')
+                salt.enforceState(saltMaster, 'upg*', 'keystone.client')
+            }
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'glance')
+            } catch (Exception e) {
+                common.warningMsg('running glance state again')
+                salt.enforceState(saltMaster, 'upg*', 'glance')
+            }
+            salt.enforceState(saltMaster, 'upg*', 'keystone.server')
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'nova')
+            } catch (Exception e) {
+                common.warningMsg('running nova state again')
+                salt.enforceState(saltMaster, 'upg*', 'nova')
+            }
+            // run nova state again as sometimes nova does not enforce itself for some reason
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'nova')
+            } catch (Exception e) {
+                common.warningMsg('running nova state again')
+                salt.enforceState(saltMaster, 'upg*', 'nova')
+            }
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'cinder')
+            } catch (Exception e) {
+                common.warningMsg('running cinder state again')
+                salt.enforceState(saltMaster, 'upg*', 'cinder')
+            }
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'neutron')
+            } catch (Exception e) {
+                common.warningMsg('running neutron state again')
+                salt.enforceState(saltMaster, 'upg*', 'neutron')
+            }
+            try {
+                salt.enforceState(saltMaster, 'upg*', 'heat')
+            } catch (Exception e) {
+                common.warningMsg('running heat state again')
+                salt.enforceState(saltMaster, 'upg*', 'heat')
+            }
+            salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
+
+            if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
+                stage('Ask for manual confirmation') {
+                    input message: "Do you want to continue with upgrade?"
                 }
+            }
+        }
+    }
+
+    if (STAGE_REAL_UPGRADE.toBoolean() == true) {
+        stage('Real upgrade') {
+            // # actual upgrade
+
+            _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
+            domain = _pillar['return'][0].values()[0].values()[0]
+            print(_pillar)
+            print(domain)
+
+            _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
+            kvm01 = _pillar['return'][0].values()[0].values()[0]
+            print(_pillar)
+            print(kvm01)
+
+            def errorOccured = false
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
+            def ctl01NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
+            def ctl02NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
+            def ctl03NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
+            def prx01NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
+            def prx02NodeProvider = _pillar['return'][0].values()[0]
+
+
+            salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
+
+
+            try {
+                salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
+            } catch (Exception e) {
+                common.warningMsg('File already exists')
+            }
+            try {
+                salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
+            } catch (Exception e) {
+                common.warningMsg('File already exists')
+            }
+            try {
+                salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
+            } catch (Exception e) {
+                common.warningMsg('File already exists')
+            }
+            try {
+                salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
+            } catch (Exception e) {
+                common.warningMsg('File already exists')
+            }
+            try {
+                salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
+            } catch (Exception e) {
+                common.warningMsg('File already exists')
+            }
+
+
+            salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
+
+            salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
+
+            try {
+                salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
+            } catch (Exception e) {
+                common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
+            }
+
+            // salt 'kvm*' state.sls salt.control
+            salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
+
+            // wait until ctl and prx nodes are registered in salt-key
+            salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
+
+
+            // salt '*' saltutil.refresh_pillar
+            salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
+            // salt '*' saltutil.sync_all
+            salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
+
+            try {
+                salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
+            } catch (Exception e) {
+                common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
+            }
+            salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
+
+            // salt 'ctl*' state.sls keepalived
+            // salt 'ctl*' state.sls haproxy
+            salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
+            // salt 'ctl*' service.restart rsyslog
+            salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
+            // salt "ctl*" state.sls memcached
+            // salt "ctl*" state.sls keystone.server
+            try {
                 try {
-                    salt.enforceState(saltMaster, 'upg*', 'keystone.client')
+                    salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
+                    salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
+                } catch (Exception e) {
+                    common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
+                    salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
+                    salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
+                }
+                // salt 'ctl01*' state.sls keystone.client
+                try {
+                    salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
                 } catch (Exception e) {
                     common.warningMsg('running keystone.client state again')
-                    salt.enforceState(saltMaster, 'upg*', 'keystone.client')
+                    salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
                 }
                 try {
-                    salt.enforceState(saltMaster, 'upg*', 'glance')
+                    salt.enforceState(saltMaster, 'ctl*', 'glance')
                 } catch (Exception e) {
                     common.warningMsg('running glance state again')
-                    salt.enforceState(saltMaster, 'upg*', 'glance')
-                }
-                salt.enforceState(saltMaster, 'upg*', 'keystone.server')
+                    salt.enforceState(saltMaster, 'ctl*', 'glance')
+                }                // salt 'ctl*' state.sls glusterfs.client
+                salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
+                // salt 'ctl*' state.sls keystone.server
+                salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
+                // salt 'ctl*' state.sls nova
                 try {
-                    salt.enforceState(saltMaster, 'upg*', 'nova')
+                    salt.enforceState(saltMaster, 'ctl*', 'nova')
                 } catch (Exception e) {
                     common.warningMsg('running nova state again')
-                    salt.enforceState(saltMaster, 'upg*', 'nova')
+                    salt.enforceState(saltMaster, 'ctl*', 'nova')
                 }
-                // run nova state again as sometimes nova does not enforce itself for some reason
+                // salt 'ctl*' state.sls cinder
                 try {
-                    salt.enforceState(saltMaster, 'upg*', 'nova')
-                } catch (Exception e) {
-                    common.warningMsg('running nova state again')
-                    salt.enforceState(saltMaster, 'upg*', 'nova')
-                }
-                try {
-                    salt.enforceState(saltMaster, 'upg*', 'cinder')
+                    salt.enforceState(saltMaster, 'ctl*', 'cinder')
                 } catch (Exception e) {
                     common.warningMsg('running cinder state again')
-                    salt.enforceState(saltMaster, 'upg*', 'cinder')
-                }                
+                    salt.enforceState(saltMaster, 'ctl*', 'cinder')
+                }
                 try {
-                    salt.enforceState(saltMaster, 'upg*', 'neutron')
+                    salt.enforceState(saltMaster, 'ctl*', 'neutron')
                 } catch (Exception e) {
                     common.warningMsg('running neutron state again')
-                    salt.enforceState(saltMaster, 'upg*', 'neutron')
+                    salt.enforceState(saltMaster, 'ctl*', 'neutron')
                 }
+                // salt 'ctl*' state.sls heat
                 try {
-                    salt.enforceState(saltMaster, 'upg*', 'heat')
+                    salt.enforceState(saltMaster, 'ctl*', 'heat')
                 } catch (Exception e) {
                     common.warningMsg('running heat state again')
-                    salt.enforceState(saltMaster, 'upg*', 'heat')
-                }
-                salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
-
-                if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
-                    stage('Ask for manual confirmation') {
-                        input message: "Do you want to continue with upgrade?"
-                    }
-                }
-            }
-        }
-
-        if (STAGE_REAL_UPGRADE.toBoolean() == true) {
-            stage('Real upgrade') {
-                // # actual upgrade
-
-                _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
-                domain = _pillar['return'][0].values()[0].values()[0]
-                print(_pillar)
-                print(domain)
-
-                _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
-                kvm01 = _pillar['return'][0].values()[0].values()[0]
-                print(_pillar)
-                print(kvm01)
-
-                def errorOccured = false
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
-                def ctl01NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
-                def ctl02NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
-                def ctl03NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
-                def prx01NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
-                def prx02NodeProvider = _pillar['return'][0].values()[0]
-
-
-                salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
-
-
-                try {
-                    salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
-                } catch (Exception e) {
-                    common.warningMsg('File already exists')
-                }
-                try {
-                    salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
-                } catch (Exception e) {
-                    common.warningMsg('File already exists')
-                }
-                try {
-                    salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
-                } catch (Exception e) {
-                    common.warningMsg('File already exists')
-                }
-                try {
-                    salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
-                } catch (Exception e) {
-                    common.warningMsg('File already exists')
-                }
-                try {
-                    salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
-                } catch (Exception e) {
-                    common.warningMsg('File already exists')
+                    salt.enforceState(saltMaster, 'ctl*', 'heat')
                 }
 
-
-                salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
-
-                salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
-
-                try {
-                    salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
-                } catch (Exception e) {
-                    common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
-                }
-
-                // salt 'kvm*' state.sls salt.control
-                salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
-
-                // wait until ctl and prx nodes are registered in salt-key
-                salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
-
-
-                // salt '*' saltutil.refresh_pillar
-                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
-                // salt '*' saltutil.sync_all
-                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
-
-                try {
-                    salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
-                } catch (Exception e) {
-                    common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
-                }
-                salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
-
-                // salt 'ctl*' state.sls keepalived
-                // salt 'ctl*' state.sls haproxy
-                salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
-                // salt 'ctl*' service.restart rsyslog
-                salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
-                // salt "ctl*" state.sls memcached
-                // salt "ctl*" state.sls keystone.server
-                try {
-                    try {
-                        salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
-                        salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
-                    } catch (Exception e) {
-                        common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
-                        salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
-                        salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
-                    }
-                    // salt 'ctl01*' state.sls keystone.client
-                    try {
-                        salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
-                    } catch (Exception e) {
-                        common.warningMsg('running keystone.client state again')
-                        salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
-                    } 
-                    try {
-                        salt.enforceState(saltMaster, 'ctl*', 'glance')
-                    } catch (Exception e) {
-                        common.warningMsg('running glance state again')
-                        salt.enforceState(saltMaster, 'ctl*', 'glance')
-                    }                // salt 'ctl*' state.sls glusterfs.client
-                    salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
-                    // salt 'ctl*' state.sls keystone.server
-                    salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
-                    // salt 'ctl*' state.sls nova
-                    try {
-                        salt.enforceState(saltMaster, 'ctl*', 'nova')
-                    } catch (Exception e) {
-                        common.warningMsg('running nova state again')
-                        salt.enforceState(saltMaster, 'ctl*', 'nova')
-                    }
-                    // salt 'ctl*' state.sls cinder
-                    try {
-                        salt.enforceState(saltMaster, 'ctl*', 'cinder')
-                    } catch (Exception e) {
-                        common.warningMsg('running cinder state again')
-                        salt.enforceState(saltMaster, 'ctl*', 'cinder')
-                    }                
-                    try {
-                        salt.enforceState(saltMaster, 'ctl*', 'neutron')
-                    } catch (Exception e) {
-                        common.warningMsg('running neutron state again')
-                        salt.enforceState(saltMaster, 'ctl*', 'neutron')
-                    }
-                    // salt 'ctl*' state.sls heat
-                    try {
-                        salt.enforceState(saltMaster, 'ctl*', 'heat')
-                    } catch (Exception e) {
-                        common.warningMsg('running heat state again')
-                        salt.enforceState(saltMaster, 'ctl*', 'heat')
-                    }
-
-                } catch (Exception e) {
-                    errorOccured = true
-                    common.warningMsg('Some states that require syncdb failed. Restoring production databases')
-
-                    // database restore section
-                    try {
-                        salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
-                    } catch (Exception er) {
-                        common.warningMsg('Mysql service already stopped')
-                    }
-                    try {
-                        salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
-                    } catch (Exception er) {
-                        common.warningMsg('Mysql service already stopped')
-                    }
-                    try {
-                        salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
-                    } catch (Exception er) {
-                        common.warningMsg('Files are not present')
-                    }
-                    try {
-                        salt.cmdRun(saltMaster, 'I@galera:master', "mkdir /root/mysql/mysql.bak")
-                    } catch (Exception er) {
-                        common.warningMsg('Directory already exists')
-                    }
-                    try {
-                        salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /root/mysql/mysql.bak/*")
-                    } catch (Exception er) {
-                        common.warningMsg('Directory already empty')
-                    }
-                    try {
-                        salt.cmdRun(saltMaster, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
-                    } catch (Exception er) {
-                        common.warningMsg('Files were already moved')
-                    }
-                    try {
-                        salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
-                    } catch (Exception er) {
-                        common.warningMsg('File is not present')
-                    }
-                    salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
-                    _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
-                    backup_dir = _pillar['return'][0].values()[0]
-                    if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
-                    print(backup_dir)
-                    salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
-                    salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
-                    salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
-
-                    // wait until mysql service on galera master is up
-                    salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
-
-                    salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
-                    //
-
-                    common.errorMsg("Stage Real control upgrade failed")
-                }
-                if(!errorOccured){
-
-                    ceph = null
-
-                    try {
-                        ceph = salt.cmdRun(saltMaster, 'ctl*', "salt-call grains.item roles | grep ceph.client")
-
-                    } catch (Exception er) {
-                        common.infoMsg("Ceph is not used")
-                    }
-
-                    if(ceph != null) {
-                        try {
-                            salt.enforceState(saltMaster, 'ctl*', 'ceph.client')
-                        } catch (Exception er) {
-                            common.warningMsg("Ceph client state on controllers failed. Please fix it manually")
-                        }
-                    }
-
-                    // salt 'cmp*' cmd.run 'service nova-compute restart'
-                    salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
-                    salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
-                    salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
-
-
-                    // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog
-                    // salt 'ctl*' state.sls keepalived
-                    // salt 'prx*' state.sls keepalived
-                    salt.enforceState(saltMaster, 'prx*', 'keepalived')
-                    // salt 'prx*' state.sls horizon
-                    salt.enforceState(saltMaster, 'prx*', 'horizon')
-                    // salt 'prx*' state.sls nginx
-                    salt.enforceState(saltMaster, 'prx*', 'nginx')
-                    // salt "prx*" state.sls memcached
-                    salt.enforceState(saltMaster, 'prx*', 'memcached')
-
-                    try {
-                        salt.enforceHighstate(saltMaster, 'ctl*')
-                    } catch (Exception er) {
-                        common.errorMsg("Highstate was executed on controller nodes but something failed. Please check it and fix it accordingly.")
-                    }
-
-                    try {
-                        salt.enforceHighstate(saltMaster, 'prx*')
-                    } catch (Exception er) {
-                        common.errorMsg("Highstate was executed on proxy nodes but something failed. Please check it and fix it accordingly.")
-                    }
-
-                    salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
-                }
-            }
-
-            if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
-                stage('Ask for manual confirmation') {
-                    input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
-                }
-            }
-        }
-
-        if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
-            stage('Rollback upgrade') {
-
-                stage('Ask for manual confirmation') {
-                    input message: "Do you really want to continue with the rollback?"
-                }
-
-                _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
-                domain = _pillar['return'][0].values()[0].values()[0]
-                print(_pillar)
-                print(domain)
-
-                _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
-                kvm01 = _pillar['return'][0].values()[0].values()[0]
-                print(_pillar)
-                print(kvm01)
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
-                def ctl01NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
-                def ctl02NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
-                def ctl03NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
-                def prx01NodeProvider = _pillar['return'][0].values()[0]
-
-                _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
-                def prx02NodeProvider = _pillar['return'][0].values()[0]
-
-                salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
-
-                salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
-
-                try {
-                    salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
-                } catch (Exception e) {
-                    common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
-                }
+            } catch (Exception e) {
+                errorOccured = true
+                common.warningMsg('Some states that require syncdb failed. Restoring production databases')
 
                 // database restore section
                 try {
                     salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
-                } catch (Exception e) {
+                } catch (Exception er) {
                     common.warningMsg('Mysql service already stopped')
                 }
                 try {
                     salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
-                } catch (Exception e) {
+                } catch (Exception er) {
                     common.warningMsg('Mysql service already stopped')
                 }
                 try {
                     salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
-                } catch (Exception e) {
+                } catch (Exception er) {
                     common.warningMsg('Files are not present')
                 }
                 try {
-                    salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /var/lib/mysql/*")
-                } catch (Exception e) {
+                    salt.cmdRun(saltMaster, 'I@galera:master', "mkdir /root/mysql/mysql.bak")
+                } catch (Exception er) {
+                    common.warningMsg('Directory already exists')
+                }
+                try {
+                    salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /root/mysql/mysql.bak/*")
+                } catch (Exception er) {
                     common.warningMsg('Directory already empty')
                 }
                 try {
+                    salt.cmdRun(saltMaster, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
+                } catch (Exception er) {
+                    common.warningMsg('Files were already moved')
+                }
+                try {
                     salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
-                } catch (Exception e) {
+                } catch (Exception er) {
                     common.warningMsg('File is not present')
                 }
                 salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
@@ -576,27 +423,178 @@
                 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
                 //
 
-                salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
-                salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
+                common.errorMsg("Stage Real control upgrade failed")
+            }
+            if(!errorOccured){
+
+                ceph = null
+
+                try {
+                    ceph = salt.cmdRun(saltMaster, 'ctl*', "salt-call grains.item roles | grep ceph.client")
+
+                } catch (Exception er) {
+                    common.infoMsg("Ceph is not used")
+                }
+
+                if(ceph != null) {
+                    try {
+                        salt.enforceState(saltMaster, 'ctl*', 'ceph.client')
+                    } catch (Exception er) {
+                        common.warningMsg("Ceph client state on controllers failed. Please fix it manually")
+                    }
+                }
 
                 // salt 'cmp*' cmd.run 'service nova-compute restart'
                 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
-
-                // wait until ctl and prx nodes are registered in salt-key
-                salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
-                salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
-
                 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
                 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
 
-                salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
+
+                // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog
+                // salt 'ctl*' state.sls keepalived
+                // salt 'prx*' state.sls keepalived
+                salt.enforceState(saltMaster, 'prx*', 'keepalived')
+                // salt 'prx*' state.sls horizon
+                salt.enforceState(saltMaster, 'prx*', 'horizon')
+                // salt 'prx*' state.sls nginx
+                salt.enforceState(saltMaster, 'prx*', 'nginx')
+                // salt "prx*" state.sls memcached
+                salt.enforceState(saltMaster, 'prx*', 'memcached')
+
+                try {
+                    salt.enforceHighstate(saltMaster, 'ctl*')
+                } catch (Exception er) {
+                    common.errorMsg("Highstate was executed on controller nodes but something failed. Please check it and fix it accordingly.")
+                }
+
+                try {
+                    salt.enforceHighstate(saltMaster, 'prx*')
+                } catch (Exception er) {
+                    common.errorMsg("Highstate was executed on proxy nodes but something failed. Please check it and fix it accordingly.")
+                }
+
+                salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
             }
         }
+
+        if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
+            stage('Ask for manual confirmation') {
+                input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
+            }
+        }
+    }
+
+    if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
+        stage('Rollback upgrade') {
+
+            stage('Ask for manual confirmation') {
+                input message: "Do you really want to continue with the rollback?"
+            }
+
+            _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
+            domain = _pillar['return'][0].values()[0].values()[0]
+            print(_pillar)
+            print(domain)
+
+            _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
+            kvm01 = _pillar['return'][0].values()[0].values()[0]
+            print(_pillar)
+            print(kvm01)
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
+            def ctl01NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
+            def ctl02NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
+            def ctl03NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
+            def prx01NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
+            def prx02NodeProvider = _pillar['return'][0].values()[0]
+
+            salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
+
+            salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
+
+            try {
+                salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
+            } catch (Exception e) {
+                common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
+            }
+
+            // database restore section
+            try {
+                salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
+            } catch (Exception e) {
+                common.warningMsg('Mysql service already stopped')
+            }
+            try {
+                salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
+            } catch (Exception e) {
+                common.warningMsg('Mysql service already stopped')
+            }
+            try {
+                salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
+            } catch (Exception e) {
+                common.warningMsg('Files are not present')
+            }
+            try {
+                salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /var/lib/mysql/*")
+            } catch (Exception e) {
+                common.warningMsg('Directory already empty')
+            }
+            try {
+                salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
+            } catch (Exception e) {
+                common.warningMsg('File is not present')
+            }
+            salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
+            _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
+            backup_dir = _pillar['return'][0].values()[0]
+            if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
+            print(backup_dir)
+            salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
+            salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
+            salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
+
+            // wait until mysql service on galera master is up
+            salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
+
+            salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
+            //
+
+            salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
+            salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
+
+            // salt 'cmp*' cmd.run 'service nova-compute restart'
+            salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
+
+            // wait until ctl and prx nodes are registered in salt-key
+            salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
+            salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
+
+            salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
+            salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
+
+            salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
+        }
     }
 }
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index 4a9c89e..89afc6c 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -13,70 +13,68 @@
 
 def saltMaster
 
-timestamps {
-    node() {
+node() {
 
-        stage('Connect to Salt API') {
-            saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    stage('Connect to Salt API') {
+        saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    }
+
+    stage('Start restore') {
+        // # actual upgrade
+
+        stage('Ask for manual confirmation') {
+            input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore Cassandra?"
+        }
+        // Cassandra restore section
+        try {
+            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['supervisor-database'], null, true)
+        } catch (Exception er) {
+            common.warningMsg('Supervisor-database service already stopped')
+        }
+        try {
+            salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir -p /root/cassandra/cassandra.bak")
+        } catch (Exception er) {
+            common.warningMsg('Directory already exists')
         }
 
-        stage('Start restore') {
-            // # actual upgrade
-
-            stage('Ask for manual confirmation') {
-                input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore Cassandra?"
-            }
-            // Cassandra restore section
-            try {
-                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['supervisor-database'], null, true)
-            } catch (Exception er) {
-                common.warningMsg('Supervisor-database service already stopped')
-            }
-            try {
-                salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir -p /root/cassandra/cassandra.bak")
-            } catch (Exception er) {
-                common.warningMsg('Directory already exists')
-            }
-
-            try {
-                salt.cmdRun(saltMaster, 'I@opencontrail:control', "mv /var/lib/cassandra/* /root/cassandra/cassandra.bak")
-            } catch (Exception er) {
-                common.warningMsg('Files were already moved')
-            }
-            try {
-                salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm -rf /var/lib/cassandra/*")
-            } catch (Exception er) {
-                common.warningMsg('Directory already empty')
-            }
-
-            _pillar = salt.getPillar(saltMaster, "I@cassandra:backup:client", 'cassandra:backup:backup_dir')
-            backup_dir = _pillar['return'][0].values()[0]
-            if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/cassandra' }
-            print(backup_dir)
-            salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
-
-            salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'service.start', ['supervisor-database'], null, true)
-
-            // wait until supervisor-database service is up
-            salt.commandStatus(saltMaster, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
-
-            // performs restore
-            salt.cmdRun(saltMaster, 'I@cassandra:backup:client', "su root -c 'salt-call state.sls cassandra'")
-            salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
-            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and not I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
-
-            // wait until supervisor-database service is up
-            salt.commandStatus(saltMaster, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
-            salt.commandStatus(saltMaster, 'I@opencontrail:control and not I@cassandra:backup:client', 'service supervisor-database status', 'running')
-            sleep(5)
-
-            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.restart', ['supervisor-database'], null, true)
-
-            // wait until contrail-status is up
-            salt.commandStatus(saltMaster, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
-            
-            salt.cmdRun(saltMaster, 'I@opencontrail:control', "nodetool status")
-            salt.cmdRun(saltMaster, 'I@opencontrail:control', "contrail-status")
+        try {
+            salt.cmdRun(saltMaster, 'I@opencontrail:control', "mv /var/lib/cassandra/* /root/cassandra/cassandra.bak")
+        } catch (Exception er) {
+            common.warningMsg('Files were already moved')
         }
+        try {
+            salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm -rf /var/lib/cassandra/*")
+        } catch (Exception er) {
+            common.warningMsg('Directory already empty')
+        }
+
+        _pillar = salt.getPillar(saltMaster, "I@cassandra:backup:client", 'cassandra:backup:backup_dir')
+        backup_dir = _pillar['return'][0].values()[0]
+        if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/cassandra' }
+        print(backup_dir)
+        salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
+
+        salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'service.start', ['supervisor-database'], null, true)
+
+        // wait until supervisor-database service is up
+        salt.commandStatus(saltMaster, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
+
+        // performs restore
+        salt.cmdRun(saltMaster, 'I@cassandra:backup:client', "su root -c 'salt-call state.sls cassandra'")
+        salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
+        salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and not I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
+
+        // wait until supervisor-database service is up
+        salt.commandStatus(saltMaster, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
+        salt.commandStatus(saltMaster, 'I@opencontrail:control and not I@cassandra:backup:client', 'service supervisor-database status', 'running')
+        sleep(5)
+
+        salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.restart', ['supervisor-database'], null, true)
+
+        // wait until contrail-status is up
+        salt.commandStatus(saltMaster, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
+
+        salt.cmdRun(saltMaster, 'I@opencontrail:control', "nodetool status")
+        salt.cmdRun(saltMaster, 'I@opencontrail:control', "contrail-status")
     }
 }
diff --git a/restore-zookeeper.groovy b/restore-zookeeper.groovy
index 3e5da6c..3ead8aa 100644
--- a/restore-zookeeper.groovy
+++ b/restore-zookeeper.groovy
@@ -13,82 +13,77 @@
 
 def saltMaster
 
-timestamps {
-    node() {
+node() {
 
-        stage('Connect to Salt API') {
-            saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    stage('Connect to Salt API') {
+        saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    }
+
+    stage('Start restore') {
+        // # actual upgrade
+
+        stage('Ask for manual confirmation') {
+            input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore Zookeeper?"
+        }
+        // Zookeeper restore section
+        try {
+            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['supervisor-config'], null, true)
+        } catch (Exception er) {
+            common.warningMsg('Supervisor-config service already stopped')
+        }
+        try {
+            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['supervisor-control'], null, true)
+        } catch (Exception er) {
+            common.warningMsg('Supervisor-control service already stopped')
+        }
+        try {
+            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['zookeeper'], null, true)
+        } catch (Exception er) {
+            common.warningMsg('Zookeeper service already stopped')
+        }
+        //sleep(5)
+        // wait until zookeeper service is down
+        salt.commandStatus(saltMaster, 'I@opencontrail:control', 'service zookeeper status', 'stop')
+
+        try {
+            salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir -p /root/zookeeper/zookeeper.bak")
+        } catch (Exception er) {
+            common.warningMsg('Directory already exists')
         }
 
-        stage('Start restore') {
-            // # actual upgrade
-
-            stage('Ask for manual confirmation') {
-                input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore Zookeeper?"
-            }
-            // Zookeeper restore section
-            try {
-                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['supervisor-config'], null, true)
-            } catch (Exception er) {
-                common.warningMsg('Supervisor-config service already stopped')
-            }
-            try {
-                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['supervisor-control'], null, true)
-            } catch (Exception er) {
-                common.warningMsg('Supervisor-control service already stopped')
-            }
-            try {
-                salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.stop', ['zookeeper'], null, true)
-            } catch (Exception er) {
-                common.warningMsg('Zookeeper service already stopped')
-            }
-            //sleep(5)
-            // wait until zookeeper service is down
-            salt.commandStatus(saltMaster, 'I@opencontrail:control', 'service zookeeper status', 'stop')
-
-            try {
-                salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir -p /root/zookeeper/zookeeper.bak")
-            } catch (Exception er) {
-                common.warningMsg('Directory already exists')
-            }
-
-            try {
-                salt.cmdRun(saltMaster, 'I@opencontrail:control', "mv /var/lib/zookeeper/version-2/* /root/zookeeper/zookeeper.bak")
-            } catch (Exception er) {
-                common.warningMsg('Files were already moved')
-            }
-            try {
-                salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm -rf /var/lib/zookeeper/version-2/*")
-            } catch (Exception er) {
-                common.warningMsg('Directory already empty')
-            }
-
-            _pillar = salt.getPillar(saltMaster, "I@opencontrail:control", 'zookeeper:backup:backup_dir')
-            backup_dir = _pillar['return'][0].values()[0]
-            if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/zookeeper' }
-            print(backup_dir)
-            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
-
-            // performs restore
-            salt.cmdRun(saltMaster, 'I@opencontrail:control', "su root -c 'salt-call state.sls zookeeper'")
-
-            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['zookeeper'], null, true)
-            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-config'], null, true)
-            salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-control'], null, true)
-
-            // wait until contrail-status is up
-            salt.commandStatus(saltMaster, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
-            
-            salt.cmdRun(saltMaster, 'I@opencontrail:control', "ls /var/lib/zookeeper/version-2")
-            try {
-                salt.cmdRun(saltMaster, 'I@opencontrail:control', "echo stat | nc localhost 2181")
-            } catch (Exception er) {
-                common.warningMsg('Check which node is zookeeper leader')
-            }
-            salt.cmdRun(saltMaster, 'I@opencontrail:control', "contrail-status")
+        try {
+            salt.cmdRun(saltMaster, 'I@opencontrail:control', "mv /var/lib/zookeeper/version-2/* /root/zookeeper/zookeeper.bak")
+        } catch (Exception er) {
+            common.warningMsg('Files were already moved')
         }
+        try {
+            salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm -rf /var/lib/zookeeper/version-2/*")
+        } catch (Exception er) {
+            common.warningMsg('Directory already empty')
+        }
+
+        _pillar = salt.getPillar(saltMaster, "I@opencontrail:control", 'zookeeper:backup:backup_dir')
+        backup_dir = _pillar['return'][0].values()[0]
+        if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/zookeeper' }
+        print(backup_dir)
+        salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
+
+        // performs restore
+        salt.cmdRun(saltMaster, 'I@opencontrail:control', "su root -c 'salt-call state.sls zookeeper'")
+
+        salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['zookeeper'], null, true)
+        salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-config'], null, true)
+        salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-control'], null, true)
+
+        // wait until contrail-status is up
+        salt.commandStatus(saltMaster, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
+
+        salt.cmdRun(saltMaster, 'I@opencontrail:control', "ls /var/lib/zookeeper/version-2")
+        try {
+            salt.cmdRun(saltMaster, 'I@opencontrail:control', "echo stat | nc localhost 2181")
+        } catch (Exception er) {
+            common.warningMsg('Check which node is zookeeper leader')
+        }
+        salt.cmdRun(saltMaster, 'I@opencontrail:control', "contrail-status")
     }
 }
-
-
-
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index cff994d..db1086a 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -92,82 +92,80 @@
   gerritRef = null
 }
 
-timestamps {
-    node("python&&docker") {
-        def templateEnv = "${env.WORKSPACE}"
-        def cutterEnv = "${env.WORKSPACE}/cutter"
-        def jinjaEnv = "${env.WORKSPACE}/jinja"
+node("python&&docker") {
+    def templateEnv = "${env.WORKSPACE}"
+    def cutterEnv = "${env.WORKSPACE}/cutter"
+    def jinjaEnv = "${env.WORKSPACE}/jinja"
 
-        try {
-            stage("Cleanup") {
-                sh("rm -rf * || true")
-            }
-
-            stage ('Download Cookiecutter template') {
-                if (gerritRef) {
-                    def gerritChange = gerrit.getGerritChange(GERRIT_NAME, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID)
-                    merged = gerritChange.status == "MERGED"
-                    if(!merged){
-                        checkouted = gerrit.gerritPatchsetCheckout ([
-                            credentialsId : CREDENTIALS_ID
-                        ])
-                    } else{
-                        common.successMsg("Change ${GERRIT_CHANGE_NUMBER} is already merged, no need to gate them")
-                    }
-                } else {
-                    git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, CREDENTIALS_ID)
-                }
-            }
-
-            stage("Setup") {
-                python.setupCookiecutterVirtualenv(cutterEnv)
-            }
-
-            def contextFiles
-            dir("${templateEnv}/contexts") {
-                contextFiles = findFiles(glob: "*.yml")
-            }
-
-            def contextFileList = []
-            for (int i = 0; i < contextFiles.size(); i++) {
-                contextFileList << contextFiles[i]
-            }
-
-            stage("generate-model") {
-                for (contextFile in contextFileList) {
-                    generateModel(contextFile, cutterEnv)
-                }
-            }
-
-            dir("${env.WORKSPACE}") {
-                sh(returnStatus: true, script: "tar -zcvf model.tar.gz -C model .")
-                archiveArtifacts artifacts: "model.tar.gz"
-            }
-
-            stage("test-nodes") {
-                def partitions = common.partitionList(contextFileList, PARALLEL_NODE_GROUP_SIZE.toInteger())
-                def buildSteps = [:]
-                for (int i = 0; i < partitions.size(); i++) {
-                    def partition = partitions[i]
-                    buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
-                    for(int k = 0; k < partition.size; k++){
-                        def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
-                        def testEnv = "${env.WORKSPACE}/model/${basename}"
-                        buildSteps.get("partition-${i}").put(basename, { testModel(basename, testEnv) })
-                    }
-                }
-                common.serial(buildSteps)
-            }
-
-            stage ('Clean workspace directories') {
-                sh(returnStatus: true, script: "rm -rfv * > /dev/null || true")
-            }
-
-        } catch (Throwable e) {
-             currentBuild.result = "FAILURE"
-             throw e
-        } finally {
-            common.sendNotification(currentBuild.result,"",["slack"])
+    try {
+        stage("Cleanup") {
+            sh("rm -rf * || true")
         }
+
+        stage ('Download Cookiecutter template') {
+            if (gerritRef) {
+                def gerritChange = gerrit.getGerritChange(GERRIT_NAME, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID)
+                merged = gerritChange.status == "MERGED"
+                if(!merged){
+                    checkouted = gerrit.gerritPatchsetCheckout ([
+                        credentialsId : CREDENTIALS_ID
+                    ])
+                } else{
+                    common.successMsg("Change ${GERRIT_CHANGE_NUMBER} is already merged, no need to gate them")
+                }
+            } else {
+                git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, CREDENTIALS_ID)
+            }
+        }
+
+        stage("Setup") {
+            python.setupCookiecutterVirtualenv(cutterEnv)
+        }
+
+        def contextFiles
+        dir("${templateEnv}/contexts") {
+            contextFiles = findFiles(glob: "*.yml")
+        }
+
+        def contextFileList = []
+        for (int i = 0; i < contextFiles.size(); i++) {
+            contextFileList << contextFiles[i]
+        }
+
+        stage("generate-model") {
+            for (contextFile in contextFileList) {
+                generateModel(contextFile, cutterEnv)
+            }
+        }
+
+        dir("${env.WORKSPACE}") {
+            sh(returnStatus: true, script: "tar -zcvf model.tar.gz -C model .")
+            archiveArtifacts artifacts: "model.tar.gz"
+        }
+
+        stage("test-nodes") {
+            def partitions = common.partitionList(contextFileList, PARALLEL_NODE_GROUP_SIZE.toInteger())
+            def buildSteps = [:]
+            for (int i = 0; i < partitions.size(); i++) {
+                def partition = partitions[i]
+                buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
+                for(int k = 0; k < partition.size; k++){
+                    def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
+                    def testEnv = "${env.WORKSPACE}/model/${basename}"
+                    buildSteps.get("partition-${i}").put(basename, { testModel(basename, testEnv) })
+                }
+            }
+            common.serial(buildSteps)
+        }
+
+        stage ('Clean workspace directories') {
+            sh(returnStatus: true, script: "rm -rfv * > /dev/null || true")
+        }
+
+    } catch (Throwable e) {
+         currentBuild.result = "FAILURE"
+         throw e
+    } finally {
+        common.sendNotification(currentBuild.result,"",["slack"])
     }
 }
diff --git a/test-openstack-component-pipeline.groovy b/test-openstack-component-pipeline.groovy
index d3504d3..f6ba0b7 100644
--- a/test-openstack-component-pipeline.groovy
+++ b/test-openstack-component-pipeline.groovy
@@ -14,18 +14,15 @@
 def common = new com.mirantis.mk.Common()
 def gerrit = new com.mirantis.mk.Gerrit()
 
+node {
+    def cred = common.getCredentials(CREDENTIALS_ID, 'key')
+    def gerritChange = gerrit.getGerritChange(cred.username, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID, true)
 
-timestamps {
-    node {
-        def cred = common.getCredentials(CREDENTIALS_ID, 'key')
-        def gerritChange = gerrit.getGerritChange(cred.username, GERRIT_HOST, GERRIT_CHANGE_NUMBER, CREDENTIALS_ID, true)
-
-        stage('Trigger deploy job') {
-            build(job: STACK_DEPLOY_JOB, parameters: [
-                [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: 'mcp-oscore'],
-                [$class: 'StringParameterValue', name: 'STACK_TEST', value: ''],
-                [$class: 'BooleanParameterValue', name: 'TEST_DOCKER_INSTALL', value: false]
-            ])
-        }
+    stage('Trigger deploy job') {
+        build(job: STACK_DEPLOY_JOB, parameters: [
+            [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: 'mcp-oscore'],
+            [$class: 'StringParameterValue', name: 'STACK_TEST', value: ''],
+            [$class: 'BooleanParameterValue', name: 'TEST_DOCKER_INSTALL', value: false]
+        ])
     }
 }
diff --git a/xtrabackup-restore-mysql-db.groovy b/xtrabackup-restore-mysql-db.groovy
index 345f280..6401789 100644
--- a/xtrabackup-restore-mysql-db.groovy
+++ b/xtrabackup-restore-mysql-db.groovy
@@ -13,74 +13,69 @@
 
 def saltMaster
 
-timestamps {
-    node() {
+node() {
 
-        stage('Connect to Salt API') {
-            saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    stage('Connect to Salt API') {
+        saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+    }
+
+    stage('Start restore') {
+        // # actual upgrade
+
+        stage('Ask for manual confirmation') {
+            input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore mysql db?"
         }
-
-        stage('Start restore') {
-            // # actual upgrade
-
-            stage('Ask for manual confirmation') {
-                input message: "Are you sure you have the correct backups ready? Do you really want to continue to restore mysql db?"
-            }
-            // database restore section
-            try {
-                salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
-            } catch (Exception er) {
-                common.warningMsg('Mysql service already stopped')
-            }
-            try {
-                salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
-            } catch (Exception er) {
-                common.warningMsg('Mysql service already stopped')
-            }
-            try {
-                salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
-            } catch (Exception er) {
-                common.warningMsg('Files are not present')
-            }
-            try {
-                salt.cmdRun(saltMaster, 'I@galera:master', "mkdir -p /root/mysql/mysql.bak")
-            } catch (Exception er) {
-                common.warningMsg('Directory already exists')
-            }
-            try {
-                salt.cmdRun(saltMaster, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
-            } catch (Exception er) {
-                common.warningMsg('Files were already moved')
-            }
-            try {
-                salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /var/lib/mysql/*")
-            } catch (Exception er) {
-                common.warningMsg('Directory already empty')
-            }
-            try {
-                salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
-            } catch (Exception er) {
-                common.warningMsg('File is not present')
-            }
-            salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
-            _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
-            backup_dir = _pillar['return'][0].values()[0]
-            if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
-            print(backup_dir)
-            salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
-            salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
-            salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
-
-            // wait until mysql service on galera master is up
-            salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
-
-            salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
-            sleep(15)
-            salt.cmdRun(saltMaster, 'I@galera:master', "su root -c 'salt-call mysql.status | grep -A1 wsrep_cluster_size'")
-
+        // database restore section
+        try {
+            salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
+        } catch (Exception er) {
+            common.warningMsg('Mysql service already stopped')
         }
+        try {
+            salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
+        } catch (Exception er) {
+            common.warningMsg('Mysql service already stopped')
+        }
+        try {
+            salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
+        } catch (Exception er) {
+            common.warningMsg('Files are not present')
+        }
+        try {
+            salt.cmdRun(saltMaster, 'I@galera:master', "mkdir -p /root/mysql/mysql.bak")
+        } catch (Exception er) {
+            common.warningMsg('Directory already exists')
+        }
+        try {
+            salt.cmdRun(saltMaster, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
+        } catch (Exception er) {
+            common.warningMsg('Files were already moved')
+        }
+        try {
+            salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /var/lib/mysql/*")
+        } catch (Exception er) {
+            common.warningMsg('Directory already empty')
+        }
+        try {
+            salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
+        } catch (Exception er) {
+            common.warningMsg('File is not present')
+        }
+        salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
+        _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
+        backup_dir = _pillar['return'][0].values()[0]
+        if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
+        print(backup_dir)
+        salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
+        salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
+        salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
+
+        // wait until mysql service on galera master is up
+        salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
+
+        salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
+        sleep(15)
+        salt.cmdRun(saltMaster, 'I@galera:master', "su root -c 'salt-call mysql.status | grep -A1 wsrep_cluster_size'")
+
     }
 }
-
-
-