diff --git a/cicd-lab-pipeline.groovy b/cicd-lab-pipeline.groovy
index c935af7..df4b477 100644
--- a/cicd-lab-pipeline.groovy
+++ b/cicd-lab-pipeline.groovy
@@ -223,7 +223,7 @@
                 //
                 def adminUser
                 def authorizedKeysFile
-                def adminUserCmdOut = salt.cmdRun(saltMaster, 'I@salt:master', "[ -d /home/ubuntu ] && echo 'ubuntu user exists'")
+                def adminUserCmdOut = salt.cmdRun(saltMaster, 'I@salt:master', "[ ! -d /home/ubuntu ] || echo 'ubuntu user exists'")
                 if (adminUserCmdOut =~ /ubuntu user exists/) {
                     adminUser = "ubuntu"
                     authorizedKeysFile = "/home/ubuntu/.ssh/authorized_keys"
@@ -264,7 +264,7 @@
                 }
                 salt.enforceState(saltMaster, 'I@nginx:server', 'nginx')
 
-                def failedSvc = salt.cmdRun(saltMaster, '*', """systemctl --failed | grep -E 'loaded[ \t]+failed' && echo 'Command execution failed'""")
+                def failedSvc = salt.cmdRun(saltMaster, '*', """systemctl --failed | grep -E 'loaded[ \t]+failed' && echo 'Command execution failed' || true""")
                 print common.prettyPrint(failedSvc)
                 if (failedSvc =~ /Command execution failed/) {
                     common.errorMsg("Some services are not running. Environment may not be fully functional!")
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index ed366b8..4defcd5 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -23,28 +23,54 @@
         state('Prepare upgrade') {
             // #ignorovat no response a chyby - nene, chyby opravíme
 
-            // read domain variable
-            def _pillar = salt.getPillar(master, 'ctl01*', '_param:cluster_domain')
+
+            salt.enforceState(master, 'I@salt:master', 'reclass')
+
+            salt.runSaltProcessStep(master, '*', 'saltutil.refresh_pillar', [], null, true)
+            // salt '*' saltutil.sync_all
+            salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
+            
+
+            def _pillar = salt.getPillar(master, 'I@salt:master', 'grains.item domain')
             def domain = _pillar['return'][0].values()[0]
+            println _pillar
+            println domain
 
-            // # test sync db
-            // salt 'kvm02*' cmd.run 'virsh destroy upg01.${domain}'
-            salt.runSaltProcessStep(saltMaster, 'kvm02*', 'cmd.run', "virsh destroy upg01.${domain}", [], null, true)
+            // read backupninja variable
+            _pillar = salt.getPillar(master, 'I@backupninja:server', '_param:single_address')
+            def backupninja_backup_host = _pillar['return'][0].values()[0]
+            println _pillar
+            println backupninja_backup_host
 
-            // salt 'kvm02*' cmd.run 'virsh undefine upg01.${domain}'
-            salt.runSaltProcessStep(saltMaster, 'kvm02*', 'cmd.run', "virsh undefine upg01.${domain}", [], null, true)
+            _pillar = salt.getPillar(master, 'I@salt:control', 'grains.item id')
+            def kvm01 = _pillar['return'][0].values()[0]
+            def kvm02 = _pillar['return'][0].values()[1]
+            def kvm03 = _pillar['return'][0].values()[2]
+            println _pillar
+            println kvm01
+            println kvm02
+            println kvm03
+
+            _pillar = salt.getPillar(master, '${kvm01}', 'salt:control:cluster:internal:node:upg01:provider')
+            def upgNodeProvider = _pillar['return'][0].values()[0]
+            println _pillar
+            println upgNodeProvider
+
+
+            salt.runSaltProcessStep(master, '${upgNodeProvider}', 'virt.destroy upg01.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${upgNodeProvider}', 'virt.undefine upg01.${domain}', [], null, true)
 
             // salt-key -d upg01.${domain} -y
             salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', "salt-key -d upg01.${domain} -y", [], null, true)
             // salt 'kvm02*' state.sls salt.control
-            salt.enforceState(saltMaster, 'kvm02*', 'salt.control')
+            salt.enforceState(saltMaster, '${upgNodeProvider}', 'salt.control')
 
             sleep(60)
 
             // salt '*' saltutil.refresh_pillar
-            salt.runSaltProcessStep(master, '*', 'saltutil.refresh_pillar', [], null, true)
+            salt.runSaltProcessStep(master, 'upg*', 'saltutil.refresh_pillar', [], null, true)
             // salt '*' saltutil.sync_all
-            salt.runSaltProcessStep(master, '*', 'saltutil.sync_all', [], null, true)
+            salt.runSaltProcessStep(master, 'upg*', 'saltutil.sync_all', [], null, true)
 
             // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
             salt.enforceState(master, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
@@ -53,98 +79,93 @@
             // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
             salt.enforceState(master, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
 
-            // read backupninja variable
-            def _pillar = salt.getPillar(master, 'I@backupninja:server', '_param:single_address')
-            def backupninja_backup_host = _pillar['return'][0].values()[0]
-
             // salt "upg*" state.sls rabbitmq
             salt.enforceState(master, 'upg*', 'rabbitmq')
             // salt "upg*" state.sls memcached
             salt.enforceState(master, 'upg*', 'memcached')
+            salt.enforceState(master, 'I@backupninja:client', 'openssh.client')
             // salt -C 'I@backupninja:server' state.sls backupninja
             salt.enforceState(master, 'I@backupninja:server', 'backupninja')
             // salt -C 'I@backupninja:client' state.sls backupninja
             salt.enforceState(master, 'I@backupninja:client', 'backupninja')
-            // salt -C 'I@backupninja:client' cmd.run 'ssh-keygen -f "/root/.ssh/known_hosts" -R ${backupninja_backup_host}'
-            salt.runSaltProcessStep(master, 'I@backupninja:client', 'cmd.run', "ssh-keygen -f /root/.ssh/known_hosts -R ${backupninja_backup_host}", [], null, true)
-            // salt -C 'I@backupninja:client' cmd.run 'ssh-keyscan -H ${backupninja_backup_host} >> /root/.ssh/known_hosts'
-            salt.runSaltProcessStep(master, 'I@backupninja:client', 'cmd.run', "ssh-keyscan -H ${backupninja_backup_host} >> /root/.ssh/known_hosts", [], null, true)
-            // salt -C 'I@backupninja:client' cmd.run 'backupninja -n --run /etc/backup.d/101.mysql'
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'ssh.rm_known_host root ${backupninja_backup_host}', [], null, true)
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'cmd.run', "arp -d ${backupninja_backup_host}", [], null, true)
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'ssh.set_known_host root ${backupninja_backup_host}', [], null, true)
             salt.runSaltProcessStep(master, 'I@backupninja:client', 'cmd.run', "backupninja -n --run /etc/backup.d/101.mysql", [], null, true)
-            // salt -C 'I@backupninja:client' cmd.run 'backupninja -n --run /etc/backup.d/200.backup.rsync'
             salt.runSaltProcessStep(master, 'I@backupninja:client', 'cmd.run', "backupninja -n --run /etc/backup.d/200.backup.rsync", [], null, true)
-            // salt -C 'I@mysql:client' state.sls mysql.client
+            
+
+            def databases = salt.runSaltProcessStep(master, 'I@mysql:client', 'mysql.db_list | grep upgrade | awk "/-/ {print $2}"', [], null, true)
+            for (String database : databases) { System.out.println(database) }
+            for (String database : databases) { salt.runSaltProcessStep(master, 'I@mysql:client', 'mysql.db_remove ${database}', [], null, true) }
+            for (String database : databases) { salt.runSaltProcessStep(master, 'I@mysql:client', 'file.remove /root/mysql/flags/${database}-installed', [], null, true) }
+
+
             salt.enforceState(master, 'I@mysql:client', 'mysql.client')
 
-            // salt "upg*" state.sls keystone.server
-            // salt "upg*" state.sls keystone.client.service
-            salt.enforceState(master, 'upg*', ['keystone.server', 'keystone.client.service'])
-            // salt 'upg*' state.sls glance
-            // salt "upg*" state.sls keystone.server
-            // salt 'upg*' state.sls nova
-            // salt 'upg*' state.sls cinder
-            // salt 'upg*' state.sls neutron
-            // salt 'upg*' state.sls heat
+            salt.enforceState(master, 'upg*', ['keystone.server', 'keystone.client'])
+
             salt.enforceState(master, 'upg*', ['glance', 'keystone.server', 'nova', 'cinder', 'neutron', 'heat'])
 
-            // salt 'upg01*' cmd.run ". /root/keystonercv3; openstack service list"
-            // salt 'upg01*' cmd.run ". /root/keystonercv3; openstack image list"
-            // salt 'upg01*' cmd.run ". /root/keystonercv3; openstack flavor list"
-            // salt 'upg01*' cmd.run ". /root/keystonercv3; openstack compute service list"
-            // salt 'upg01*' cmd.run ". /root/keystonercv3; openstack server list"
-            // salt 'upg01*' cmd.run ". /root/keystonercv3; openstack network list"
-            // salt 'upg01*' cmd.run ". /root/keystonercv3; openstack orchestration service list"
-            salt.runSaltProcessStep(master, 'upg01*', 'cmd.run', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack orchestration service list', null, true)
+            salt.runSaltProcessStep(master, 'upg01*', 'cmd.run', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list', null, true)
         }
 
         stage('Ask for manual confirmation') {
-            input message: "Do you want to continue with update?"
+            input message: "Do you want to continue with upgrade?"
         }
-
         state('Start upgrade') {
             // # actual upgrade
-            // salt 'kvm01*' cmd.run 'virsh destroy prx01.${domain}'
-            salt.runSaltProcessStep(master, 'kvm01*', 'cmd.run', "virsh destroy prx01.${domain}", null, true)
-            // salt 'kvm03*' cmd.run 'virsh destroy prx02.${domain}'
-            salt.runSaltProcessStep(master, 'kvm03*', 'cmd.run', "virsh destroy prx02.${domain}", null, true)
-            // salt 'kvm01*' cmd.run 'virsh destroy ctl01.${domain}'
-            salt.runSaltProcessStep(master, 'kvm01*', 'cmd.run', "virsh destroy ctl01.${domain}", null, true)
-            // salt 'kvm02*' cmd.run 'virsh destroy ctl02.${domain}'
-            salt.runSaltProcessStep(master, 'kvm02*', 'cmd.run', "virsh destroy ctl02.${domain}", null, true)
-            // salt 'kvm03*' cmd.run 'virsh destroy ctl03.${domain}'
-            salt.runSaltProcessStep(master, 'kvm03*', 'cmd.run', "virsh destroy ctl03.${domain}", null, true)
+
+            _pillar = salt.getPillar(master, '${kvm01}', 'salt:control:cluster:internal:node:ctl01:provider')
+            def ctl01NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(master, '${kvm01}', 'salt:control:cluster:internal:node:ctl02:provider')
+            def ctl02NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(master, '${kvm01}', 'salt:control:cluster:internal:node:ctl03:provider')
+            def ctl03NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(master, '${kvm01}', 'salt:control:cluster:internal:node:prx01:provider')
+            def prx01NodeProvider = _pillar['return'][0].values()[0]
+
+            _pillar = salt.getPillar(master, '${kvm01}', 'salt:control:cluster:internal:node:prx02:provider')
+            def prx02NodeProvider = _pillar['return'][0].values()[0]
+
+
+            salt.runSaltProcessStep(master, '${prx01NodeProvider}', 'virt.destroy prx01.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${prx02NodeProvider}', 'virt.destroy prx01.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${ctl01NodeProvider}', 'virt.destroy ctl01.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${ctl02NodeProvider}', 'virt.destroy ctl02.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${ctl03NodeProvider}', 'virt.destroy ctl03.${domain}', [], null, true)
+
 
             // salt 'kvm01*' cmd.run '[ ! -f ./prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak'
-            salt.runSaltProcessStep(master, 'kvm01*', 'cmd.run', "[ ! -f ./prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak", null, true)
+            salt.runSaltProcessStep(master, '${prx01NodeProvider}', 'cmd.run', "[ ! -f ./prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak", null, true)
             // salt 'kvm03*' cmd.run '[ ! -f ./prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak'
-            salt.runSaltProcessStep(master, 'kvm03*', 'cmd.run', "[ ! -f ./prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak", null, true)
+            salt.runSaltProcessStep(master, '${prx02NodeProvider}', 'cmd.run', "[ ! -f ./prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak", null, true)
             // salt 'kvm01*' cmd.run '[ ! -f ./ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak'
-            salt.runSaltProcessStep(master, 'kvm01*', 'cmd.run', "[ ! -f ./ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak", null, true)
+            salt.runSaltProcessStep(master, '${ctl01NodeProvider}', 'cmd.run', "[ ! -f ./ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak", null, true)
             // salt 'kvm02*' cmd.run '[ ! -f ./ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak'
-            salt.runSaltProcessStep(master, 'kvm02*', 'cmd.run', "[ ! -f ./ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak", null, true)
+            salt.runSaltProcessStep(master, '${ctl02NodeProvider}', 'cmd.run', "[ ! -f ./ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak", null, true)
             // salt 'kvm03*' cmd.run '[ ! -f ./ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak'
-            salt.runSaltProcessStep(master, 'kvm03*', 'cmd.run', "[ ! -f ./ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak", null, true)
+            salt.runSaltProcessStep(master, '${ctl03NodeProvider}', 'cmd.run', "[ ! -f ./ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak", null, true)
 
-            // salt 'kvm01*' cmd.run 'virsh undefine prx01.${domain}'
-            salt.runSaltProcessStep(master, 'kvm01*', 'cmd.run', "virsh undefine prx01.${domain}", null, true)
-            // salt 'kvm03*' cmd.run 'virsh undefine prx02.${domain}'
-            salt.runSaltProcessStep(master, 'kvm03*', 'cmd.run', "virsh undefine prx02.${domain}", null, true)
-            // salt 'kvm01*' cmd.run 'virsh undefine ctl01.${domain}'
-            salt.runSaltProcessStep(master, 'kvm01*', 'cmd.run', "virsh undefine ctl01.${domain}", null, true)
-            // salt 'kvm02*' cmd.run 'virsh undefine ctl02.${domain}'
-            salt.runSaltProcessStep(master, 'kvm02*', 'cmd.run', "virsh undefine ctl02.${domain}", null, true)
-            // salt 'kvm03*' cmd.run 'virsh undefine ctl03.${domain}'
-            salt.runSaltProcessStep(master, 'kvm03*', 'cmd.run', "virsh undefine ctl03.${domain}", null, true)
 
-            // salt-key -d ctl01.${domain} -y
-            // salt-key -d ctl02.${domain} -y
-            // salt-key -d ctl03.${domain} -y
-            // salt-key -d prx01.${domain} -y
-            // salt-key -d prx02.${domain} -y
+            salt.runSaltProcessStep(master, '${prx01NodeProvider}', 'virt.undefine prx01.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${prx02NodeProvider}', 'virt.undefine prx02.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${ctl01NodeProvider}', 'virt.undefine ctl01.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${ctl02NodeProvider}', 'virt.undefine ctl02.${domain}', [], null, true)
+            salt.runSaltProcessStep(master, '${ctl03NodeProvider}', 'virt.undefine ctl03.${domain}', [], null, true)
+
+
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'cmd.run', "backupninja -n --run /etc/backup.d/101.mysql", [], null, true)
+            salt.runSaltProcessStep(master, 'I@backupninja:client', 'cmd.run', "backupninja -n --run /etc/backup.d/200.backup.rsync", [], null, true)
+            
+
             salt.runSaltProcessStep(master, 'I@salt', 'cmd.run', "salt-key -d ctl01.${domain},ctl01.${domain},ctl03.${domain},prx01.${domain},prx02.${domain}", null, true)
 
             // salt 'kvm*' state.sls salt.control
-            salt.enforceState(master, 'kvm*', 'salt.control')
+            salt.enforceState(master, 'I@salt:control', 'salt.control')
 
             sleep(60)
 
@@ -166,7 +187,7 @@
             // salt "ctl*" state.sls keystone.server
             salt.enforceState(master, 'ctl*', ['memcached', 'keystone.server'])
             // salt 'ctl01*' state.sls keystone.client
-            salt.enforceState(master, 'ctl01*', ['keystone.client'])
+            salt.enforceState(master, 'I@keystone:client and ctl*', 'keystone.client')
             // salt 'ctl*' state.sls glance
             salt.enforceState(master, 'ctl*', 'glance')
             // salt 'ctl*' state.sls glusterfs.client
@@ -194,15 +215,11 @@
             // salt 'prx*' state.sls nginx
             salt.enforceState(master, 'prx*', 'nginx')
 
-            // salt 'ctl01*' cmd.run ". /root/keystonercv3; openstack service list"
-            // salt 'ctl01*' cmd.run ". /root/keystonercv3; openstack image list"
-            // salt 'ctl01*' cmd.run ". /root/keystonercv3; openstack flavor list"
-            // salt 'ctl01*' cmd.run ". /root/keystonercv3; openstack compute service list"
-            // salt 'ctl01*' cmd.run ". /root/keystonercv3; openstack server list"
-            // salt 'ctl01*' cmd.run ". /root/keystonercv3; openstack network list"
-            // salt 'ctl01*' cmd.run ". /root/keystonercv3; openstack orchestration service list"
-            salt.runSaltProcessStep(master, 'upg01*', 'cmd.run', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack orchestration service list', null, true)
+            salt.runSaltProcessStep(master, 'ctl01*', 'cmd.run', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list', null, true)
         }
 
+        stage('Verification') {
+            input message: "Please verify the control upgrade and if was not successful, in the worst scenario, you can use the openstack-control-upgrade-rollover pipeline"
+        }
     }
 }
diff --git a/test-salt-models-pipeline.groovy b/test-salt-models-pipeline.groovy
index fed6ed5..622b86a 100644
--- a/test-salt-models-pipeline.groovy
+++ b/test-salt-models-pipeline.groovy
@@ -28,7 +28,6 @@
 }
 def checkouted = false
 def merged = false
-
 node("python&&docker") {
   try{
     stage("checkout") {
@@ -62,20 +61,18 @@
     }
 
     stage("test-nodes") {
-      dir ('nodes') {
-        def nodes = sh script: "find ./ -type f -name 'cfg*.yml'", returnStdout: true
-        def buildSteps = [:]
-        def partitions = common.partitionList(nodes.tokenize(), 3)
-        for (int i=0; i< partitions.size();i++) {
-          def partition = partitions[i]
-          buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
-          for(int k=0; k < partition.size;k++){
-              def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
-              buildSteps.get("partition-${i}").put(basename, { setupAndTestNode(basename) })
-          }
+      def nodes = sh script: "find ./nodes -type f -name 'cfg*.yml'", returnStdout: true
+      def buildSteps = [:]
+      def partitions = common.partitionList(nodes.tokenize(), 3)
+      for (int i=0; i< partitions.size();i++) {
+        def partition = partitions[i]
+        buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
+        for(int k=0; k < partition.size;k++){
+            def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
+            buildSteps.get("partition-${i}").put(basename, { setupAndTestNode(basename) })
         }
-        common.serial(buildSteps)
       }
+      common.serial(buildSteps)
     }
 
   } catch (Throwable e) {