ctl upg pipeline fixes

Change-Id: I099734eec1ee31a0de97f3c92bf13e2d1e2e7b55
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index ff6016a..fe39b1b 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -132,10 +132,15 @@
                 try {
                     salt.enforceState(saltMaster, 'upg*', 'keystone.server')
                 } catch (Exception e) {
-                    salt.runSaltProcessStep(saltMaster, 'upg*', 'service.reload', ['apache2'], null, true)
-                    common.warningMsg('reload of apache2. We should continue to run')
+                    common.warningMsg('Enforcing keystone.server state again')
+                    salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
                 }
-                salt.enforceState(saltMaster, 'upg*', 'keystone.client')
+                try {
+                    salt.enforceState(saltMaster, 'upg*', 'keystone.client')
+                } catch (Exception e) {
+                    common.warningMsg('running keystone.client state again')
+                    salt.enforceState(saltMaster, 'upg*', 'keystone.client')
+                }
                 try {
                     salt.enforceState(saltMaster, 'upg*', 'glance')
                 } catch (Exception e) {
@@ -177,7 +182,6 @@
             }
         }
 
-
         if (STAGE_REAL_UPGRADE.toBoolean() == true) {
             stage('Real upgrade') {
                 // # actual upgrade
@@ -255,7 +259,6 @@
 
                 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
                 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
-                
 
                 try {
                     salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
@@ -288,50 +291,75 @@
                 // salt "ctl*" state.sls memcached
                 // salt "ctl*" state.sls keystone.server
                 try {
-                    salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
+                    try {
+                        salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
+                    } catch (Exception e) {
+                        common.warningMsg('Enforcing keystone.server state again')
+                        salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
+                    }
+                    // salt 'ctl01*' state.sls keystone.client
+                    try {
+                        salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
+                    } catch (Exception e) {
+                        common.warningMsg('running keystone.client state again')
+                        salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
+                    } 
+                    try {
+                        salt.enforceState(saltMaster, 'ctl*', 'glance')
+                    } catch (Exception e) {
+                        common.warningMsg('running glance state again')
+                        salt.enforceState(saltMaster, 'ctl*', 'glance')
+                    }                // salt 'ctl*' state.sls glusterfs.client
+                    salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
+                    // salt 'ctl*' state.sls keystone.server
+                    salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
+                    // salt 'ctl*' state.sls nova
+                    try {
+                        salt.enforceState(saltMaster, 'ctl*', 'nova')
+                    } catch (Exception e) {
+                        common.warningMsg('running nova state again')
+                        salt.enforceState(saltMaster, 'ctl*', 'nova')
+                    }
+                    // salt 'ctl*' state.sls cinder
+                    try {
+                        salt.enforceState(saltMaster, 'ctl*', 'cinder')
+                    } catch (Exception e) {
+                        common.warningMsg('running cinder state again')
+                        salt.enforceState(saltMaster, 'ctl*', 'cinder')
+                    }                
+                    try {
+                        salt.enforceState(saltMaster, 'ctl*', 'neutron')
+                    } catch (Exception e) {
+                        common.warningMsg('running neutron state again')
+                        salt.enforceState(saltMaster, 'ctl*', 'neutron')
+                    }
+                    // salt 'ctl*' state.sls heat
+                    try {
+                        salt.enforceState(saltMaster, 'ctl*', 'heat')
+                    } catch (Exception e) {
+                        common.warningMsg('running heat state again')
+                        salt.enforceState(saltMaster, 'ctl*', 'heat')
+                    }
+
                 } catch (Exception e) {
-                    salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.reload', ['apache2'], null, true)
-                    common.warningMsg('reload of apache2. We should continue to run')
+                    common.warningMsg('Some states that require syncdb failed. Restoring production databases')
+                    databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
+                    if(databases && databases != ""){
+                        databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
+                        for( i = 0; i < databasesList.size(); i++){ 
+                            if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
+                                salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
+                                common.warningMsg("removing database ${databasesList[i]}")
+                                salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
+                            }
+                        }
+                    }else{
+                        common.errorMsg("No none _upgrade databases were returned")
+                    }
+
+                    salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
                 }
-                // salt 'ctl01*' state.sls keystone.client
-                salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
-                // salt 'ctl*' state.sls glance
-                try {
-                    salt.enforceState(saltMaster, 'ctl*', 'glance')
-                } catch (Exception e) {
-                    common.warningMsg('running glance state again')
-                    salt.enforceState(saltMaster, 'ctl*', 'glance')
-                }                // salt 'ctl*' state.sls glusterfs.client
-                salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
-                // salt 'ctl*' state.sls keystone.server
-                salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
-                // salt 'ctl*' state.sls nova
-                try {
-                    salt.enforceState(saltMaster, 'ctl*', 'nova')
-                } catch (Exception e) {
-                    common.warningMsg('running nova state again')
-                    salt.enforceState(saltMaster, 'ctl*', 'nova')
-                }
-                // salt 'ctl*' state.sls cinder
-                try {
-                    salt.enforceState(saltMaster, 'ctl*', 'cinder')
-                } catch (Exception e) {
-                    common.warningMsg('running cinder state again')
-                    salt.enforceState(saltMaster, 'ctl*', 'cinder')
-                }                
-                try {
-                    salt.enforceState(saltMaster, 'ctl*', 'neutron')
-                } catch (Exception e) {
-                    common.warningMsg('running neutron state again')
-                    salt.enforceState(saltMaster, 'ctl*', 'neutron')
-                }
-                // salt 'ctl*' state.sls heat
-                try {
-                    salt.enforceState(saltMaster, 'ctl*', 'heat')
-                } catch (Exception e) {
-                    common.warningMsg('running heat state again')
-                    salt.enforceState(saltMaster, 'ctl*', 'heat')
-                }
+                    
                 // salt 'cmp*' cmd.run 'service nova-compute restart'
                 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)