Merge "Added parameter to kitchen test function"
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index fb8559c..5cb11db 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -37,6 +37,9 @@
  *
  * optional parameters for overwriting soft params
  *   KUBERNETES_HYPERKUBE_IMAGE  Docker repository and tag for hyperkube image
+ *   CALICO_CNI_IMAGE            Docker repository and tag for calico CNI image
+ *   CALICO_NODE_IMAGE           Docker repository and tag for calico node image
+ *   CALICOCTL_IMAGE             Docker repository and tag for calicoctl image
  *
  */
 
@@ -160,6 +163,16 @@
                     if (env.getEnvironment().containsKey("KUBERNETES_HYPERKUBE_IMAGE")) {
                         salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'file.append', overwriteFile, "    kubernetes_hyperkube_image: ${KUBERNETES_HYPERKUBE_IMAGE}")
                     }
+                    // Overwrite Calico vars if specified
+                    if (env.getEnvironment().containsKey("CALICO_CNI_IMAGE")) {
+                      salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, "    kubernetes_calico_cni_image: ${CALICO_CNI_IMAGE}")
+                    }
+                    if (env.getEnvironment().containsKey("CALICO_NODE_IMAGE")) {
+                      salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, "    kubernetes_calico_node_image: ${CALICO_NODE_IMAGE}")
+                    }
+                    if (env.getEnvironment().containsKey("CALICOCTL_IMAGE")) {
+                      salt.runSaltProcessStep(saltmaster, 'I@salt:master', 'file.append', overwriteFile, "    kubernetes_calicoctl_image: ${CALICOCTL_IMAGE}")
+                    }
 
                     orchestrate.installKubernetesControl(saltMaster)
                 }
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 9e6fd01..5d4afaa 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -26,11 +26,15 @@
         if (STAGE_TEST_UPGRADE.toBoolean() == true) {
             stage('Test upgrade') {
 
-                //salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
 
-                // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
-                // salt '*' saltutil.sync_all
-                // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
+                try {
+                    salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
+                } catch (Exception e) {
+                    common.warningMsg(" Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
+                }
+
+                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
+                salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
                 
 
                 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
@@ -46,12 +50,8 @@
 
                 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
                 def kvm01 = _pillar['return'][0].values()[0].values()[0]
-                def kvm03 = _pillar['return'][0].values()[2].values()[0]
-                def kvm02 = _pillar['return'][0].values()[1].values()[0]
                 print(_pillar)
                 print(kvm01)
-                print(kvm02)
-                print(kvm03)
 
                 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
                 def upgNodeProvider = _pillar['return'][0].values()[0]
@@ -131,7 +131,7 @@
                 try {
                     salt.enforceState(saltMaster, 'upg*', 'keystone.server')
                 } catch (Exception e) {
-                    common.warningMsg('Reloading Apache2 and enforcing keystone.server state again')
+                    common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
                     salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
                 }
                 try {
@@ -153,6 +153,13 @@
                     common.warningMsg('running nova state again')
                     salt.enforceState(saltMaster, 'upg*', 'nova')
                 }
+                // run nova state again as sometimes nova does not enforce itself for some reason
+                try {
+                    salt.enforceState(saltMaster, 'upg*', 'nova')
+                } catch (Exception e) {
+                    common.warningMsg('running nova state again')
+                    salt.enforceState(saltMaster, 'upg*', 'nova')
+                }
                 try {
                     salt.enforceState(saltMaster, 'upg*', 'cinder')
                 } catch (Exception e) {
@@ -192,12 +199,10 @@
 
                 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
                 kvm01 = _pillar['return'][0].values()[0].values()[0]
-                kvm03 = _pillar['return'][0].values()[2].values()[0]
-                kvm02 = _pillar['return'][0].values()[1].values()[0]
                 print(_pillar)
                 print(kvm01)
-                print(kvm02)
-                print(kvm03)
+
+                def errorOccured = false
 
                 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
                 def ctl01NodeProvider = _pillar['return'][0].values()[0]
@@ -293,7 +298,7 @@
                     try {
                         salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
                     } catch (Exception e) {
-                        common.warningMsg('Reloading Apache2 and enforcing keystone.server state again')
+                        common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
                         salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
                         salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
                     }
@@ -342,6 +347,7 @@
                     }
 
                 } catch (Exception e) {
+                    errorOccured = true
                     common.warningMsg('Some states that require syncdb failed. Restoring production databases')
                     databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
                     if(databases && databases != ""){
@@ -359,22 +365,22 @@
                     }
                     common.errorMsg("Stage Real control upgrade failed")
                 }
-                    
-                // salt 'cmp*' cmd.run 'service nova-compute restart'
-                salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
+                if(!errorOccured){
+                    // salt 'cmp*' cmd.run 'service nova-compute restart'
+                    salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
 
-                // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
-                // salt 'ctl*' state.sls keepalived
-                // salt 'prx*' state.sls keepalived
-                salt.enforceState(saltMaster, 'prx*', 'keepalived')
-                // salt 'prx*' state.sls horizon
-                salt.enforceState(saltMaster, 'prx*', 'horizon')
-                // salt 'prx*' state.sls nginx
-                salt.enforceState(saltMaster, 'prx*', 'nginx')
+                    // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
+                    // salt 'ctl*' state.sls keepalived
+                    // salt 'prx*' state.sls keepalived
+                    salt.enforceState(saltMaster, 'prx*', 'keepalived')
+                    // salt 'prx*' state.sls horizon
+                    salt.enforceState(saltMaster, 'prx*', 'horizon')
+                    // salt 'prx*' state.sls nginx
+                    salt.enforceState(saltMaster, 'prx*', 'nginx')
 
-                salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
+                    salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
+                }
             }
-
         }
 
 
@@ -397,12 +403,8 @@
 
                 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
                 kvm01 = _pillar['return'][0].values()[0].values()[0]
-                kvm03 = _pillar['return'][0].values()[2].values()[0]
-                kvm02 = _pillar['return'][0].values()[1].values()[0]
                 print(_pillar)
                 print(kvm01)
-                print(kvm02)
-                print(kvm03)
 
                 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
                 def ctl01NodeProvider = _pillar['return'][0].values()[0]