Merge "Improved outputs in lab pipelines"
diff --git a/lab-pipeline.groovy b/lab-pipeline.groovy
index a6efaca..b4014b6 100644
--- a/lab-pipeline.groovy
+++ b/lab-pipeline.groovy
@@ -50,6 +50,7 @@
             // Prepare machines
             //
             stage ('Create infrastructure') {
+
                 if (STACK_TYPE == 'heat') {
                     // value defaults
                     def openstackCloud
@@ -99,7 +100,7 @@
                     saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
                     currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
 
-                    if (INSTALL.toLowerCase().contains('kvm')) {
+                    if (common.checkContains('INSTALL', 'kvm')) {
                         saltPort = 6969
                     } else {
                         saltPort = 6969
@@ -122,7 +123,7 @@
             // Install
             //
 
-            if (INSTALL.toLowerCase().contains('core')) {
+            if (common.checkContains('INSTALL', 'core')) {
                 stage('Install core infrastructure') {
                     // salt.master, reclass
                     // refresh_pillar
@@ -131,17 +132,19 @@
 
                     //orchestrate.installFoundationInfra(saltMaster)
                     salt.enforceState(saltMaster, 'I@salt:master', ['salt.master', 'reclass'], true)
+                    salt.enforceState(saltMaster, '*', ['linux.system'], true)
+                    salt.enforceState(saltMaster, '*', ['salt.minion'], true)
                     salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
                     salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
                     salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
 
 
-                    if (INSTALL.toLowerCase().contains('kvm')) {
+                    if (common.checkContains('INSTALL', 'kvm')) {
                         //orchestrate.installInfraKvm(saltMaster)
-                        salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
-                        salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
+                        //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
+                        //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
 
-                        salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
+                        //salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
                         salt.enforceState(saltMaster, 'I@salt:control', 'libvirt', true)
                         salt.enforceState(saltMaster, 'I@salt:control', 'salt.control', true)
 
@@ -167,7 +170,7 @@
             }
 
             // install k8s
-            if (INSTALL.toLowerCase().contains('k8s')) {
+            if (common.checkContains('INSTALL', 'k8s')) {
                 stage('Install Kubernetes infra') {
                     //orchestrate.installOpenstackMcpInfra(saltMaster)
 
@@ -235,7 +238,7 @@
             }
 
             // install openstack
-            if (INSTALL.toLowerCase().contains('openstack')) {
+            if (common.checkContains('INSTALL', 'openstack')) {
                 // install Infra and control, tests, ...
 
                 stage('Install OpenStack infra') {
@@ -258,14 +261,19 @@
                     salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'], null, true)
 
                     // Install rabbitmq
-                    salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true, false)
-
+                    withEnv(['ASK_ON_ERROR=false']){
+                        retry(2) {
+                            salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true)
+                        }
+                    }
                     // Check the rabbitmq status
                     salt.runSaltProcessStep(saltMaster, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
 
                     // Install galera
-                    retry(2) {
-                        salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
+                    withEnv(['ASK_ON_ERROR=false']){
+                        retry(2) {
+                            salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
+                        }
                     }
                     salt.enforceState(saltMaster, 'I@galera:slave', 'galera', true)
 
@@ -348,7 +356,7 @@
                 stage('Install OpenStack network') {
                     //orchestrate.installOpenstackMkNetwork(saltMaster, physical)
 
-                    if (INSTALL.toLowerCase().contains('contrail')) {
+                    if (common.checkContains('INSTALL', 'contrail')) {
                         // Install opencontrail database services
                         //runSaltProcessStep(saltMaster, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
                         try {
@@ -371,7 +379,7 @@
 
                         // Test opencontrail
                         salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.run', ['contrail-status'], null, true)
-                    } else if (INSTALL.toLowerCase().contains('ovs')) {
+                    } else if (common.checkContains('INSTALL', 'ovs')) {
                         // Apply gateway
                         salt.runSaltProcessStep(saltMaster, 'I@neutron:gateway', 'state.apply', [], null, true)
                     }
@@ -388,7 +396,7 @@
                         salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply', [], null, true)
                     }
 
-                    if (INSTALL.toLowerCase().contains('contrail')) {
+                    if (common.checkContains('INSTALL', 'contrail')) {
                         // Provision opencontrail control services
                         salt.enforceState(saltMaster, 'I@opencontrail:database:id:1', 'opencontrail.client', true)
                         // Provision opencontrail virtual routers
@@ -402,7 +410,7 @@
             }
 
 
-            if (INSTALL.toLowerCase().contains('stacklight')) {
+            if (common.checkContains('INSTALL', 'stacklight')) {
                 stage('Install StackLight') {
                     // infra install
                     // Install the StackLight backends
@@ -506,7 +514,7 @@
             // Test
             //
 
-            if (TEST.toLowerCase().contains('k8s')) {
+            if (common.checkContains('TEST', 'k8s')) {
                 stage('Run k8s bootstrap tests') {
                     orchestrate.runConformanceTests(saltMaster, K8S_API_SERVER, 'tomkukral/k8s-scripts')
                 }
@@ -516,7 +524,7 @@
                 }
             }
 
-            if (TEST.toLowerCase().contains('openstack')) {
+            if (common.checkContains('TEST', 'openstack')) {
                 stage('Run OpenStack tests') {
                     test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
                 }
@@ -538,14 +546,15 @@
             throw e
         } finally {
 
-            // send notification
-            common.sendNotification(currentBuild.result,HEAT_STACK_NAME,["slack"])
 
             //
             // Clean
             //
 
             if (STACK_TYPE == 'heat') {
+                // send notification
+                common.sendNotification(currentBuild.result, HEAT_STACK_NAME, ["slack"])
+
                 if (HEAT_STACK_DELETE.toBoolean() == true) {
                     common.errorMsg('Heat job cleanup triggered')
                     stage('Trigger cleanup job') {