Add tempest run to MOSK BM

Related_Prod: PROD-37294

Change-Id: Idef64974c46cf5900572d96291231a1d6c1747c6
diff --git a/bm_mcc_mosk/child/kaas_workloads/osdpl.yaml b/bm_mcc_mosk/child/kaas_workloads/osdpl.yaml
index 48f7889..168f14e 100644
--- a/bm_mcc_mosk/child/kaas_workloads/osdpl.yaml
+++ b/bm_mcc_mosk/child/kaas_workloads/osdpl.yaml
@@ -68,15 +68,13 @@
           - flat
           physnet: physnet1
           vlan_ranges: null
-    services:
-      - tempest
   services:
     tempest:
       tempest:
         values:
           conf:
             script: |
-              tempest run --config-file /etc/tempest/tempest.conf --concurrency 4 --blacklist-file /etc/tempest/test-blacklist --regex test
+              cat /etc/tempest/tempest.conf; tempest run --config-file /etc/tempest/tempest.conf --concurrency 4 --blacklist-file /etc/tempest/test-blacklist --regex test || true; stestr last --subunit > /var/lib/tempest/data/tempest_report.xml
     orchestration:
       heat:
         values:
diff --git a/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml b/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml
new file mode 100644
index 0000000..a60bf83
--- /dev/null
+++ b/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1

+kind: Pod

+metadata:

+  name: tempest-test-results-pod

+  namespace: openstack

+spec:

+  nodeSelector:

+    openstack-control-plane: enabled

+  volumes:

+    - name: tempest-pvc-storage

+      persistentVolumeClaim:

+        claimName: pvc-tempest

+  containers:

+    - name: tempest-pvc-container

+      image: mirantis.azurecr.io/general/external/docker.io/library/ubuntu:bionic-20201119

+      command: ['sh', '-c', 'sleep infinity']

+      volumeMounts:

+        - mountPath: "/var/lib/tempest/data"

+          name: tempest-pvc-storage
\ No newline at end of file
diff --git a/jobs/pipelines/mosk/deploy-bm.groovy b/jobs/pipelines/mosk/deploy-bm.groovy
index 983b1e8..0dddf3d 100644
--- a/jobs/pipelines/mosk/deploy-bm.groovy
+++ b/jobs/pipelines/mosk/deploy-bm.groovy
@@ -13,7 +13,7 @@
 def kubectl_openstack_cmd = ''
 ssh_params = "-o ConnectTimeout=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
 
-timeout(time: 3, unit: 'HOURS') {
+timeout(time: 9, unit: 'HOURS') {
   timestamps {
     node ("${NODE_LABEL}") {
       checkout scm
@@ -112,6 +112,46 @@
             sh "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig apply -f coredns.patched.conf"
 
           } // stage DNS
+
+          stage("Get openstack credentials") {
+            kubectl_openstack_external_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack-external "
+            cmd = "${kubectl_openstack_external_cmd} get secrets openstack-identity-credentials -o jsonpath='{.data.clouds\\\\.yaml}' | base64 -d"
+            openstack_credentials = sh(returnStdout: true, script: "${cmd}")
+            println openstack_credentials
+            writeFile(file: "openstack_credentials.yaml", text: openstack_credentials, encoding: "UTF-8")
+          } //stage get credentials
+
+          stage("Tempest configuration and run") {
+            kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
+            sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[\\\"tempest\\\"]}}}'"
+            tempest_pod_name_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get pods -o custom-columns=POD:.metadata.name --no-headers | grep openstack-tempest-run-tests'"
+            tempest_pod_name = ''
+            while(tempest_pod_name==''){
+              try{
+                tempest_pod_name = sh(returnStdout: true, script: "${tempest_pod_name_cmd}").trim()
+              } catch(Exception e) {
+              println("Wait tempest pod start (Exception: ${e}")}
+              sleep 30
+            }
+            tempest_check_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get -o jsonpath=\'{.status.phase}\' pod/${tempest_pod_name}'"
+            status = ""
+            while (status != "Succeeded"){ //Completed
+              status = sh(returnStdout: true, script: "${tempest_check_cmd}").trim()
+              println("Tempest pod status is ${status}")
+              sleep 60
+            }
+          } //stage tempest
+          stage("Archive artifacts"){
+            sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml"
+            sleep 60
+            ssh_cmd = "ssh ${ssh_params} root@${seed_ext_ip} "
+            sh "${ssh_cmd} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack cp tempest-test-results-pod:var/lib/tempest/data/ /root/tempest_log/'"
+            sh "scp -r -v ${ssh_params} root@${seed_ext_ip}:/root/tempest_log ."
+            sh "tar -czvf tempest_log/tempest_full.tar.gz tempest_log/tempest.log"
+            archiveArtifacts artifacts: "openstack_credentials.yaml, tempest_log/*/tempest.log, tempest_log/tempest_full.tar.gz"
+            sh "${kubectl_openstack_cmd} delete pod tempest-test-results-pod"
+            sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[]}}}'"
+          }
         } //withCredentials
       } //sshagent
     } //node