Separating MOSK Deployment Stages
Related_prod: PROD-37294
Change-Id: I5581d7c6920605f778ac4f4cd4ac0d235bc65990
diff --git a/jobs/pipelines/mosk/deploy-bm.groovy b/jobs/pipelines/mosk/deploy-bm.groovy
index 0dddf3d..a2cae19 100644
--- a/jobs/pipelines/mosk/deploy-bm.groovy
+++ b/jobs/pipelines/mosk/deploy-bm.groovy
@@ -120,37 +120,18 @@
println openstack_credentials
writeFile(file: "openstack_credentials.yaml", text: openstack_credentials, encoding: "UTF-8")
} //stage get credentials
-
- stage("Tempest configuration and run") {
- kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
- sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[\\\"tempest\\\"]}}}'"
- tempest_pod_name_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get pods -o custom-columns=POD:.metadata.name --no-headers | grep openstack-tempest-run-tests'"
- tempest_pod_name = ''
- while(tempest_pod_name==''){
- try{
- tempest_pod_name = sh(returnStdout: true, script: "${tempest_pod_name_cmd}").trim()
- } catch(Exception e) {
- println("Wait tempest pod start (Exception: ${e}")}
- sleep 30
+ if (env.RUN_TESTS_AFTER_DEPLOY == "true") {
+ stage("Run tempest job"){
+ def deploy = build job: 'mosk-tempest',
+ parameters: [
+ string(name: 'NODE_LABEL', value: NODE_LABEL),
+ string(name: 'MAINTENANCE_TEAM_SSH_ID', value: MAINTENANCE_TEAM_SSH_ID),
+ string(name: 'TCP_QA_REFS', value: env.TCP_QA_REFS),
+ string(name: 'SEED_EXT_IP', value: seed_ext_ip)
+ ],
+ wait: false,
+ propagate: false
}
- tempest_check_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get -o jsonpath=\'{.status.phase}\' pod/${tempest_pod_name}'"
- status = ""
- while (status != "Succeeded"){ //Completed
- status = sh(returnStdout: true, script: "${tempest_check_cmd}").trim()
- println("Tempest pod status is ${status}")
- sleep 60
- }
- } //stage tempest
- stage("Archive artifacts"){
- sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml"
- sleep 60
- ssh_cmd = "ssh ${ssh_params} root@${seed_ext_ip} "
- sh "${ssh_cmd} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack cp tempest-test-results-pod:var/lib/tempest/data/ /root/tempest_log/'"
- sh "scp -r -v ${ssh_params} root@${seed_ext_ip}:/root/tempest_log ."
- sh "tar -czvf tempest_log/tempest_full.tar.gz tempest_log/tempest.log"
- archiveArtifacts artifacts: "openstack_credentials.yaml, tempest_log/*/tempest.log, tempest_log/tempest_full.tar.gz"
- sh "${kubectl_openstack_cmd} delete pod tempest-test-results-pod"
- sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[]}}}'"
}
} //withCredentials
} //sshagent
diff --git a/jobs/pipelines/mosk/mosk-archive-artifacts.groovy b/jobs/pipelines/mosk/mosk-archive-artifacts.groovy
new file mode 100644
index 0000000..7762945
--- /dev/null
+++ b/jobs/pipelines/mosk/mosk-archive-artifacts.groovy
@@ -0,0 +1,44 @@
+@Library('tcp-qa')_
+
+import groovy.xml.XmlUtil
+
+common = new com.mirantis.mk.Common()
+shared = new com.mirantis.system_qa.SharedPipeline()
+
+NODE_LABEL = env.NODE_LABEL ?: "sre-team-infra"
+MAINTENANCE_TEAM_SSH_ID = env.MAINTENANCE_TEAM_SSH_ID ?: 'maintenance-team-ssh'
+//IPMI_CREDS = 'lab_engineer' // base bm lab
+seed_ext_ip = env.SEED_EXT_IP ?: '172.16.180.2'
+ssh_params = "-o ConnectTimeout=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
+
+
+timeout(time: 6, unit: 'HOURS') {
+ timestamps {
+ node ("${NODE_LABEL}") {
+ checkout scm
+ shared.update_working_dir()
+ sshagent(credentials: ['maintenance-team-ssh']) {
+ withCredentials(
+ [[$class: 'SSHUserPrivateKeyBinding',
+ keyFileVariable: "MAINTENANCE_TEAM_SSH_KEY",
+ credentialsId: MAINTENANCE_TEAM_SSH_ID,
+ usernameVariable: "MAINTENANCE_TEAM_SSH_USERNAME"]]) {
+ stage("Archive artifacts"){
+ sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml"
+ sleep 60
+ ssh_cmd = "ssh ${ssh_params} root@${seed_ext_ip} "
+ sh "${ssh_cmd} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack cp tempest-test-results-pod:var/lib/tempest/data/ /root/tempest_log/'"
+ sh "scp -r -v ${ssh_params} root@${seed_ext_ip}:/root/tempest_log ."
+ sh "tar -czvf tempest_log/tempest_full.tar.gz tempest_log/tempest.log"
+ archiveArtifacts artifacts: "openstack_credentials.yaml, tempest_log/*/tempest.log, tempest_log/tempest_full.tar.gz"
+ sh "${kubectl_openstack_cmd} delete pod tempest-test-results-pod"
+ if( env.DELETE_TEMPEST_POD == true ){
+ sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[]}}}'"
+ }
+ }
+ } //withCredentials
+ } //sshagent
+ } //node
+ } //timestamps
+} //timeout
\ No newline at end of file
diff --git a/jobs/pipelines/mosk/mosk-tempest.groovy b/jobs/pipelines/mosk/mosk-tempest.groovy
new file mode 100644
index 0000000..bbce7c9
--- /dev/null
+++ b/jobs/pipelines/mosk/mosk-tempest.groovy
@@ -0,0 +1,70 @@
+@Library('tcp-qa')_
+
+import groovy.xml.XmlUtil
+
+common = new com.mirantis.mk.Common()
+shared = new com.mirantis.system_qa.SharedPipeline()
+
+NODE_LABEL = env.NODE_LABEL ?: "sre-team-infra"
+MAINTENANCE_TEAM_SSH_ID = env.MAINTENANCE_TEAM_SSH_ID ?: 'maintenance-team-ssh'
+//IPMI_CREDS = 'lab_engineer' // base bm lab
+seed_ext_ip = env.SEED_EXT_IP ?: '172.16.180.2'
+ssh_params = "-o ConnectTimeout=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
+
+timeout(time: 6, unit: 'HOURS') {
+ timestamps {
+ node ("${NODE_LABEL}") {
+ checkout scm
+ shared.update_working_dir()
+ sshagent(credentials: ['maintenance-team-ssh']) {
+ withCredentials(
+ [[$class: 'SSHUserPrivateKeyBinding',
+ keyFileVariable: "MAINTENANCE_TEAM_SSH_KEY",
+ credentialsId: MAINTENANCE_TEAM_SSH_ID,
+ usernameVariable: "MAINTENANCE_TEAM_SSH_USERNAME"]]) {
+ stage("Delete old tempest pod") {
+ cmd_get_service_list_wo_tempest = "${kubectl_openstack_cmd} get osdpl osh-dev -o jsonpath='{.spec.features.services}' | jq 'del(.[index(\\\"tempest\\\")])' "
+ try {
+ service_list_wo_tempest = sh(returnStdout: true, script: "${cmd_get_service_list_wo_tempest}").trim()
+ sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":${service_list_wo_tempest}}}}'"
+ sleep 60
+ }
+ catch(Exception e) {
+ println("Tempest not included in services (Exception: ${e}")}
+ }
+ stage("Tempest configuration and run") {
+ sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[\\\"tempest\\\"]}}}'"
+ tempest_pod_name_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get pods -o custom-columns=POD:.metadata.name --no-headers | grep openstack-tempest-run-tests'"
+ tempest_pod_name = ''
+ while(tempest_pod_name==''){
+ try{
+ tempest_pod_name = sh(returnStdout: true, script: "${tempest_pod_name_cmd}").trim()
+ } catch(Exception e) {
+ println("Wait tempest pod start (Exception: ${e}")}
+ sleep 30
+ }
+ tempest_check_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get -o jsonpath=\'{.status.phase}\' pod/${tempest_pod_name}'"
+ status = ""
+ while (status != "Succeeded"){ //Completed
+ status = sh(returnStdout: true, script: "${tempest_check_cmd}").trim()
+ println("Tempest pod status is ${status}")
+ sleep 60
+ }
+ } //stage tempest
+ stage("Run archive artifacts job") {
+ def deploy = build job: 'mosk-archive-artifacts',
+ parameters: [
+ string(name: 'NODE_LABEL', value: NODE_LABEL),
+ string(name: 'MAINTENANCE_TEAM_SSH_ID', value: MAINTENANCE_TEAM_SSH_ID),
+ string(name: 'TCP_QA_REFS', value: env.TCP_QA_REFS),
+ string(name: 'SEED_EXT_IP', value: seed_ext_ip)
+ ],
+ wait: false,
+ propagate: false
+ }
+ } //withCredentials
+ } //sshagent
+ } //node
+ } //timestamps
+} //timeout
\ No newline at end of file