Separating MOSK Deployment Stages
Related_prod: PROD-37294
Change-Id: I5581d7c6920605f778ac4f4cd4ac0d235bc65990
diff --git a/jobs/pipelines/mosk/deploy-bm.groovy b/jobs/pipelines/mosk/deploy-bm.groovy
index 0dddf3d..a2cae19 100644
--- a/jobs/pipelines/mosk/deploy-bm.groovy
+++ b/jobs/pipelines/mosk/deploy-bm.groovy
@@ -120,37 +120,18 @@
println openstack_credentials
writeFile(file: "openstack_credentials.yaml", text: openstack_credentials, encoding: "UTF-8")
} //stage get credentials
-
- stage("Tempest configuration and run") {
- kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
- sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[\\\"tempest\\\"]}}}'"
- tempest_pod_name_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get pods -o custom-columns=POD:.metadata.name --no-headers | grep openstack-tempest-run-tests'"
- tempest_pod_name = ''
- while(tempest_pod_name==''){
- try{
- tempest_pod_name = sh(returnStdout: true, script: "${tempest_pod_name_cmd}").trim()
- } catch(Exception e) {
- println("Wait tempest pod start (Exception: ${e}")}
- sleep 30
+ if (env.RUN_TESTS_AFTER_DEPLOY == "true") {
+ stage("Run tempest job"){
+ def deploy = build job: 'mosk-tempest',
+ parameters: [
+ string(name: 'NODE_LABEL', value: NODE_LABEL),
+ string(name: 'MAINTENANCE_TEAM_SSH_ID', value: MAINTENANCE_TEAM_SSH_ID),
+ string(name: 'TCP_QA_REFS', value: env.TCP_QA_REFS),
+ string(name: 'SEED_EXT_IP', value: seed_ext_ip)
+ ],
+ wait: false,
+ propagate: false
}
- tempest_check_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get -o jsonpath=\'{.status.phase}\' pod/${tempest_pod_name}'"
- status = ""
- while (status != "Succeeded"){ //Completed
- status = sh(returnStdout: true, script: "${tempest_check_cmd}").trim()
- println("Tempest pod status is ${status}")
- sleep 60
- }
- } //stage tempest
- stage("Archive artifacts"){
- sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml"
- sleep 60
- ssh_cmd = "ssh ${ssh_params} root@${seed_ext_ip} "
- sh "${ssh_cmd} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack cp tempest-test-results-pod:var/lib/tempest/data/ /root/tempest_log/'"
- sh "scp -r -v ${ssh_params} root@${seed_ext_ip}:/root/tempest_log ."
- sh "tar -czvf tempest_log/tempest_full.tar.gz tempest_log/tempest.log"
- archiveArtifacts artifacts: "openstack_credentials.yaml, tempest_log/*/tempest.log, tempest_log/tempest_full.tar.gz"
- sh "${kubectl_openstack_cmd} delete pod tempest-test-results-pod"
- sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[]}}}'"
}
} //withCredentials
} //sshagent
diff --git a/jobs/pipelines/mosk/mosk-archive-artifacts.groovy b/jobs/pipelines/mosk/mosk-archive-artifacts.groovy
new file mode 100644
index 0000000..7762945
--- /dev/null
+++ b/jobs/pipelines/mosk/mosk-archive-artifacts.groovy
@@ -0,0 +1,44 @@
+@Library('tcp-qa')_
+
+import groovy.xml.XmlUtil
+
+common = new com.mirantis.mk.Common()
+shared = new com.mirantis.system_qa.SharedPipeline()
+
+NODE_LABEL = env.NODE_LABEL ?: "sre-team-infra"
+MAINTENANCE_TEAM_SSH_ID = env.MAINTENANCE_TEAM_SSH_ID ?: 'maintenance-team-ssh'
+//IPMI_CREDS = 'lab_engineer' // base bm lab
+seed_ext_ip = env.SEED_EXT_IP ?: '172.16.180.2'
+ssh_params = "-o ConnectTimeout=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
+
+
+timeout(time: 6, unit: 'HOURS') {
+ timestamps {
+ node ("${NODE_LABEL}") {
+ checkout scm
+ shared.update_working_dir()
+ sshagent(credentials: ['maintenance-team-ssh']) {
+ withCredentials(
+ [[$class: 'SSHUserPrivateKeyBinding',
+ keyFileVariable: "MAINTENANCE_TEAM_SSH_KEY",
+ credentialsId: MAINTENANCE_TEAM_SSH_ID,
+ usernameVariable: "MAINTENANCE_TEAM_SSH_USERNAME"]]) {
+ stage("Archive artifacts"){
+ sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml"
+ sleep 60
+ ssh_cmd = "ssh ${ssh_params} root@${seed_ext_ip} "
+ sh "${ssh_cmd} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack cp tempest-test-results-pod:var/lib/tempest/data/ /root/tempest_log/'"
+ sh "scp -r -v ${ssh_params} root@${seed_ext_ip}:/root/tempest_log ."
+ sh "tar -czvf tempest_log/tempest_full.tar.gz tempest_log/tempest.log"
+ archiveArtifacts artifacts: "openstack_credentials.yaml, tempest_log/*/tempest.log, tempest_log/tempest_full.tar.gz"
+ sh "${kubectl_openstack_cmd} delete pod tempest-test-results-pod"
+ if( env.DELETE_TEMPEST_POD == true ){
+ sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[]}}}'"
+ }
+ }
+ } //withCredentials
+ } //sshagent
+ } //node
+ } //timestamps
+} //timeout
\ No newline at end of file
diff --git a/jobs/pipelines/mosk/mosk-tempest.groovy b/jobs/pipelines/mosk/mosk-tempest.groovy
new file mode 100644
index 0000000..bbce7c9
--- /dev/null
+++ b/jobs/pipelines/mosk/mosk-tempest.groovy
@@ -0,0 +1,70 @@
+@Library('tcp-qa')_
+
+import groovy.xml.XmlUtil
+
+common = new com.mirantis.mk.Common()
+shared = new com.mirantis.system_qa.SharedPipeline()
+
+NODE_LABEL = env.NODE_LABEL ?: "sre-team-infra"
+MAINTENANCE_TEAM_SSH_ID = env.MAINTENANCE_TEAM_SSH_ID ?: 'maintenance-team-ssh'
+//IPMI_CREDS = 'lab_engineer' // base bm lab
+seed_ext_ip = env.SEED_EXT_IP ?: '172.16.180.2'
+ssh_params = "-o ConnectTimeout=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
+
+timeout(time: 6, unit: 'HOURS') {
+ timestamps {
+ node ("${NODE_LABEL}") {
+ checkout scm
+ shared.update_working_dir()
+ sshagent(credentials: ['maintenance-team-ssh']) {
+ withCredentials(
+ [[$class: 'SSHUserPrivateKeyBinding',
+ keyFileVariable: "MAINTENANCE_TEAM_SSH_KEY",
+ credentialsId: MAINTENANCE_TEAM_SSH_ID,
+ usernameVariable: "MAINTENANCE_TEAM_SSH_USERNAME"]]) {
+ stage("Delete old tempest pod") {
+ cmd_get_service_list_wo_tempest = "${kubectl_openstack_cmd} get osdpl osh-dev -o jsonpath='{.spec.features.services}' | jq 'del(.[index(\\\"tempest\\\")])' "
+ try {
+ service_list_wo_tempest = sh(returnStdout: true, script: "${cmd_get_service_list_wo_tempest}").trim()
+ sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":${service_list_wo_tempest}}}}'"
+ sleep 60
+ }
+ catch(Exception e) {
+ println("Tempest not included in services (Exception: ${e}")}
+ }
+ stage("Tempest configuration and run") {
+ sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[\\\"tempest\\\"]}}}'"
+ tempest_pod_name_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get pods -o custom-columns=POD:.metadata.name --no-headers | grep openstack-tempest-run-tests'"
+ tempest_pod_name = ''
+ while(tempest_pod_name==''){
+ try{
+ tempest_pod_name = sh(returnStdout: true, script: "${tempest_pod_name_cmd}").trim()
+ } catch(Exception e) {
+ println("Wait tempest pod start (Exception: ${e}")}
+ sleep 30
+ }
+ tempest_check_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get -o jsonpath=\'{.status.phase}\' pod/${tempest_pod_name}'"
+ status = ""
+ while (status != "Succeeded"){ //Completed
+ status = sh(returnStdout: true, script: "${tempest_check_cmd}").trim()
+ println("Tempest pod status is ${status}")
+ sleep 60
+ }
+ } //stage tempest
+ stage("Run archive artifacts job") {
+ def deploy = build job: 'mosk-archive-artifacts',
+ parameters: [
+ string(name: 'NODE_LABEL', value: NODE_LABEL),
+ string(name: 'MAINTENANCE_TEAM_SSH_ID', value: MAINTENANCE_TEAM_SSH_ID),
+ string(name: 'TCP_QA_REFS', value: env.TCP_QA_REFS),
+ string(name: 'SEED_EXT_IP', value: seed_ext_ip)
+ ],
+ wait: false,
+ propagate: false
+ }
+ } //withCredentials
+ } //sshagent
+ } //node
+ } //timestamps
+} //timeout
\ No newline at end of file
diff --git a/jobs/project.yaml b/jobs/project.yaml
index 27a7d3b..956edf7 100644
--- a/jobs/project.yaml
+++ b/jobs/project.yaml
@@ -60,6 +60,8 @@
- swarm-testrail-report-jjb
- swarm-bootstrap-salt-cluster-heat-jjb
- swarm-create-cfg-config-drive-jjb
+ - mosk-tempest
+ - mosk-archive-artifacts
# - mark-reports - can't moved to JJB, a lot of parameters is not supported like Inject environment variables
# - diff_params_reclass_system - can't moved to JJB, unsupported parameters
# - diff_params_reclass_system - can't moved to JJB, unsupported parameters
diff --git a/jobs/templates/bm-mcc-mosk-base.yml b/jobs/templates/bm-mcc-mosk-base.yml
index ad6b8eb..90b8980 100644
--- a/jobs/templates/bm-mcc-mosk-base.yml
+++ b/jobs/templates/bm-mcc-mosk-base.yml
@@ -49,6 +49,10 @@
Add env vars for deploy and testing
name: EXTRA_VARS
trim: 'false'
+ - bool:
+ default: true
+ description: Run tempest after deploy
+ name: RUN_TESTS_AFTER_DEPLOY
pipeline-scm:
lightweight-checkout: false
scm:
diff --git a/jobs/templates/mosk-archive-artifacts.yml b/jobs/templates/mosk-archive-artifacts.yml
new file mode 100644
index 0000000..c9ea60c
--- /dev/null
+++ b/jobs/templates/mosk-archive-artifacts.yml
@@ -0,0 +1,51 @@
+- job-template:
+ project-type: pipeline
+ description: '{job-description}'
+ concurrent: true
+ disabled: false
+ name: mosk-archive-artifacts
+ parameters:
+ - string:
+ default: "sre-team-infra"
+ description: ''
+ name: NODE_LABEL
+ trim: 'false'
+ - string:
+ name: SEED_EXT_IP
+ description: ''
+ trim: 'false'
+ default: '172.16.180.2'
+ - string:
+ default: bm-mcc-mosk
+ description: ''
+ name: ENV_NAME
+ trim: 'false'
+ - string:
+ default: ''
+ description: |-
+ Example: refs/changes/89/411189/36
+ (for now - only one reference allowed)
+ name: TCP_QA_REFS
+ trim: 'false'
+ - text:
+ default: ''
+ description: |-
+ Add env vars for deploy and testing
+ name: EXTRA_VARS
+ trim: 'false'
+ - bool:
+ default: false
+ description: Delete tempest pod
+ name: DELETE_TEMPEST_POD
+ trim: 'false'
+ pipeline-scm:
+ lightweight-checkout: false
+ scm:
+ - git:
+ branches:
+ - FETCH_HEAD
+ refspec: ${{TCP_QA_REFS}}
+ url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa
+ script-path: jobs/pipelines/mosk/mosk-archive-artifacts.groovy
+ logrotate:
+ daysToKeep: 365
diff --git a/jobs/templates/mosk-tempest.yml b/jobs/templates/mosk-tempest.yml
new file mode 100644
index 0000000..5c2d762
--- /dev/null
+++ b/jobs/templates/mosk-tempest.yml
@@ -0,0 +1,46 @@
+- job-template:
+ project-type: pipeline
+ description: '{job-description}'
+ concurrent: true
+ disabled: false
+ name: mosk-tempest
+ parameters:
+ - string:
+ default: "sre-team-infra"
+ description: ''
+ name: NODE_LABEL
+ trim: 'false'
+ - string:
+ name: SEED_EXT_IP
+ description: ''
+ trim: 'false'
+ default: '172.16.180.2'
+ - string:
+ default: bm-mcc-mosk
+ description: ''
+ name: ENV_NAME
+ trim: 'false'
+ - string:
+ default: ''
+ description: |-
+ Example: refs/changes/89/411189/36
+ (for now - only one reference allowed)
+ name: TCP_QA_REFS
+ trim: 'false'
+ - text:
+ default: ''
+ description: |-
+ Add env vars for deploy and testing
+ name: EXTRA_VARS
+ trim: 'false'
+ pipeline-scm:
+ lightweight-checkout: false
+ scm:
+ - git:
+ branches:
+ - FETCH_HEAD
+ refspec: ${{TCP_QA_REFS}}
+ url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa
+ script-path: jobs/pipelines/mosk/mosk-tempest.groovy
+ logrotate:
+ daysToKeep: 365
diff --git a/jobs/view.yaml b/jobs/view.yaml
index 1397b1c..8128246 100644
--- a/jobs/view.yaml
+++ b/jobs/view.yaml
@@ -91,7 +91,6 @@
filter-executors: true
filter-queue: true
job-name:
- - bm-mcc-mosk-base
- deploy_bm
- bm-cicd-pike-ovs-maas
- bm-cicd-queens-ovs-maas
@@ -191,3 +190,22 @@
- last-failure
- last-duration
- build-button
+- view:
+ name: MOSK
+ view-type: list
+ description: |
+ Managed by JJB
+ filter-executors: true
+ filter-queue: true
+ job-name:
+ - bm-mcc-mosk-base
+ - mosk-tempest
+ - mosk-archive-artifacts
+ columns:
+ - status
+ - weather
+ - job
+ - last-success
+ - last-failure
+ - last-duration
+ - build-button