jobs/pipelines/mosk/deploy-bm.groovy

@Library('tcp-qa')_

import groovy.xml.XmlUtil

common = new com.mirantis.mk.Common()
shared = new com.mirantis.system_qa.SharedPipeline()

NODE_LABEL = "sre-team-infra"
// ENV_NAME = "bm-mcc-mosk"
MAINTENANCE_TEAM_SSH_ID = 'maintenance-team-ssh'
IPMI_CREDS = 'lab_engineer' // base bm lab
def seed_ext_ip = '172.16.180.2'
def kubectl_openstack_cmd = ''
ssh_params = "-o ConnectTimeout=20 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"

timeout(time: 9, unit: 'HOURS') {
  timestamps {
    node ("${NODE_LABEL}") {
      checkout scm
      shared.update_working_dir()
      withCredentials(
        [[$class          : 'UsernamePasswordMultiBinding',
          credentialsId   : env.OS_CREDENTIALS,
          passwordVariable: 'OS_PASSWORD',
          usernameVariable: 'OS_USERNAME'
        ]]) {
        env.OS_IDENTITY_API_VERSION = 3
        stage("Pre-cleanup: erase BM labs") {
        // TODO: wipe all bm labs, including MCP1
          println "Remove heat stack '${ENV_NAME}'"
          shared.run_cmd("""\
            openstack stack delete -y ${ENV_NAME} || true
            timeout 20m /bin/bash -c "while openstack stack show ${ENV_NAME} -f value -c stack_status; do sleep 10; done"
            """)
          withCredentials([
            [$class          : 'UsernamePasswordMultiBinding',
            credentialsId   : "${IPMI_CREDS}",
            passwordVariable: 'IPMI_PASS',
            usernameVariable: 'IPMI_USER']
             ]) {
             env.IPMI_PASS = IPMI_PASS
             env.IPMI_USER = IPMI_USER
             shared.reboot_hardware_nodes("${IPMI_CREDS}")
            } //withCredentials IPMI
        } //stage
        stage("Boot seed node's heat stack"){
          println "Create stack '${ENV_NAME}'"
          shared.run_cmd("pushd bm_mcc_mosk && openstack stack create --wait -t mcc_seed_2401.yaml --parameter env_name=${ENV_NAME} ${ENV_NAME}")
          seed_ext_ip = shared.run_cmd_stdout("openstack stack output show ${ENV_NAME} seed_ext_ip -f value -c output_value").trim().split().last()
          println "SEED IP: '${seed_ext_ip}'"
        } //stage
      } //withCredentials OS cloud
      sshagent(credentials: ['maintenance-team-ssh']) {
        withCredentials(
        [[$class: 'SSHUserPrivateKeyBinding',
          keyFileVariable: "MAINTENANCE_TEAM_SSH_KEY",
          credentialsId: MAINTENANCE_TEAM_SSH_ID,
          usernameVariable: "MAINTENANCE_TEAM_SSH_USERNAME"]]) {
          stage("Bootstrap MCC+child"){
             sh "scp ${ssh_params} ${MAINTENANCE_TEAM_SSH_KEY} root@${seed_ext_ip}:/root/.ssh/id_rsa "
             sh "scp -r ${ssh_params} bm_mcc_mosk root@${seed_ext_ip}: "
             sh "ssh ${ssh_params} root@${seed_ext_ip} bash do_deploy_mcc_mgmt.sh"
             sh "ssh ${ssh_params} root@${seed_ext_ip} bash do_deploy_child.sh"
             // TODO: unhardcode cluster names
             check_child_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get cluster mosk -o jsonpath='{.status.providerStatus.ready}' 2>/dev/null || echo 'ssh error, ignoring'"
             check_kcc_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/kubeconfig -n mosk get kcc mosk-ceph -o jsonpath='{.status.shortClusterInfo.state}' 2>/dev/null || echo 'ssh error, ignoring'"
               child_ready = false
               kcc_ready = false
               println "Waiting child cluster to become ready..."
               while(!child_ready){
                 sleep 60
                 status = sh(returnStdout: true, script: "${check_child_cmd}").trim()
                 if (status == "true"){
                   child_ready = true
                 } //if
               } //while
               println "Waiting child Ceph cluster to become ready..."
               while(!kcc_ready){
                 sleep 60
                 status = sh(returnStdout: true, script: "${check_kcc_cmd}").trim()
                 if (status == "Ready"){
                   kcc_ready = true
                 } //if
               } //while
          } //stage MCC
          stage("Prepare and deploy MOSK"){
            sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/get_child_kubeconfig.sh'"
            sh "ssh ${ssh_params} root@${seed_ext_ip} '/bin/bash /root/bm_mcc_mosk/utils/tsl_gen.sh'"
            kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
            sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/osdpl-ssl-secrets.yaml"
            sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/osdpl.yaml"
            mosk_check_cmd = "${kubectl_openstack_cmd} get osdplst osh-dev -o jsonpath='{.status.osdpl.state}'"
            mosk_ready = false
              while(!mosk_ready){
                sleep 60
                status = sh(returnStdout: true, script: "${mosk_check_cmd}").trim()
                if (status == "APPLIED"){
                  mosk_ready = true
                } //if
              } //while
            } //stage MOSK
          stage("Configure DNS server") {
            get_ingress_ip_cmd = "${kubectl_openstack_cmd} get service ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}'"
            get_dns_internal_ip_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n coredns get service coredns-coredns -o jsonpath='{.spec.clusterIP}'"
            ingress_ip = sh(returnStdout: true, script: get_ingress_ip_cmd).trim()
            sh "ssh ${ssh_params} root@${seed_ext_ip} sed -i 's/!!!EXT_DNS_IP/${ingress_ip}/g' /root/bm_mcc_mosk/child/kaas_workloads/coredns.yaml"
            sh "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig apply -f /root/bm_mcc_mosk/child/kaas_workloads/coredns.yaml"
            sleep 120
            dns_internal_ip = sh(returnStdout: true, script: get_dns_internal_ip_cmd).trim()
            sh "ssh ${ssh_params} root@${seed_ext_ip} \"/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n kube-system get configmap coredns -oyaml > coredns.conf\""
            sh "ssh ${ssh_params} root@${seed_ext_ip} /usr/bin/python3 /root/bm_mcc_mosk/utils/update_kube_dns_conf.py ${dns_internal_ip}"
            sh "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig apply -f coredns.patched.conf"

          } // stage DNS

          stage("Get openstack credentials") {
            kubectl_openstack_external_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack-external "
            cmd = "${kubectl_openstack_external_cmd} get secrets openstack-identity-credentials -o jsonpath='{.data.clouds\\\\.yaml}' | base64 -d"
            openstack_credentials = sh(returnStdout: true, script: "${cmd}")
            println openstack_credentials
            writeFile(file: "openstack_credentials.yaml", text: openstack_credentials, encoding: "UTF-8")
          } //stage get credentials

          stage("Tempest configuration and run") {
            kubectl_openstack_cmd = "ssh ${ssh_params} root@${seed_ext_ip} /root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack "
            sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[\\\"tempest\\\"]}}}'"
            tempest_pod_name_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get pods -o custom-columns=POD:.metadata.name --no-headers | grep openstack-tempest-run-tests'"
            tempest_pod_name = ''
            while(tempest_pod_name==''){
              try{
                tempest_pod_name = sh(returnStdout: true, script: "${tempest_pod_name_cmd}").trim()
              } catch(Exception e) {
              println("Wait tempest pod start (Exception: ${e}")}
              sleep 30
            }
            tempest_check_cmd = "ssh ${ssh_params} root@${seed_ext_ip} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack get -o jsonpath=\'{.status.phase}\' pod/${tempest_pod_name}'"
            status = ""
            while (status != "Succeeded"){ //Completed
              status = sh(returnStdout: true, script: "${tempest_check_cmd}").trim()
              println("Tempest pod status is ${status}")
              sleep 60
            }
          } //stage tempest
          stage("Archive artifacts"){
            sh "${kubectl_openstack_cmd} apply -f /root/bm_mcc_mosk/child/kaas_workloads/tempest-pvc-storage.yaml"
            sleep 60
            ssh_cmd = "ssh ${ssh_params} root@${seed_ext_ip} "
            sh "${ssh_cmd} '/root/kaas-bootstrap/bin/kubectl --kubeconfig /root/child.kubeconfig -n openstack cp tempest-test-results-pod:var/lib/tempest/data/ /root/tempest_log/'"
            sh "scp -r -v ${ssh_params} root@${seed_ext_ip}:/root/tempest_log ."
            sh "tar -czvf tempest_log/tempest_full.tar.gz tempest_log/tempest.log"
            archiveArtifacts artifacts: "openstack_credentials.yaml, tempest_log/*/tempest.log, tempest_log/tempest_full.tar.gz"
            sh "${kubectl_openstack_cmd} delete pod tempest-test-results-pod"
            sh "${kubectl_openstack_cmd} patch osdpl osh-dev --type merge -p '{\\\"spec\\\":{\\\"features\\\":{\\\"services\\\":[]}}}'"
          }
        } //withCredentials
      } //sshagent
    } //node
  } //timestamps
} //timeout