jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy

/**
 *
 * Create fuel-devops environment, generate a model for it
 * and bootstrap a salt cluster on the environment nodes
 *
 * Expected parameters:

 *   PARENT_NODE_NAME              Name of the jenkins slave to create the environment
 *   PARENT_WORKSPACE              Path to the workspace of the parent job to use tcp-qa repo
 *   LAB_CONFIG_NAME               Name of the tcp-qa deployment template
 *   ENV_NAME                      Fuel-devops environment name
 *   MCP_VERSION                   MCP version, like 2018.4 or proposed
 *   MCP_IMAGE_PATH1604            Local path to the image http://ci.mcp.mirantis.net:8085/images/ubuntu-16-04-x64-mcpproposed.qcow2
 *   IMAGE_PATH_CFG01_DAY01        Local path to the image http://ci.mcp.mirantis.net:8085/images/cfg01-day01-proposed.qcow2
 *   CFG01_CONFIG_IMAGE_NAME       Name for the creating config drive image, like cfg01.${LAB_CONFIG_NAME}-config-drive.iso
 *   TCP_QA_REFS                   Reference to the tcp-qa change on Gerrit, like refs/changes/46/418546/41
 *   PIPELINE_LIBRARY_REF          Reference to the pipeline-library change
 *   MK_PIPELINES_REF              Reference to the mk-pipelines change
 *   COOKIECUTTER_TEMPLATE_COMMIT  Commit/tag/branch for cookiecutter-templates repository. If empty, then takes ${MCP_VERSION} value
 *   SALT_MODELS_SYSTEM_COMMIT     Commit/tag/branch for reclass-system repository. If empty, then takes ${MCP_VERSION} value
 *   SHUTDOWN_ENV_ON_TEARDOWN      optional, shutdown fuel-devops environment at the end of the job
 *   MCP_SALT_REPO_URL             Base URL for MCP repositories required to bootstrap cfg01 node. Leave blank to use default
 *                                 (http://mirror.mirantis.com/ from mcp-common-scripts)
 *   MCP_SALT_REPO_KEY             URL of the key file. Leave blank to use default
 *                                 (${MCP_SALT_REPO_URL}/${MCP_VERSION}/salt-formulas/xenial/archive-salt-formulas.key from mcp-common-scripts)
 *   OS_AUTH_URL                   OpenStack keystone catalog URL
 *   OS_PROJECT_NAME               OpenStack project (tenant) name
 *   OS_USER_DOMAIN_NAME           OpenStack user domain name
 *   OS_CREDENTIALS                OpenStack username and password credentials ID in Jenkins
 *   JENKINS_PIPELINE_BRANCH       Should be set in release/proposed/2019.2.0 when we test non-released version
 *   UPDATE_VERSION                Version of update to deploy
 *   LAB_PARAM_DEFAULTS            Filename placed in tcp_tests/templates/_heat_environments, with default parameters for the heat template
 *   DEPLOY_EMPTY_NODE             Add extra node to heat stack. Node without a role and with salt-minion
 *
 *   CREATE_JENKINS_NODE_CREDENTIALS   Jenkins username and password with rights to add/delete Jenkins agents
 */

@Library('tcp-qa')_

import groovy.xml.XmlUtil

common = new com.mirantis.mk.Common()
shared = new com.mirantis.system_qa.SharedPipeline()

if (! env.PARENT_NODE_NAME) {
    error "'PARENT_NODE_NAME' must be set from the parent deployment job!"
}

currentBuild.description = "${PARENT_NODE_NAME}:${ENV_NAME}"
def cfg01_day01_image_name = "cfg01-day01-2019.2.0"
def ubuntu_vcp_image_name = "ubuntu-vcp-2019.2.0"
def ubuntu_foundation_image_name = "ubuntu-16.04-foundation-2019.2.0"

timeout(time: 3, unit: 'HOURS') {
    timestamps {
        node ("${PARENT_NODE_NAME}") {
            if (! fileExists("${PARENT_WORKSPACE}")) {
                error "'PARENT_WORKSPACE' contains path to non-existing directory ${PARENT_WORKSPACE} on the node '${PARENT_NODE_NAME}'."
            }
            dir("${PARENT_WORKSPACE}") {

                if (env.TCP_QA_REFS) {
                    stage("Update working dir to patch ${TCP_QA_REFS}") {
                        shared.update_working_dir()
                    }
                }

                withCredentials([
                   [$class          : 'UsernamePasswordMultiBinding',
                   credentialsId   : env.OS_CREDENTIALS,
                   passwordVariable: 'OS_PASSWORD',
                   usernameVariable: 'OS_USERNAME']
                ]) {
                    env.OS_IDENTITY_API_VERSION = 3

                    stage("Cleanup: erase ${ENV_NAME} and remove config drive") {

                        // delete heat stack
                        println "Remove heat stack '${ENV_NAME}'"
                        shared.run_cmd("""\
                            # export OS_IDENTITY_API_VERSION=3
                            # export OS_AUTH_URL=${OS_AUTH_URL}
                            # export OS_USERNAME=${OS_USERNAME}
                            # export OS_PASSWORD=${OS_PASSWORD}
                            # export OS_PROJECT_NAME=${OS_PROJECT_NAME}
                            openstack --insecure stack delete -y ${ENV_NAME} || true
                            timeout 20m /bin/bash -c "while openstack --insecure stack show ${ENV_NAME} -f value -c stack_status; do sleep 10; done"
                        """)

                        println "Remove config drive ISO"
                        shared.run_cmd("""\
                            rm /home/jenkins/images/${CFG01_CONFIG_IMAGE_NAME} || true
                        """)
                    }

                    stage("Generate the model") {
                        def IPV4_NET_ADMIN=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py management_subnet_cidr").trim().split().last()
                        def IPV4_NET_CONTROL=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py control_subnet_cidr").trim().split().last()
                        def IPV4_NET_TENANT=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py tenant_subnet_cidr").trim().split().last()
                        def IPV4_NET_EXTERNAL=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py external_subnet_cidr").trim().split().last()
                        shared.generate_cookied_model(IPV4_NET_ADMIN, IPV4_NET_CONTROL, IPV4_NET_TENANT, IPV4_NET_EXTERNAL)
                    }

                    stage("Generate config drive ISO") {
                        SALT_MASTER_IP=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py management_subnet_cfg01_ip").trim().split().last()
                        def ADMIN_NETWORK_GW=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py management_subnet_gateway_ip").trim().split().last()
                        shared.generate_configdrive_iso(SALT_MASTER_IP, ADMIN_NETWORK_GW)
                    }

                    stage("Upload Ubuntu image for foundation node") {
                        shared.run_cmd("""\
                            if ! openstack --insecure image show ${ubuntu_foundation_image_name} -f value -c name; then
                                wget -O ./${ubuntu_foundation_image_name} https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
                                openstack --insecure image create ${ubuntu_foundation_image_name} --file ./${ubuntu_foundation_image_name} --disk-format qcow2 --container-format bare
                                rm ./${ubuntu_foundation_image_name}
                            else
                                echo Image ${ubuntu_foundation_image_name} already exists
                            fi
                        """)
                    }

                    stage("Upload cfg01-day01 and VCP images") {
                        shared.run_cmd("""\
                            # export OS_IDENTITY_API_VERSION=3
                            # export OS_AUTH_URL=${OS_AUTH_URL}
                            # export OS_USERNAME=${OS_USERNAME}
                            # export OS_PASSWORD=${OS_PASSWORD}
                            # export OS_PROJECT_NAME=${OS_PROJECT_NAME}

                            openstack --insecure image show ${cfg01_day01_image_name} -f value -c name || openstack --insecure image create ${cfg01_day01_image_name} --file ${IMAGE_PATH_CFG01_DAY01} --disk-format qcow2 --container-format bare
                            openstack --insecure image show ${ubuntu_vcp_image_name} -f value -c name || openstack --insecure image create ${ubuntu_vcp_image_name} --file ${MCP_IMAGE_PATH1604} --disk-format qcow2 --container-format bare
                        """)
                    }

                    stage("Upload generated config drive ISO into volume on cfg01 node") {
                        shared.run_cmd("""\
                            # export OS_IDENTITY_API_VERSION=3
                            # export OS_AUTH_URL=${OS_AUTH_URL}
                            # export OS_USERNAME=${OS_USERNAME}
                            # export OS_PASSWORD=${OS_PASSWORD}
                            # export OS_PROJECT_NAME=${OS_PROJECT_NAME}

                            openstack --insecure image delete cfg01.${ENV_NAME}-config-drive.iso || true
                            sleep 3
                            openstack --insecure image create cfg01.${ENV_NAME}-config-drive.iso --file /home/jenkins/images/${CFG01_CONFIG_IMAGE_NAME} --disk-format iso --container-format bare
                        """)
                    }

                    stage("Create Heat stack '${ENV_NAME}'") {
                        // Create stack and wait for CREATE_COMPLETED status, manual analog:
                        //    openstack --insecure stack create ${ENV_NAME} \
                        //        --template ./tcp_tests/templates/${LAB_CONFIG_NAME}/underlay.hot \
                        //        --environment ./tcp_tests/templates/_heat_environments/${LAB_PARAM_DEFAULTS} \
                        //        --parameter env_name=${ENV_NAME} --parameter mcp_version=${MCP_VERSION}
                        shared.run_cmd("""\
                            export BOOTSTRAP_TIMEOUT=3600
                            export ENV_MANAGER=heat
                            export TEST_GROUP=test_create_environment
                            export SHUTDOWN_ENV_ON_TEARDOWN=false
                            export PYTHONIOENCODING=UTF-8
                            export REPOSITORY_SUITE=${MCP_VERSION}
                            export UPDATE_VERSION=${UPDATE_VERSION}
                            export ENV_NAME=${ENV_NAME}
                            export LAB_CONFIG_NAME=${LAB_CONFIG_NAME}
                            export LAB_PARAM_DEFAULTS=${LAB_PARAM_DEFAULTS}
                            export DEPLOY_EMPTY_NODE=${DEPLOY_EMPTY_NODE}
                            export LOG_NAME=swarm_test_create_environment.log
                            py.test --cache-clear -vvv -s -p no:django -p no:ipdb --junit-xml=deploy_hardware.xml -k \${TEST_GROUP}
                        """)
                    }

                    stage("Add the Jenkins slave node") {
                        def jenkins_slave_ip_value_name = "foundation_public_ip"
                        def jenkins_slave_ip = shared.run_cmd_stdout("openstack --insecure stack output show ${ENV_NAME} ${jenkins_slave_ip_value_name} -f value -c output_value").trim().split().last()
                        def jenkins_slave_executors = 2
                        common.printMsg("JENKINS_SLAVE_NODE_NAME=${JENKINS_SLAVE_NODE_NAME}", "green")
                        common.printMsg("JENKINS_SLAVE_IP=${jenkins_slave_ip}", "green")

                        withCredentials([[$class          : 'UsernamePasswordMultiBinding',
                                          credentialsId   : "${CREATE_JENKINS_NODE_CREDENTIALS}",
                                          passwordVariable: 'JENKINS_PASS',
                                          usernameVariable: 'JENKINS_USER']]) {
                            sh "curl ${JENKINS_URL}/jnlpJars/jenkins-cli.jar --output jenkins-cli.jar"

                            sh "java -jar jenkins-cli.jar -s ${JENKINS_URL} -auth ${JENKINS_USER}:${JENKINS_PASS} delete-node ${JENKINS_SLAVE_NODE_NAME} || true"
                            create_node_xml="""\
                                            <slave>
                                            <name>${JENKINS_SLAVE_NODE_NAME}</name>
                                            <description>${ENV_NAME}</description>
                                            <remoteFS>/home/jenkins/workspace</remoteFS>
                                            <numExecutors>${jenkins_slave_executors}</numExecutors>
                                            <mode>EXCLUSIVE</mode>
                                            <retentionStrategy class="hudson.slaves.RetentionStrategy\$Always"/>
                                            <launcher class="hudson.plugins.sshslaves.SSHLauncher">
                                              <host>${jenkins_slave_ip}</host>
                                              <port>22</port>
                                              <credentialsId>${ACCESS_JENKINS_NODE_CREDENTIALS}</credentialsId>
                                            </launcher>
                                            <label>${ENV_NAME}</label>
                                            <nodeProperties/>
                                            </slave>""".stripIndent().replaceAll("\\n", "")
                            sh "echo '${create_node_xml}' | java -jar jenkins-cli.jar -s ${JENKINS_URL} -auth ${JENKINS_USER}:${JENKINS_PASS} create-node ${JENKINS_SLAVE_NODE_NAME}"

                            timeout(time: 30, unit: 'MINUTES') {
                                node("${JENKINS_SLAVE_NODE_NAME}") {
                                    sh "echo 'ok'"
                                    println "Jenkins agent is available now and can executes commands"
                                }
                            }
                            // Store jenkins agent IP address
                            jenkins_agent_description = "ssh jenkins@${jenkins_slave_ip}  # foundation node with Jenkins agent <a href=${JENKINS_URL}/computer/${JENKINS_SLAVE_NODE_NAME}>${JENKINS_SLAVE_NODE_NAME}</a><br>ssh root@${SALT_MASTER_IP}  # cfg01 node<br>"
                            writeFile(file: "jenkins_agent_description.txt", text: jenkins_agent_description, encoding: "UTF-8")

                        } // withCredentials

                    }// stage

                } // withCredentials

            } // dir
        } // node


        node ("${JENKINS_SLAVE_NODE_NAME}") {
            dir("${PARENT_WORKSPACE}") {

                stage("Clean the environment and clone tcp-qa") {
                    deleteDir()
                    // Install  TestRail reporter to upload test results to TestRail
                    shared.verbose_sh("""\
                        [ -d /home/jenkins/venv_testrail_reporter ] || virtualenv --python=python3.7 /home/jenkins/venv_testrail_reporter
                    """, true, false, true)
                    sshagent(credentials: ['maintenance-team-ssh']) {
                        shared.run_cmd("""\
                            . /home/jenkins/venv_testrail_reporter/bin/activate; \
                            pip install urllib3==1.26.6; \
                            pip install git+ssh://maintenance-ci-robot@gerrit.mcp.mirantis.com:29418/mcp/testrail-reporter -U;
                        """)
                    }
                    // Install DEVOPS tools
                    shared.verbose_sh("""\
                        [ -d /home/jenkins/fuel-devops30 ] || virtualenv /home/jenkins/fuel-devops30
                    """, true, false, true)
                    shared.run_cmd("""\
                        git clone https://gerrit.mcp.mirantis.com/mcp/tcp-qa ${PARENT_WORKSPACE}
                    """)
                    // Install  TestRail reporter to mark known failures test results to TestRail
                    shared.verbose_sh("""\
                        [ -d /home/jenkins/venv_testrail_analyzer ] || virtualenv --python=python3.7 /home/jenkins/venv_testrail_analyzer
                    """, true, false, true)
                    shared.run_cmd("""\
                        . /home/jenkins/venv_testrail_analyzer/bin/activate; \
                        pip install urllib3==1.26.6; \
                        pip install git+https://review.gerrithub.io/ibumarskov/testrail-reporter@b9041b241f3364022cd0ad63510f02aab4352a6b;
                    """)
                    shared.update_working_dir()
                    // Add info about foundation node for description
                    writeFile(file: "jenkins_agent_description.txt", text: jenkins_agent_description, encoding: "UTF-8")
                }

                withCredentials([
                   [$class          : 'UsernamePasswordMultiBinding',
                   credentialsId   : env.OS_CREDENTIALS,
                   passwordVariable: 'OS_PASSWORD',
                   usernameVariable: 'OS_USERNAME']
                ]) {


                    stage("Run the 'underlay' and 'salt-deployed' fixtures to bootstrap salt cluster") {
                        def xml_report_name = "deploy_salt.xml"
                        try {
                            // deploy_salt.xmli
                            shared.run_sh("""\
                                export ENV_NAME=${ENV_NAME}
                                export LAB_CONFIG_NAME=${LAB_CONFIG_NAME}
                                export LAB_PARAM_DEFAULTS=${LAB_PARAM_DEFAULTS}
                                export ENV_MANAGER=heat
                                export SHUTDOWN_ENV_ON_TEARDOWN=false
                                export BOOTSTRAP_TIMEOUT=3600
                                export PYTHONIOENCODING=UTF-8
                                export REPOSITORY_SUITE=${MCP_VERSION}
                                export UPDATE_VERSION=${UPDATE_VERSION}
                                export TEST_GROUP=test_bootstrap_salt
                                export LOG_NAME=swarm_test_bootstrap_salt.log
                                py.test -vvv -s -p no:django -p no:ipdb --junit-xml=${xml_report_name} -k \${TEST_GROUP}
                            """)
                            // Wait for jenkins to start and IO calm down
                            sleep(60)

                        } catch (e) {
                              common.printMsg("Saltstack cluster deploy is failed", "purple")
                              if (fileExists(xml_report_name)) {
                                  shared.download_logs("deploy_salt_${ENV_NAME}")
                                  def String junit_report_xml = readFile(xml_report_name)
                                  def String junit_report_xml_pretty = new XmlUtil().serialize(junit_report_xml)
                                  throw new Exception(junit_report_xml_pretty)
                              } else {
                                  throw e
                              }
                        }   // finally {
                            // TODO(ddmitriev): add checks for salt cluster
                            // }
                    } // stage
                } // withCredentials
            } // dir
        } // node
    } // timestamps
} // timeout