jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy

/**
 *
 * Create fuel-devops environment, generate a model for it
 * and bootstrap a salt cluster on the environment nodes
 *
 * Expected parameters:

 *   PARENT_NODE_NAME              Name of the jenkins slave to create the environment
 *   PARENT_WORKSPACE              Path to the workspace of the parent job to use tcp-qa repo
 *   LAB_CONFIG_NAME               Name of the tcp-qa deployment template
 *   ENV_NAME                      Fuel-devops environment name
 *   MCP_VERSION                   MCP version, like 2018.4 or proposed
 *   MCP_IMAGE_PATH1604            Local path to the image http://ci.mcp.mirantis.net:8085/images/ubuntu-16-04-x64-mcpproposed.qcow2
 *   IMAGE_PATH_CFG01_DAY01        Local path to the image http://ci.mcp.mirantis.net:8085/images/cfg01-day01-proposed.qcow2
 *   CFG01_CONFIG_IMAGE_NAME       Name for the creating config drive image, like cfg01.${LAB_CONFIG_NAME}-config-drive.iso
 *   TCP_QA_REFS                   Reference to the tcp-qa change on Gerrit, like refs/changes/46/418546/41
 *   PIPELINE_LIBRARY_REF          Reference to the pipeline-library change
 *   MK_PIPELINES_REF              Reference to the mk-pipelines change
 *   COOKIECUTTER_TEMPLATE_COMMIT  Commit/tag/branch for cookiecutter-templates repository. If empty, then takes ${MCP_VERSION} value
 *   SALT_MODELS_SYSTEM_COMMIT     Commit/tag/branch for reclass-system repository. If empty, then takes ${MCP_VERSION} value
 *   SHUTDOWN_ENV_ON_TEARDOWN      optional, shutdown fuel-devops environment at the end of the job
 *   MCP_SALT_REPO_URL             Base URL for MCP repositories required to bootstrap cfg01 node. Leave blank to use default
 *                                 (http://mirror.mirantis.com/ from mcp-common-scripts)
 *   MCP_SALT_REPO_KEY             URL of the key file. Leave blank to use default
 *                                 (${MCP_SALT_REPO_URL}/${MCP_VERSION}/salt-formulas/xenial/archive-salt-formulas.key from mcp-common-scripts)
 *   OS_AUTH_URL                   OpenStack keystone catalog URL
 *   OS_PROJECT_NAME               OpenStack project (tenant) name
 *   OS_USER_DOMAIN_NAME           OpenStack user domain name
 *   OS_CREDENTIALS                OpenStack username and password credentials ID in Jenkins
 *   JENKINS_PIPELINE_BRANCH       Should be set in release/proposed/2019.2.0 when we test non-released version
 *   UPDATE_VERSION                Version of update to deploy
 *   LAB_PARAM_DEFAULTS            Filename placed in tcp_tests/templates/_heat_environments, with default parameters for the heat template
 *
 *   CREATE_JENKINS_NODE_CREDENTIALS   Jenkins username and password with rights to add/delete Jenkins agents
 */

@Library('tcp-qa')_

import groovy.xml.XmlUtil

common = new com.mirantis.mk.Common()
shared = new com.mirantis.system_qa.SharedPipeline()

if (! env.PARENT_NODE_NAME) {
    error "'PARENT_NODE_NAME' must be set from the parent deployment job!"
}

currentBuild.description = "${PARENT_NODE_NAME}:${ENV_NAME}"
def cfg01_day01_image_name = "cfg01-day01-2019.2.0"
def ubuntu_vcp_image_name = "ubuntu-vcp-2019.2.0"
def ubuntu_foundation_image_name = "ubuntu-16.04-foundation-2019.2.0"

timeout(time: 2, unit: 'HOURS') {
    timestamps {
        node ("${PARENT_NODE_NAME}") {
            if (! fileExists("${PARENT_WORKSPACE}")) {
                error "'PARENT_WORKSPACE' contains path to non-existing directory ${PARENT_WORKSPACE} on the node '${PARENT_NODE_NAME}'."
            }
            dir("${PARENT_WORKSPACE}") {

                if (env.TCP_QA_REFS) {
                    stage("Update working dir to patch ${TCP_QA_REFS}") {
                        shared.update_working_dir()
                    }
                }

                withCredentials([
                   [$class          : 'UsernamePasswordMultiBinding',
                   credentialsId   : env.OS_CREDENTIALS,
                   passwordVariable: 'OS_PASSWORD',
                   usernameVariable: 'OS_USERNAME']
                ]) {
                    env.OS_IDENTITY_API_VERSION = 3

                    stage("Cleanup: erase ${ENV_NAME} and remove config drive") {

                        // delete heat stack
                        println "Remove heat stack '${ENV_NAME}'"
                        shared.run_cmd("""\
                            # export OS_IDENTITY_API_VERSION=3
                            # export OS_AUTH_URL=${OS_AUTH_URL}
                            # export OS_USERNAME=${OS_USERNAME}
                            # export OS_PASSWORD=${OS_PASSWORD}
                            # export OS_PROJECT_NAME=${OS_PROJECT_NAME}
                            openstack --insecure stack delete -y ${ENV_NAME} || true
                            timeout 20m /bin/bash -c "while openstack --insecure stack show ${ENV_NAME} -f value -c stack_status; do sleep 10; done"
                        """)

                        println "Remove config drive ISO"
                        shared.run_cmd("""\
                            rm /home/jenkins/images/${CFG01_CONFIG_IMAGE_NAME} || true
                        """)
                    }

                    stage("Generate the model") {
                        def IPV4_NET_ADMIN=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py management_subnet_cidr").trim().split().last()
                        def IPV4_NET_CONTROL=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py control_subnet_cidr").trim().split().last()
                        def IPV4_NET_TENANT=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py tenant_subnet_cidr").trim().split().last()
                        def IPV4_NET_EXTERNAL=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py external_subnet_cidr").trim().split().last()
                        shared.generate_cookied_model(IPV4_NET_ADMIN, IPV4_NET_CONTROL, IPV4_NET_TENANT, IPV4_NET_EXTERNAL)
                    }

                    stage("Generate config drive ISO") {
                        SALT_MASTER_IP=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py management_subnet_cfg01_ip").trim().split().last()
                        def ADMIN_NETWORK_GW=shared.run_cmd_stdout("./tcp_tests/utils/get_param_heat_template.py management_subnet_gateway_ip").trim().split().last()
                        shared.generate_configdrive_iso(SALT_MASTER_IP, ADMIN_NETWORK_GW)
                    }

                    stage("Upload Ubuntu image for foundation node") {
                        shared.run_cmd("""\
                            if ! openstack --insecure image show ${ubuntu_foundation_image_name} -f value -c name; then
                                wget -O ./${ubuntu_foundation_image_name} https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
                                openstack --insecure image create ${ubuntu_foundation_image_name} --file ./${ubuntu_foundation_image_name} --disk-format qcow2 --container-format bare
                                rm ./${ubuntu_foundation_image_name}
                            else
                                echo Image ${ubuntu_foundation_image_name} already exists
                            fi
                        """)
                    }

                    stage("Upload cfg01-day01 and VCP images") {
                        shared.run_cmd("""\
                            # export OS_IDENTITY_API_VERSION=3
                            # export OS_AUTH_URL=${OS_AUTH_URL}
                            # export OS_USERNAME=${OS_USERNAME}
                            # export OS_PASSWORD=${OS_PASSWORD}
                            # export OS_PROJECT_NAME=${OS_PROJECT_NAME}

                            openstack --insecure image show ${cfg01_day01_image_name} -f value -c name || openstack --insecure image create ${cfg01_day01_image_name} --file ${IMAGE_PATH_CFG01_DAY01} --disk-format qcow2 --container-format bare
                            openstack --insecure image show ${ubuntu_vcp_image_name} -f value -c name || openstack --insecure image create ${ubuntu_vcp_image_name} --file ${MCP_IMAGE_PATH1604} --disk-format qcow2 --container-format bare
                        """)
                    }

                    stage("Upload generated config drive ISO into volume on cfg01 node") {
                        shared.run_cmd("""\
                            # export OS_IDENTITY_API_VERSION=3
                            # export OS_AUTH_URL=${OS_AUTH_URL}
                            # export OS_USERNAME=${OS_USERNAME}
                            # export OS_PASSWORD=${OS_PASSWORD}
                            # export OS_PROJECT_NAME=${OS_PROJECT_NAME}

                            openstack --insecure image delete cfg01.${ENV_NAME}-config-drive.iso || true
                            sleep 3
                            openstack --insecure image create cfg01.${ENV_NAME}-config-drive.iso --file /home/jenkins/images/${CFG01_CONFIG_IMAGE_NAME} --disk-format iso --container-format bare
                        """)
                    }

                    stage("Create Heat stack '${ENV_NAME}'") {
                        // Create stack and wait for CREATE_COMPLETED status, manual analog:
                        //    openstack --insecure stack create ${ENV_NAME} \
                        //        --template ./tcp_tests/templates/${LAB_CONFIG_NAME}/underlay.hot \
                        //        --environment ./tcp_tests/templates/_heat_environments/${LAB_PARAM_DEFAULTS} \
                        //        --parameter env_name=${ENV_NAME} --parameter mcp_version=${MCP_VERSION}
                        shared.run_cmd("""\
                            export BOOTSTRAP_TIMEOUT=3600
                            export ENV_MANAGER=heat
                            export TEST_GROUP=test_create_environment
                            export SHUTDOWN_ENV_ON_TEARDOWN=false
                            export PYTHONIOENCODING=UTF-8
                            export REPOSITORY_SUITE=${MCP_VERSION}
                            export UPDATE_VERSION=${UPDATE_VERSION}
                            export ENV_NAME=${ENV_NAME}
                            export LAB_CONFIG_NAME=${LAB_CONFIG_NAME}
                            export LAB_PARAM_DEFAULTS=${LAB_PARAM_DEFAULTS}
                            export LOG_NAME=swarm_test_create_environment.log
                            py.test --cache-clear -vvv -s -p no:django -p no:ipdb --junit-xml=deploy_hardware.xml -k \${TEST_GROUP}
                        """)
                    }

                    stage("Add the Jenkins slave node") {
                        def jenkins_slave_ip_value_name = "foundation_public_ip"
                        def jenkins_slave_ip = shared.run_cmd_stdout("openstack --insecure stack output show ${ENV_NAME} ${jenkins_slave_ip_value_name} -f value -c output_value").trim().split().last()
                        def jenkins_slave_executors = 2
                        common.printMsg("JENKINS_SLAVE_NODE_NAME=${JENKINS_SLAVE_NODE_NAME}", "green")
                        common.printMsg("JENKINS_SLAVE_IP=${jenkins_slave_ip}", "green")

                withCredentials([
                   [$class          : 'UsernamePasswordMultiBinding',
                   credentialsId   : "${CREATE_JENKINS_NODE_CREDENTIALS}",
                   passwordVariable: 'JENKINS_PASS',
                   usernameVariable: 'JENKINS_USER']
                ]) {

                        script_delete_agent = ("""\
                            CRUMB=\$(curl --fail -0 -u \"\${JENKINS_USER}:\${JENKINS_PASS}\" \${JENKINS_URL}\'/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)\' 2>/dev/null)
                            curl -w '%{http_code}' -o /dev/null \
                                -u \"\${JENKINS_USER}:\${JENKINS_PASS}\" \
                                -H \"Content-Type:application/x-www-form-urlencoded\" \
                                -H \"\$CRUMB\" \
                                \"\${JENKINS_URL}/computer/\${JENKINS_SLAVE_NODE_NAME}/doDelete\" \
                                --request \'POST\' --data \'\'
                            sleep 10
                        """)

                        script_create_agent = ("""\
                            CRUMB=\$(curl --fail -0 -u \"\${JENKINS_USER}:\${JENKINS_PASS}\" \${JENKINS_URL}\'/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)\' 2>/dev/null)

                            curl -L -sS -w '%{http_code}' -o /dev/null \
                                -u \"\${JENKINS_USER}:\${JENKINS_PASS}\" \
                                -H \"Content-Type:application/x-www-form-urlencoded\" \
                                -H \"\$CRUMB\" \
                                -X POST -d 'json={\
                                    \"name\": \"'\"\$JENKINS_SLAVE_NODE_NAME\"'\", \
                                    \"nodeDescription\": \"'\"\$ENV_NAME\"'\", \
                                    \"numExecutors\": \"'\"${jenkins_slave_executors}\"'\", \
                                    \"remoteFS\": \"'\"/home/jenkins/workspace\"'\", \
                                    \"labelString\": \"'\"\$ENV_NAME\"'\", \
                                    \"mode\": \"EXCLUSIVE\", \
                                    \"\": [\"hudson.plugins.sshslaves.SSHLauncher\", \"hudson.slaves.RetentionStrategy\$Always\"], \
                                    \"launcher\": {\
                                        \"stapler-class\": \"hudson.plugins.sshslaves.SSHLauncher\", \
                                        \"\$class\": \"hudson.plugins.sshslaves.SSHLauncher\", \
                                        \"host\": \"'\"${jenkins_slave_ip}\"'\", \
                                        \"credentialsId\": \"'\"\$ACCESS_JENKINS_NODE_CREDENTIALS\"'\", \
                                        \"port\": \"'\"22\"'\", \
                                        \"javaPath\": \"\", \
                                        \"jvmOptions\": \"\", \
                                        \"prefixStartSlaveCmd\": \"\", \
                                        \"suffixStartSlaveCmd\": \"\", \
                                        \"launchTimeoutSeconds\": \"\", \
                                        \"maxNumRetries\": \"\", \
                                        \"retryWaitTime\": \"\", \
                                        \"sshHostKeyVerificationStrategy\": {\
                                            \"\$class\": \"hudson.plugins.sshslaves.verifiers.NonVerifyingKeyVerificationStrategy\" \
                                        }, \
                                        \"tcpNoDelay\": \"true\"\
                                    }, \
                                    \"retentionStrategy\": {\
                                        \"stapler-class\": \"hudson.slaves.RetentionStrategy\$Always\", \
                                        \"\$class\": \"hudson.slaves.RetentionStrategy\$Always\"\
                                    }, \
                                    \"nodeProperties\": {\
                                        \"stapler-class-bag\": \"true\"\
                                    }, \
                                    \"type\": \"hudson.slaves.DumbSlave\", \
                                    \"crumb\": \"'\"\$CRUMB\"'\"}' \
                                \"\${JENKINS_URL}/computer/doCreateItem?name=\${JENKINS_SLAVE_NODE_NAME}&type=hudson.slaves.DumbSlave\"
                        """)
                        shared.verbose_sh(script_delete_agent, true, false, true)
                        shared.verbose_sh(script_create_agent, true, false, true)
                        timeout(time: 30, unit: 'MINUTES') {
                            node("${JENKINS_SLAVE_NODE_NAME}") {
                                sh "echo 'ok'"
                                println "Jenkins agent is available now and can executes commands"
                            }
                        }
                        // Store jenkins agent IP address
                        jenkins_agent_description = "ssh jenkins@${jenkins_slave_ip}  # foundation node with Jenkins agent <a href=${JENKINS_URL}/computer/${JENKINS_SLAVE_NODE_NAME}>${JENKINS_SLAVE_NODE_NAME}</a><br>ssh root@${SALT_MASTER_IP}  # cfg01 node<br>"
                        writeFile(file: "jenkins_agent_description.txt", text: jenkins_agent_description, encoding: "UTF-8")

                } // withCredentials

                    }// stage

                } // withCredentials

            } // dir
        } // node


        node ("${JENKINS_SLAVE_NODE_NAME}") {
            dir("${PARENT_WORKSPACE}") {

                stage("Clean the environment and clone tcp-qa") {
                    deleteDir()
                    shared.verbose_sh("""\
                        [ -d /home/jenkins/venv_testrail_reporter ] || virtualenv /home/jenkins/venv_testrail_reporter
                    """, true, false, true)
                    shared.run_cmd("""\
                        . /home/jenkins/venv_testrail_reporter/bin/activate; pip install git+https://github.com/dis-xcom/testrail_reporter -U
                    """)
                    shared.verbose_sh("""\
                        [ -d /home/jenkins/fuel-devops30 ] || virtualenv /home/jenkins/fuel-devops30
                    """, true, false, true)
                    shared.run_cmd("""\
                        git clone https://gerrit.mcp.mirantis.com/mcp/tcp-qa ${PARENT_WORKSPACE}
                    """)
                    shared.update_working_dir()
                }

                withCredentials([
                   [$class          : 'UsernamePasswordMultiBinding',
                   credentialsId   : env.OS_CREDENTIALS,
                   passwordVariable: 'OS_PASSWORD',
                   usernameVariable: 'OS_USERNAME']
                ]) {


                    stage("Run the 'underlay' and 'salt-deployed' fixtures to bootstrap salt cluster") {
                        def xml_report_name = "deploy_salt.xml"
                        try {
                            // deploy_salt.xml
                            shared.run_sh("""\
                                export ENV_NAME=${ENV_NAME}
                                export LAB_CONFIG_NAME=${LAB_CONFIG_NAME}
                                export LAB_PARAM_DEFAULTS=${LAB_PARAM_DEFAULTS}
                                export ENV_MANAGER=heat
                                export SHUTDOWN_ENV_ON_TEARDOWN=false
                                export BOOTSTRAP_TIMEOUT=3600
                                export PYTHONIOENCODING=UTF-8
                                export REPOSITORY_SUITE=${MCP_VERSION}
                                export UPDATE_VERSION=${UPDATE_VERSION}
                                export TEST_GROUP=test_bootstrap_salt
                                export LOG_NAME=swarm_test_bootstrap_salt.log
                                py.test -vvv -s -p no:django -p no:ipdb --junit-xml=${xml_report_name} -k \${TEST_GROUP}
                            """)
                            // Wait for jenkins to start and IO calm down
                            sleep(60)

                        } catch (e) {
                              common.printMsg("Saltstack cluster deploy is failed", "purple")
                              if (fileExists(xml_report_name)) {
                                  shared.download_logs("deploy_salt_${ENV_NAME}")
                                  def String junit_report_xml = readFile(xml_report_name)
                                  def String junit_report_xml_pretty = new XmlUtil().serialize(junit_report_xml)
                                  throw new Exception(junit_report_xml_pretty)
                              } else {
                                  throw e
                              }
                        } finally {
                            // TODO(ddmitriev): add checks for salt cluster
                        }
                    } // stage
                } // withCredentials
            } // dir
        } // node
    } // timestamps
} // timeout