validate-cloud.groovy

/**
 *
 * Launch validation of the cloud with Rally
 *
 * Expected parameters:
 *
 *   JOB_TIMEOUT                 Job timeout in hours
 *   SALT_MASTER_URL             URL of Salt master
 *   SALT_MASTER_CREDENTIALS     Credentials to the Salt API
 *   VALIDATE_PARAMS             Validate job YAML params (see below)
 *
 *   Rally - map with parameters for starting Rally tests
 *
 *   AVAILABILITY_ZONE           The name of availability zone
 *   FLOATING_NETWORK            The name of the external(floating) network
 *   K8S_RALLY                   Use Kubernetes Rally plugin for testing K8S cluster
 *   STACKLIGHT_RALLY            Use Stacklight Rally plugin for testing Stacklight
 *   RALLY_IMAGE                 The name of the image for Rally tests
 *   RALLY_FLAVOR                The name of the flavor for Rally image
 *   RALLY_PLUGINS_REPO          Git repository with Rally plugins
 *   RALLY_PLUGINS_BRANCH        Git branch which will be used during the checkout
 *   RALLY_CONFIG_REPO           Git repository with files for Rally
 *   RALLY_CONFIG_BRANCH         Git branch which will be used during the checkout
 *   RALLY_SCENARIOS             Path to file or directory with rally scenarios
 *   RALLY_SL_SCENARIOS          Path to file or directory with stacklight rally scenarios
 *   RALLY_TASK_ARGS_FILE        Path to file with rally tests arguments
 *   RALLY_DB_CONN_STRING        Rally-compliant DB connection string for long-term storing
 *                               results to external DB
 *   RALLY_TAGS                  List of tags for marking Rally tasks. Can be used when
 *                               generating Rally trends based on particular group of tasks
 *   RALLY_TRENDS                If enabled, generate Rally trends report. Requires external DB
 *                               connection string to be set. If RALLY_TAGS was set, trends will
 *                               be generated based on finished tasks with these tags, otherwise
 *                               on all the finished tasks available in DB
 *   SKIP_LIST                   List of the Rally scenarios which should be skipped
 *
 *   PARALLEL_PERFORMANCE        If enabled, run Rally tests separately in parallel for each sub directory found
 *                               inside RALLY_SCENARIOS and RALLY_SL_SCENARIOS (if STACKLIGHT_RALLY is enabled)
 *   GENERATE_REPORT             Set this to false if you are running longevity tests on a cicd node with less than
 *                               21GB memory. Rally consumes lots of memory when generating reports sourcing week
 *                               amounts of data (BUG PROD-30433)
 */

common = new com.mirantis.mk.Common()
validate = new com.mirantis.mcp.Validate()
salt = new com.mirantis.mk.Salt()
salt_testing = new com.mirantis.mk.SaltModelTesting()

def VALIDATE_PARAMS = readYaml(text: env.getProperty('VALIDATE_PARAMS')) ?: [:]
if (! VALIDATE_PARAMS) {
    throw new Exception("VALIDATE_PARAMS yaml is empty.")
}
def TEST_IMAGE = env.getProperty('TEST_IMAGE') ?: 'xrally-openstack:1.4.0'
def JOB_TIMEOUT = env.getProperty('JOB_TIMEOUT').toInteger() ?: 12
def SLAVE_NODE = env.getProperty('SLAVE_NODE') ?: 'docker'
def rally = VALIDATE_PARAMS.get('rally') ?: [:]
def scenariosRepo = rally.get('RALLY_CONFIG_REPO') ?: 'https://review.gerrithub.io/Mirantis/scale-scenarios'
def scenariosBranch = rally.get('RALLY_CONFIG_BRANCH') ?: 'master'
def pluginsRepo = rally.get('RALLY_PLUGINS_REPO') ?: 'https://github.com/Mirantis/rally-plugins'
def pluginsBranch = rally.get('RALLY_PLUGINS_BRANCH') ?: 'master'
def tags = rally.get('RALLY_TAGS') ?: []
def generateReport = rally.get('GENERATE_REPORT', true).toBoolean()

// contrainer working dir vars
def rallyWorkdir = '/home/rally'
def rallyPluginsDir = "${rallyWorkdir}/rally-plugins"
def rallyScenariosDir = "${rallyWorkdir}/rally-scenarios"
def rallyResultsDir = "${rallyWorkdir}/test_results"
def rallySecrets = "${rallyWorkdir}/secrets"

// env vars
def env_vars = []
def platform = [
    type: 'unknown',
    stacklight: [enabled: false, grafanaPass: ''],
]
def cmp_count

// test results vars
def testResult
def tasksParallel = [:]
def parallelResults = [:]
def configRun = [:]

timeout(time: JOB_TIMEOUT, unit: 'HOURS') {
    node (SLAVE_NODE) {

        // local dir vars
        def workDir = "${env.WORKSPACE}/rally"
        def pluginsDir = "${workDir}/rally-plugins"
        def scenariosDir = "${workDir}/rally-scenarios"
        def secrets = "${workDir}/secrets"
        def artifacts = "${workDir}/validation_artifacts"

        stage('Configure env') {

            def master = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)

            // create local directories
            sh "rm -rf ${workDir} || true"
            sh "mkdir -p ${artifacts} ${secrets}"
            writeFile file: "${workDir}/entrypoint.sh", text: '''#!/bin/bash
set -xe
exec "$@"
'''
            sh "chmod 755 ${workDir}/entrypoint.sh"

            // clone repo with Rally plugins and checkout refs/branch
            checkout([
                $class           : 'GitSCM',
                branches         : [[name: 'FETCH_HEAD']],
                extensions       : [[$class: 'RelativeTargetDirectory', relativeTargetDir: pluginsDir]],
                userRemoteConfigs: [[url: pluginsRepo, refspec: pluginsBranch]],
            ])

            // clone scenarios repo and switch branch / fetch refspecs
            checkout([
                $class           : 'GitSCM',
                branches         : [[name: 'FETCH_HEAD']],
                extensions       : [[$class: 'RelativeTargetDirectory', relativeTargetDir: scenariosDir]],
                userRemoteConfigs: [[url: scenariosRepo, refspec: scenariosBranch]],
            ])

            // get number of computes in the cluster
            platform['cluster_name'] = salt.getPillar(
                master, 'I@salt:master', '_param:cluster_name'
            )['return'][0].values()[0]
            def rcs_str_node = salt.getPillar(
                master, 'I@salt:master', 'reclass:storage:node'
            )['return'][0].values()[0]

            // set up Openstack env variables
            if (rally.get('K8S_RALLY').toBoolean() == false) {

                platform['type'] = 'openstack'
                platform['cmp_count'] = rcs_str_node.openstack_compute_rack01['repeat']['count']
                def rally_variables = [
                    "floating_network=${rally.FLOATING_NETWORK}",
                    "rally_image=${rally.RALLY_IMAGE}",
                    "rally_flavor=${rally.RALLY_FLAVOR}",
                    "availability_zone=${rally.AVAILABILITY_ZONE}",
                ]

                env_vars = validate._get_keystone_creds_v3(master)
                if (!env_vars) {
                    env_vars = validate._get_keystone_creds_v2(master)
                }
                env_vars = env_vars + rally_variables

            } else {
            // set up Kubernetes env variables get required secrets
                platform['type'] = 'k8s'
                platform['cmp_count'] = rcs_str_node.kubernetes_compute_rack01['repeat']['count']

                def kubernetes = salt.getPillar(
                    master, 'I@kubernetes:master and *01*', 'kubernetes:master'
                )['return'][0].values()[0]

                env_vars = [
                    "KUBERNETES_HOST=http://${kubernetes.apiserver.vip_address}" +
                    ":${kubernetes.apiserver.insecure_port}",
                    "KUBERNETES_CERT_AUTH=${rallySecrets}/k8s-ca.crt",
                    "KUBERNETES_CLIENT_KEY=${rallySecrets}/k8s-client.key",
                    "KUBERNETES_CLIENT_CERT=${rallySecrets}/k8s-client.crt",
                ]

                // get K8S certificates to manage cluster
                def k8s_ca = salt.getFileContent(
                    master, 'I@kubernetes:master and *01*', '/etc/kubernetes/ssl/ca-kubernetes.crt'
                )
                def k8s_client_key = salt.getFileContent(
                    master, 'I@kubernetes:master and *01*', '/etc/kubernetes/ssl/kubelet-client.key'
                )
                def k8s_client_crt = salt.getFileContent(
                    master, 'I@kubernetes:master and *01*', '/etc/kubernetes/ssl/kubelet-client.crt'
                )
                writeFile file: "${secrets}/k8s-ca.crt", text: k8s_ca
                writeFile file: "${secrets}/k8s-client.key", text: k8s_client_key
                writeFile file: "${secrets}/k8s-client.crt", text: k8s_client_crt

            }

            // get Stacklight data
            if (rally.STACKLIGHT_RALLY.toBoolean() == true) {
                platform['stacklight']['enabled'] = true

                def grafana = salt.getPillar(
                    master, 'I@grafana:client', 'grafana:client:server'
                )['return'][0].values()[0]

                platform['stacklight']['grafanaPass'] = grafana['password']
            }

            if (! rally.PARALLEL_PERFORMANCE.toBoolean()) {

                // Define map with docker commands
                def commands = validate.runRallyTests(
                    platform, rally.RALLY_SCENARIOS,
                    rally.RALLY_SL_SCENARIOS, rally.RALLY_TASK_ARGS_FILE,
                    rally.RALLY_DB_CONN_STRING, tags,
                    rally.RALLY_TRENDS.toBoolean(), rally.SKIP_LIST, generateReport
                )
                def commands_list = commands.collectEntries{ [ (it.key) : { sh("${it.value}") } ] }

                configRun = [
                    'image': TEST_IMAGE,
                    'baseRepoPreConfig': false,
                    'dockerMaxCpus': 2,
                    'dockerHostname': 'localhost',
                    'dockerExtraOpts': [
                        "--network=host",
                        "--entrypoint=/entrypoint.sh",
                        "-w ${rallyWorkdir}",
                        "-v ${workDir}/entrypoint.sh:/entrypoint.sh",
                        "-v ${pluginsDir}/:${rallyPluginsDir}",
                        "-v ${scenariosDir}/:${rallyScenariosDir}",
                        "-v ${artifacts}/:${rallyResultsDir}",
                        "-v ${secrets}/:${rallySecrets}",
                    ],
                    'envOpts'         : env_vars,
                    'runCommands'     : commands_list,
                ]
                common.infoMsg('Docker config:')
                println configRun
                common.infoMsg('Docker commands list:')
                println commands

            } else {

                // Perform parallel testing of the components with Rally
                def components = [
                    Common: [],
                    Stacklight: [],
                ]

                // get list of directories inside scenarios path
                def scenPath = "${scenariosDir}/${rally.RALLY_SCENARIOS}"
                def mainComponents = sh(
                    script: "find ${scenPath} -maxdepth 1 -mindepth 1 -type d -exec basename {} \\;",
                    returnStdout: true,
                ).trim()
                if (! mainComponents) {
                    error(
                        "No directories found inside RALLY_SCENARIOS ${rally.RALLY_SCENARIOS}\n" +
                        "Either set PARALLEL_PERFORMANCE=false or populate ${rally.RALLY_SCENARIOS} " +
                        "with component directories which include corresponding scenarios"
                    )
                }
                components['Common'].addAll(mainComponents.split('\n'))
                common.infoMsg( "Adding for parallel execution sub dirs found in " +
                    "RALLY_SCENARIOS (${rally.RALLY_SCENARIOS}):"
                )
                print mainComponents

                if (rally.STACKLIGHT_RALLY.toBoolean() == true) {
                    def slScenPath = "${scenariosDir}/${rally.RALLY_SL_SCENARIOS}"
                    def slComponents = sh(
                        script: "find ${slScenPath} -maxdepth 1 -mindepth 1 -type d -exec basename {} \\;",
                        returnStdout: true,
                    ).trim()
                    if (! slComponents) {
                        error(
                            "No directories found inside RALLY_SCENARIOS ${rally.RALLY_SL_SCENARIOS}\n" +
                            "Either set PARALLEL_PERFORMANCE=false or populate ${rally.RALLY_SL_SCENARIOS} " +
                            "with component directories which include corresponding scenarios"
                        )
                    }
                    components['Stacklight'].addAll(slComponents.split('\n'))
                    common.infoMsg( "Adding for parallel execution sub dirs found in " +
                        "RALLY_SL_SCENARIOS (${rally.RALLY_SL_SCENARIOS}):"
                    )
                    print slComponents
                }

                // build up a map with tasks for parallel execution
                def allComponents = components.values().flatten()
                for (int i=0; i < allComponents.size(); i++) {
                    // randomize run so we don't bump each other at the startup
                    // also we need to let first thread create rally deployment
                    // so all the rest rally threads can use it after
                    def sleepSeconds = 15 * i

                    def task = allComponents[i]
                    def task_name = 'rally_' + task
                    def curComponent = components.find { task in it.value }.key
                    // inherit platform common data
                    def curPlatform = platform

                    // setup scenarios and stacklight switch per component
                    def commonScens = "${rally.RALLY_SCENARIOS}/${task}"
                    def stacklightScens = "${rally.RALLY_SL_SCENARIOS}/${task}"

                    switch (curComponent) {
                        case 'Common':
                            stacklightScens = ''
                            curPlatform['stacklight']['enabled'] = false
                        break
                        case 'Stacklight':
                            commonScens = ''
                            curPlatform['stacklight']['enabled'] = true
                        break
                    }

                    def curCommands = validate.runRallyTests(
                        curPlatform, commonScens,
                        stacklightScens, rally.RALLY_TASK_ARGS_FILE,
                        rally.RALLY_DB_CONN_STRING, tags,
                        rally.RALLY_TRENDS.toBoolean(), rally.SKIP_LIST,
                        generateReport
                    )

                    // copy required files for the current task
                    def taskWorkDir = "${env.WORKSPACE}/rally_" + task
                    def taskPluginsDir = "${taskWorkDir}/rally-plugins"
                    def taskScenariosDir = "${taskWorkDir}/rally-scenarios"
                    def taskArtifacts = "${taskWorkDir}/validation_artifacts"
                    def taskSecrets = "${taskWorkDir}/secrets"
                    sh "rm -rf ${taskWorkDir} || true"
                    sh "cp -ra ${workDir} ${taskWorkDir}"

                    def curCommandsList = curCommands.collectEntries{ [ (it.key) : { sh("${it.value}") } ] }
                    def curConfigRun = [
                        'image': TEST_IMAGE,
                        'baseRepoPreConfig': false,
                        'dockerMaxCpus': 2,
                        'dockerHostname': 'localhost',
                        'dockerExtraOpts': [
                            "--network=host",
                            "--entrypoint=/entrypoint.sh",
                            "-w ${rallyWorkdir}",
                            "-v ${taskWorkDir}/entrypoint.sh:/entrypoint.sh",
                            "-v ${taskPluginsDir}/:${rallyPluginsDir}",
                            "-v ${taskScenariosDir}/:${rallyScenariosDir}",
                            "-v ${taskArtifacts}/:${rallyResultsDir}",
                            "-v ${taskSecrets}/:${rallySecrets}",
                        ],
                        'envOpts'         : env_vars,
                        'runCommands'     : curCommandsList,
                    ]

                    tasksParallel['rally_' + task] = {
                        sleep sleepSeconds
                        common.infoMsg("Docker config for task $task")
                        println curConfigRun
                        common.infoMsg("Docker commands list for task $task")
                        println curCommands
                        parallelResults[task_name] = salt_testing.setupDockerAndTest(curConfigRun)
                    }
                }
            }
        }

        stage('Run Rally tests') {

            def dockerStatuses = [:]

            // start tests in Docker
            if (! rally.PARALLEL_PERFORMANCE.toBoolean()) {
                testResult = salt_testing.setupDockerAndTest(configRun)
                dockerStatuses['rally'] = (testResult) ? 'OK' : 'FAILED'
            } else {
                common.infoMsg('Jobs to run in threads: ' + tasksParallel.keySet().join(' '))
                parallel tasksParallel
                parallelResults.each { task ->
                    dockerStatuses[task.key] = (task.value) ? 'OK' : 'FAILED'
                }
            }
            // safely archiving all possible results
            dockerStatuses.each { task ->
                print "Collecting results for ${task.key} (docker status = '${task.value}')"
                try {
                    archiveArtifacts artifacts: "${task.key}/validation_artifacts/*"
                } catch (Throwable e) {
                    print 'failed to get artifacts'
                }
            }
            // setting final job status
            def failed = dockerStatuses.findAll { it.value == 'FAILED' }
            if (failed.size() == dockerStatuses.size()) {
                currentBuild.result = 'FAILURE'
            } else if (dockerStatuses.find { it.value != 'OK' }) {
                currentBuild.result = 'UNSTABLE'
            }
        }

        stage('Clean env') {
            // remove secrets
            sh 'find ./ -type d -name secrets -exec rm -rf \\\"{}\\\" \\; || true'
        }
    }
}