blob: 03a43a6d9bacd8b432c446ca42fc60ab844f6fe4 [file] [log] [blame]
package com.mirantis.mk
/**
*
* Run a simple workflow
*
* Function runScenario() executes a sequence of jobs, like
* - Parameters for the jobs are taken from the 'env' object
* - URLs of artifacts from completed jobs may be passed
* as parameters to the next jobs.
*
* No constants, environment specific logic or other conditional dependencies.
* All the logic should be placed in the workflow jobs, and perform necessary
* actions depending on the job parameters.
* The runScenario() function only provides the
*
*/
/**
* Get Jenkins parameter names, values and types from jobName
* @param jobName job name
* @return Map with parameter names as keys and the following map as values:
* [
* <str name1>: [type: <str cls1>, use_variable: <str name1>, defaultValue: <cls value1>],
* <str name2>: [type: <str cls2>, use_variable: <str name2>, defaultValue: <cls value2>],
* ]
*/
def getJobDefaultParameters(jobName) {
def jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
def item = jenkinsUtils.getJobByName(env.JOB_NAME)
def parameters = [:]
def prop = item.getProperty(ParametersDefinitionProperty.class)
if (prop != null) {
for (param in prop.getParameterDefinitions()) {
def defaultParam = param.getDefaultParameterValue()
def cls = defaultParam.getClass().getName()
def value = defaultParam.getValue()
def name = defaultParam.getName()
parameters[name] = [type: cls, use_variable: name, defaultValue: value]
}
}
return parameters
}
/**
* Run a Jenkins job using the collected parameters
*
* @param job_name Name of the running job
* @param job_parameters Map that declares which values from global_variables should be used, in the following format:
* {'PARAM_NAME': {'type': <job parameter $class name>, 'use_variable': <a key from global_variables>}, ...}
* or
* {'PARAM_NAME': {'type': <job parameter $class name>, 'get_variable_from_url': <a key from global_variables which contains URL with required content>}, ...}
* or
* {'PARAM_NAME': {'type': <job parameter $class name>, 'use_template': <a GString multiline template with variables from global_variables>}, ...}
* or
* {'PARAM_NAME': {'type': <job parameter $class name>, 'get_variable_from_yaml': {'yaml_url': <URL with YAML content>,
* 'yaml_key': <a groovy-interpolating path to the key in the YAML, starting from dot '.'> } }, ...}
* @param global_variables Map that keeps the artifact URLs and used 'env' objects:
* {'PARAM1_NAME': <param1 value>, 'PARAM2_NAME': 'http://.../artifacts/param2_value', ...}
* @param propagate Boolean. If false: allows to collect artifacts after job is finished, even with FAILURE status
* If true: immediatelly fails the pipeline. DO NOT USE 'true' if you want to collect artifacts
* for 'finally' steps
*/
def runJob(job_name, job_parameters, global_variables, Boolean propagate = false) {
def parameters = []
def common = new com.mirantis.mk.Common()
def http = new com.mirantis.mk.Http()
def engine = new groovy.text.GStringTemplateEngine()
def template
def yamls_from_urls = [:]
def base = [:]
base["url"] = ''
def variable_content
// Collect required parameters from 'global_variables' or 'env'
for (param in job_parameters) {
if (param.value.containsKey('use_variable')) {
if (!global_variables[param.value.use_variable]) {
global_variables[param.value.use_variable] = env[param.value.use_variable] ?: ''
}
parameters.add([$class: "${param.value.type}", name: "${param.key}", value: global_variables[param.value.use_variable]])
common.infoMsg("${param.key}: <${param.value.type}> ${global_variables[param.value.use_variable]}")
} else if (param.value.containsKey('get_variable_from_url')) {
if (!global_variables[param.value.get_variable_from_url]) {
global_variables[param.value.get_variable_from_url] = env[param.value.get_variable_from_url] ?: ''
}
if (global_variables[param.value.get_variable_from_url]) {
variable_content = http.restGet(base, global_variables[param.value.get_variable_from_url]).trim()
parameters.add([$class: "${param.value.type}", name: "${param.key}", value: variable_content])
common.infoMsg("${param.key}: <${param.value.type}> ${variable_content}")
} else {
common.warningMsg("${param.key} is empty, skipping get_variable_from_url")
}
} else if (param.value.containsKey('get_variable_from_yaml')) {
if (param.value.get_variable_from_yaml.containsKey('yaml_url') && param.value.get_variable_from_yaml.containsKey('yaml_key')) {
// YAML url is stored in an environment or a global variable (like 'SI_CONFIG_ARTIFACT')
def yaml_url_var = param.value.get_variable_from_yaml.yaml_url
if (!global_variables[yaml_url_var]) {
global_variables[yaml_url_var] = env[yaml_url_var] ?: ''
}
yaml_url = global_variables[yaml_url_var] // Real YAML URL
yaml_key = param.value.get_variable_from_yaml.yaml_key
// Key to get the data from YAML, to interpolate in the groovy, for example:
// <yaml_map_variable>.key.to.the[0].required.data , where yaml_key = '.key.to.the[0].required.data'
if (yaml_url) {
if (!yamls_from_urls[yaml_url]) {
common.infoMsg("Reading YAML from ${yaml_url} for ${param.key}")
yaml_content = http.restGet(base, yaml_url)
yamls_from_urls[yaml_url] = readYaml text: yaml_content
}
common.infoMsg("Getting key ${yaml_key} from YAML ${yaml_url} for ${param.key}")
template_variables = [
'yaml_data': yamls_from_urls[yaml_url]
]
request = "\${yaml_data${yaml_key}}"
def result
// Catch errors related to wrong key or index in the list or map objects
// For wrong key in map or wrong index in list, groovy returns <null> object,
// but it can be catched only after the string interpolation <template.toString()>,
// so we should catch the string 'null' instead of object <null>.
try {
template = engine.createTemplate(request).make(template_variables)
result = template.toString()
if (result == 'null') {
error "No such key or index, got 'null'"
}
} catch (e) {
error("Failed to get the key ${yaml_key} from YAML ${yaml_url}: " + e.toString())
}
parameters.add([$class: "${param.value.type}", name: "${param.key}", value: result])
common.infoMsg("${param.key}: <${param.value.type}>\n${result}")
} else {
common.warningMsg("'yaml_url' in ${param.key} is empty, skipping get_variable_from_yaml")
}
} else {
common.warningMsg("${param.key} missing 'yaml_url'/'yaml_key' parameters, skipping get_variable_from_yaml")
}
} else if (param.value.containsKey('use_template')) {
template = engine.createTemplate(param.value.use_template).make(global_variables)
parameters.add([$class: "${param.value.type}", name: "${param.key}", value: template.toString()])
common.infoMsg("${param.key}: <${param.value.type}>\n${template.toString()}")
}
}
// Build the job
def job_info = build job: "${job_name}", parameters: parameters, propagate: propagate
return job_info
}
def runOrGetJob(job_name, job_parameters, global_variables, propagate, String fullTaskName = '') {
/**
* Run job directly or try to find already executed build
* Flow, in case CI_JOBS_OVERRIDES passed:
*
*
* CI_JOBS_OVERRIDES = text in yaml|json format
* CI_JOBS_OVERRIDES = 'kaas-testing-core-release-artifact' : 3505
* 'reindex-testing-core-release-index-with-rc' : 2822
* 'si-test-release-sanity-check-prepare-configuration': 1877
*/
common = new com.mirantis.mk.Common()
def jobsOverrides = readYaml(text: env.CI_JOBS_OVERRIDES ?: '---') ?: [:]
// get id of overriding job
def jobOverrideID = jobsOverrides.getOrDefault(fullTaskName, '')
if (fullTaskName in jobsOverrides.keySet()) {
common.warningMsg("Overriding: ${fullTaskName}/${job_name} <<< ${jobOverrideID}")
common.infoMsg("For debug pin use:\n'${fullTaskName}' : ${jobOverrideID}")
return Jenkins.instance.getItemByFullName(job_name,
hudson.model.Job.class).getBuildByNumber(jobOverrideID.toInteger())
} else {
return runJob(job_name, job_parameters, global_variables, propagate)
}
}
/**
* Store URLs of the specified artifacts to the global_variables
*
* @param build_url URL of the completed job
* @param step_artifacts Map that contains artifact names in the job, and variable names
* where the URLs to that atrifacts should be stored, for example:
* {'ARTIFACT1': 'logs.tar.gz', 'ARTIFACT2': 'test_report.xml', ...}
* @param global_variables Map that will keep the artifact URLs. Variable 'ARTIFACT1', for example,
* be used in next job parameters: {'ARTIFACT1_URL':{ 'use_variable': 'ARTIFACT1', ...}}
*
* If the artifact with the specified name not found, the parameter ARTIFACT1_URL
* will be empty.
*
*/
def storeArtifacts(build_url, step_artifacts, global_variables, job_name, build_num, artifactory_url = '') {
def common = new com.mirantis.mk.Common()
def http = new com.mirantis.mk.Http()
if (!artifactory_url) {
artifactory_url = 'https://artifactory.mcp.mirantis.net/api/storage/si-local/jenkins-job-artifacts'
}
def baseJenkins = [:]
def baseArtifactory = [:]
build_url = build_url.replaceAll(~/\/+$/, "")
baseArtifactory["url"] = artifactory_url + "/${job_name}/${build_num}"
baseJenkins["url"] = build_url
def job_config = http.restGet(baseJenkins, "/api/json/")
def job_artifacts = job_config['artifacts']
common.infoMsg("Attempt to storeArtifacts for: ${job_name}/${build_num}")
for (artifact in step_artifacts) {
try {
def artifactoryResp = http.restGet(baseArtifactory, "/${artifact.value}")
global_variables[artifact.key] = artifactoryResp.downloadUri
common.infoMsg("Artifact URL ${artifactoryResp.downloadUri} stored to ${artifact.key}")
continue
} catch (Exception e) {
common.warningMsg("Can't find an artifact in ${artifactory_url}/${job_name}/${build_num}/${artifact.value} to store in ${artifact.key}\n" +
"error code ${e.message}")
}
def job_artifact = job_artifacts.findAll { item -> artifact.value == item['fileName'] || artifact.value == item['relativePath'] }
if (job_artifact.size() == 1) {
// Store artifact URL
def artifact_url = "${build_url}/artifact/${job_artifact[0]['relativePath']}"
global_variables[artifact.key] = artifact_url
common.infoMsg("Artifact URL ${artifact_url} stored to ${artifact.key}")
} else if (job_artifact.size() > 1) {
// Error: too many artifacts with the same name, fail the job
error "Multiple artifacts ${artifact.value} for ${artifact.key} found in the build results ${build_url}, expected one:\n${job_artifact}"
} else {
// Warning: no artifact with expected name
common.warningMsg("Artifact ${artifact.value} for ${artifact.key} not found in the build results ${build_url} and in the artifactory ${artifactory_url}/${job_name}/${build_num}/, found the following artifacts in Jenkins:\n${job_artifacts}")
global_variables[artifact.key] = ''
}
}
}
/**
* Update workflow job build description
*
* @param jobs_data Map with all job names and result statuses, to showing it in description
*/
def updateDescription(jobs_data) {
def common = new com.mirantis.mk.Common()
def table = ''
def child_jobs_description = '<strong>Descriptions from jobs:</strong><br>'
def table_template_start = "<div><table style='border: solid 1px;'><tr><th>Job:</th><th>Duration:</th><th>Status:</th></tr>"
def table_template_end = "</table></div>"
for (jobdata in jobs_data) {
def trstyle = "<tr>"
// Grey background for 'finally' jobs in list
if (jobdata['type'] == 'finally') {
trstyle = "<tr style='background: #DDDDDD;'>"
}
// 'description' instead of job name if it exists
def display_name = "'${jobdata['name']}': ${jobdata['build_id']}"
if (jobdata['desc'].toString() != "") {
display_name = "'${jobdata['desc']}': ${jobdata['build_id']}"
}
// Attach url for already built jobs
def build_url = display_name
if (jobdata['build_url'] != "0") {
build_url = "<a href=${jobdata['build_url']}>$display_name</a>"
}
// Styling the status of job result
switch (jobdata['status'].toString()) {
case "SUCCESS":
status_style = "<td style='color: green;'><img src='/images/16x16/blue.png' alt='SUCCESS'>"
break
case "UNSTABLE":
status_style = "<td style='color: #FF5733;'><img src='/images/16x16/yellow.png' alt='UNSTABLE'>"
break
case "ABORTED":
status_style = "<td style='color: red;'><img src='/images/16x16/aborted.png' alt='ABORTED'>"
break
case "NOT_BUILT":
status_style = "<td style='color: red;'><img src='/images/16x16/aborted.png' alt='NOT_BUILT'>"
break
case "FAILURE":
status_style = "<td style='color: red;'><img src='/images/16x16/red.png' alt='FAILURE'>"
break
default:
status_style = "<td>-"
}
// Collect table
table += "$trstyle<td>$build_url</td><td>${jobdata['duration']}</td>$status_style</td></tr>"
// Collecting descriptions of builded child jobs
if (jobdata['child_desc'] != "") {
child_jobs_description += "<b><small><a href=${jobdata['build_url']}>- ${jobdata['name']} (${jobdata['status']}):</a></small></b><br>"
// remove "null" message-result from description, but leave XXX:JOBRESULT in description
if (jobdata['child_desc'] != "null") {
child_jobs_description += "<small>${jobdata['child_desc']}</small><br>"
}
}
}
currentBuild.description = table_template_start + table + table_template_end + child_jobs_description
}
def runStep(global_variables, step, Boolean propagate = false, artifactoryBaseUrl = '') {
return {
def common = new com.mirantis.mk.Common()
def engine = new groovy.text.GStringTemplateEngine()
String jobDescription = step['description'] ?: ''
def jobName = step['job']
def jobParameters = [:]
def stepParameters = step['parameters'] ?: [:]
if (step['inherit_parent_params'] ?: false) {
// add parameters from the current job for the child job
jobParameters << getJobDefaultParameters(env.JOB_NAME)
}
// add parameters from the workflow for the child job
jobParameters << stepParameters
def wfPauseStepBeforeRun = (step['wf_pause_step_before_run'] ?: false).toBoolean()
def wfPauseStepTimeout = (step['wf_pause_step_timeout'] ?: 10).toInteger()
def wfPauseStepSlackReportChannel = step['wf_pause_step_slack_report_channel'] ?: ''
if (wfPauseStepBeforeRun) {
// Try-catch construction will allow to continue Steps, if timeout reached
try {
if (wfPauseStepSlackReportChannel) {
def slack = new com.mirantis.mcp.SlackNotification()
slack.jobResultNotification('wf_pause_step_before_run',
wfPauseStepSlackReportChannel,
env.JOB_NAME, null,
env.BUILD_URL, 'slack_webhook_url')
}
timeout(time: wfPauseStepTimeout, unit: 'MINUTES') {
input("Workflow pause requested before run: ${jobName}/${jobDescription}\n" +
"Timeout set to ${wfPauseStepTimeout}.\n" +
"Do you want to proceed workflow?")
}
} catch (err) { // timeout reached or input false
def user = err.getCauses()[0].getUser()
if (user.toString() != 'SYSTEM') { // SYSTEM means timeout.
error("Aborted after workFlow pause by: [${user}]")
} else {
common.infoMsg("Timeout finished, continue..")
}
}
}
common.infoMsg("Attempt to run: ${jobName}/${jobDescription}")
// Collect job parameters and run the job
// WARN(alexz): desc must not contain invalid chars for yaml
def jobResult = runOrGetJob(jobName, jobParameters,
global_variables, propagate, jobDescription)
def buildDuration = jobResult.durationString ?: '-'
if (buildDuration.toString() == null) {
buildDuration = '-'
}
def jobSummary = [
job_result : jobResult.getResult().toString(),
build_url : jobResult.getAbsoluteUrl().toString(),
build_id : jobResult.getId().toString(),
buildDuration : buildDuration,
desc : engine.createTemplate(jobDescription).make(global_variables),
]
def _buildDescription = jobResult.getDescription().toString()
if(_buildDescription){
jobSummary['build_description'] = _buildDescription
}
// Store links to the resulting artifacts into 'global_variables'
storeArtifacts(jobSummary['build_url'], step['artifacts'],
global_variables, jobName, jobSummary['build_id'], artifactoryBaseUrl)
return jobSummary
}
}
/**
* Run the workflow or final steps one by one
*
* @param steps List of steps (Jenkins jobs) to execute
* @param global_variables Map where the collected artifact URLs and 'env' objects are stored
* @param failed_jobs Map with failed job names and result statuses, to report it later
* @param jobs_data Map with all job names and result statuses, to showing it in description
* @param step_id Counter for matching step ID with cell ID in description table
* @param propagate Boolean. If false: allows to collect artifacts after job is finished, even with FAILURE status
* If true: immediatelly fails the pipeline. DO NOT USE 'true' with runScenario().
*/
def runSteps(steps, global_variables, failed_jobs, jobs_data, step_id, Boolean propagate = false, artifactoryBaseUrl = '') {
common = new com.mirantis.mk.Common()
// Show expected jobs list in description
updateDescription(jobs_data)
for (step in steps) {
stage("Preparing for run job ${step['job']}") {
def job_summary = runStep(global_variables, step, propagate, artifactoryBaseUrl).call()
// Update jobs_data for updating description
jobs_data[step_id]['build_url'] = job_summary['build_url']
jobs_data[step_id]['build_id'] = job_summary['build_id']
jobs_data[step_id]['status'] = job_summary['job_result']
jobs_data[step_id]['duration'] = job_summary['buildDuration']
jobs_data[step_id]['desc'] = job_summary['desc']
if (job_summary['build_description']) {
jobs_data[step_id]['child_desc'] = job_summary['build_description']
}
updateDescription(jobs_data)
def job_result = job_summary['job_result']
def build_url = job_summary['build_url']
// Check job result, in case of SUCCESS, move to next step.
// In case job has status NOT_BUILT, fail the build or keep going depending on 'ignore_not_built' flag
// In other cases check flag ignore_failed, if true ignore any statuses and keep going additionally
// if skip_results is not set or set to false fail entrie workflow, otherwise succed.
if (job_result != 'SUCCESS') {
def ignoreStepResult = false
switch (job_result) {
// In cases when job was waiting too long in queue or internal job logic allows to skip building,
// job may have NOT_BUILT status. In that case ignore_not_built flag can be used not to fail scenario.
case "NOT_BUILT":
ignoreStepResult = step['ignore_not_built'] ?: false
break;
case "UNSTABLE":
ignoreStepResult = step['ignore_unstable'] ?: (step['ignore_failed'] ?: false)
break;
default:
ignoreStepResult = step['ignore_failed'] ?: false
if (ignoreStepResult && !step['skip_results'] ?: false) {
failed_jobs[build_url] = job_result
}
}
if (!ignoreStepResult) {
currentBuild.result = job_result
error "Job ${build_url} finished with result: ${job_result}"
}
}
common.infoMsg("Job ${build_url} finished with result: ${job_result}")
}
// Jump to next ID for updating next job data in description table
step_id++
}
}
/**
* Run the workflow scenario
*
* @param scenario: Map with scenario steps.
* There are two keys in the scenario:
* workflow: contains steps to run deploy and test jobs
* finally: contains steps to run report and cleanup jobs
*
* Scenario execution example:
*
* scenario_yaml = """\
* workflow:
* - job: deploy-kaas
* ignore_failed: false
* description: "Management cluster ${KAAS_VERSION}"
* parameters:
* KAAS_VERSION:
* type: StringParameterValue
* use_variable: KAAS_VERSION
* artifacts:
* KUBECONFIG_ARTIFACT: artifacts/management_kubeconfig
* DEPLOYED_KAAS_VERSION: artifacts/management_version
*
* - job: create-child
* inherit_parent_params: true
* ignore_failed: false
* parameters:
* KUBECONFIG_ARTIFACT_URL:
* type: StringParameterValue
* use_variable: KUBECONFIG_ARTIFACT
* KAAS_VERSION:
* type: StringParameterValue
* get_variable_from_url: DEPLOYED_KAAS_VERSION
* RELEASE_NAME:
* type: StringParameterValue
* get_variable_from_yaml:
* yaml_url: SI_CONFIG_ARTIFACT
* yaml_key: .clusters[0].release_name
*
* - job: test-kaas-ui
* ignore_not_built: false
* parameters:
* KUBECONFIG_ARTIFACT_URL:
* type: StringParameterValue
* use_variable: KUBECONFIG_ARTIFACT
* KAAS_VERSION:
* type: StringParameterValue
* get_variable_from_url: DEPLOYED_KAAS_VERSION
* artifacts:
* REPORT_SI_KAAS_UI: artifacts/test_kaas_ui_result.xml
* finally:
* - job: testrail-report
* ignore_failed: true
* parameters:
* KAAS_VERSION:
* type: StringParameterValue
* get_variable_from_url: DEPLOYED_KAAS_VERSION
* REPORTS_LIST:
* type: TextParameterValue
* use_template: |
* REPORT_SI_KAAS_UI: \$REPORT_SI_KAAS_UI
* """
*
* runScenario(scenario)
*
* Scenario workflow keys:
*
* job: string. Jenkins job name
* ignore_failed: bool. if true, keep running the workflow jobs if the job is failed, but fail the workflow at finish
* skip_results: bool. if true, keep running the workflow jobs if the job is failed, but do not fail the workflow at finish. Makes sense only when ignore_failed is set.
* ignore_not_built: bool. if true, keep running the workflow jobs if the job set own status to NOT_BUILT, do not fail the workflow at finish for such jobs
* inherit_parent_params: bool. if true, provide all parameters from the parent job to the child job as defaults
* parameters: dict. parameters name and type to inherit from parent to child job, or from artifact to child job
* wf_pause_step_before_run: bool. Interactive pause exact step before run.
* wf_pause_step_slack_report_channel: If step paused, send message about it in slack.
* wf_pause_step_timeout: timeout im minutes to wait for manual unpause.
*/
def runScenario(scenario, slackReportChannel = '', artifactoryBaseUrl = '') {
// Clear description before adding new messages
currentBuild.description = ''
// Collect the parameters for the jobs here
def global_variables = [:]
// List of failed jobs to show at the end
def failed_jobs = [:]
// Jobs data to use for wf job build description
def jobs_data = []
// Counter for matching step ID with cell ID in description table
def step_id = 0
// Generate expected list jobs for description
def list_id = 0
for (step in scenario['workflow']) {
def display_name = step['job']
if (step['description'] != null && step['description'].toString() != "") {
display_name = step['description']
}
jobs_data.add([list_id : "$list_id",
type : "workflow",
name : "$display_name",
build_url : "0",
build_id : "-",
status : "-",
desc : "",
child_desc: "",
duration : '-'])
list_id += 1
}
def finally_step_id = list_id
for (step in scenario['finally']) {
def display_name = step['job']
if (step['description'] != null && step['description'].toString() != "") {
display_name = step['description']
}
jobs_data.add([list_id : "$list_id",
type : "finally",
name : "$display_name",
build_url : "0",
build_id : "-",
status : "-",
desc : "",
child_desc: "",
duration : '-'])
list_id += 1
}
try {
// Run the 'workflow' jobs
runSteps(scenario['workflow'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
} catch (InterruptedException x) {
error "The job was aborted"
} catch (e) {
error("Build failed: " + e.toString())
} finally {
// Switching to 'finally' step index
step_id = finally_step_id
// Run the 'finally' jobs
runSteps(scenario['finally'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
if (failed_jobs) {
def statuses = []
failed_jobs.each {
statuses += it.value
}
if (statuses.contains('FAILURE')) {
currentBuild.result = 'FAILURE'
} else if (statuses.contains('UNSTABLE')) {
currentBuild.result = 'UNSTABLE'
} else {
currentBuild.result = 'FAILURE'
}
println "Failed jobs: ${failed_jobs}"
} else {
currentBuild.result = 'SUCCESS'
}
if (slackReportChannel) {
def slack = new com.mirantis.mcp.SlackNotification()
slack.jobResultNotification(currentBuild.result, slackReportChannel, '', null, '', 'slack_webhook_url')
}
} // finally
}