Merge "Revert "commitMessage is provided in the job from the EXTRA_VARIABLES_YAML""
diff --git a/src/com/mirantis/mk/Gerrit.groovy b/src/com/mirantis/mk/Gerrit.groovy
index 4d04906..51771cb 100644
--- a/src/com/mirantis/mk/Gerrit.groovy
+++ b/src/com/mirantis/mk/Gerrit.groovy
@@ -188,7 +188,7 @@
if(includeCurrentPatchset){
curPatchset = "--current-patch-set"
}
- return common.parseJSON(ssh.agentSh(String.format("ssh -p %s %s@%s gerrit query ${curPatchset} --format=JSON change:%s", gerritPort, gerritName, gerritHost, gerritChangeNumber)))
+ return common.parseJSON(ssh.agentSh(String.format("ssh -p %s %s@%s gerrit query ${curPatchset} --commit-message --format=JSON change:%s", gerritPort, gerritName, gerritHost, gerritChangeNumber)))
}
/**
@@ -552,7 +552,8 @@
}
common.infoMsg("GERRIT_EVENT_COMMENT_TEXT is ${gerritEventCommentTextStr}")
}
- if (gerritEventType == 'comment-added' && gerritEventCommentTextStr =~ /^Patch Set \d+:\s.*Workflow\+1$/) {
+ def gateComment = gerritEventCommentTextStr.split('\n')[0]
+ if (gerritEventType == 'comment-added' && gateComment =~ /^Patch Set \d+:\s.*Workflow\+1.*$/) {
common.infoMsg("Running in gate mode")
res = true
}
diff --git a/src/com/mirantis/mk/KaasUtils.groovy b/src/com/mirantis/mk/KaasUtils.groovy
index a96bd2d..0782f54 100644
--- a/src/com/mirantis/mk/KaasUtils.groovy
+++ b/src/com/mirantis/mk/KaasUtils.groovy
@@ -80,6 +80,9 @@
def runMgmtDeleteMasterTest = env.RUN_MGMT_DELETE_MASTER_TEST ? env.RUN_MGMT_DELETE_MASTER_TEST.toBoolean() : false
def runRgnlDeleteMasterTest = env.RUN_RGNL_DELETE_MASTER_TEST ? env.RUN_RGNL_DELETE_MASTER_TEST.toBoolean() : false
def runChildDeleteMasterTest = env.RUN_CHILD_DELETE_MASTER_TEST ? env.RUN_CHILD_DELETE_MASTER_TEST.toBoolean() : false
+ def runGracefulRebootTest = env.RUN_GRACEFUL_REBOOT_TEST ? env.RUN_GRACEFUL_REBOOT_TEST.toBoolean() : false
+ def pauseForDebug = env.PAUSE_FOR_DEBUG ? env.PAUSE_FOR_DEBUG.toBoolean() : false
+ def runChildMachineDeletionPolicyTest = env.RUN_CHILD_MACHINE_DELETION_POLICY_TEST ? env.RUN_CHILD_MACHINE_DELETION_POLICY_TEST.toBoolean() : false
// multiregion configuration from env variable: comma-separated string in form $mgmt_provider,$regional_provider
def multiregionalMappings = env.MULTIREGION_SETUP ? multiregionWorkflowParser(env.MULTIREGION_SETUP) : [
enabled: false,
@@ -110,6 +113,8 @@
def childOsBootFromVolume = env.OPENSTACK_BOOT_FROM_VOLUME ? env.OPENSTACK_BOOT_FROM_VOLUME.toBoolean() : false
def bootstrapV2Scenario = env.BOOTSTRAP_V2_ENABLED ? env.BOOTSTRAP_V2_ENABLED.toBoolean() : false
def equinixMetalV2Metro = env.EQUINIX_MGMT_METRO ? env.EQUINIX_MGMT_METRO : ''
+ def enableFips = env.ENABLE_FIPS ? env.ENABLE_FIPS.toBoolean() : false
+ def aioCluster = env.AIO_CLUSTER ? env.AIO_CLUSTER.toBoolean() : false
def commitMsg = env.GERRIT_CHANGE_COMMIT_MESSAGE ? new String(env.GERRIT_CHANGE_COMMIT_MESSAGE.decodeBase64()) : ''
if (commitMsg ==~ /(?s).*\[mgmt-proxy\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*mgmt-proxy.*/) {
@@ -230,6 +235,19 @@
}
if (commitMsg ==~ /(?s).*\[child-delete-master-test\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*child-delete-master-test.*/) {
runChildDeleteMasterTest = true
+ deployChild = true
+ common.infoMsg('Child cluster deployment will be enabled since delete child master node test suite will be executed')
+ }
+ if (commitMsg ==~ /(?s).*\[child-machine-deletion-policy-test\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*child-machine-deletion-policy-test.*/) {
+ runChildMachineDeletionPolicyTest = true
+ deployChild = true
+ common.infoMsg('Child cluster deployment will be enabled since machine deletion child policy test suite will be executed')
+ }
+ if (commitMsg ==~ /(?s).*\[graceful-reboot-test\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*graceful-reboot-test.*/) {
+ runGracefulRebootTest = true
+ }
+ if (commitMsg ==~ /(?s).*\[pause-for-debug\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*pause-for-debug.*/) {
+ pauseForDebug = true
}
if (commitMsg ==~ /(?s).*\[child-offline\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*child-offline.*/) {
proxyConfig['childOffline'] = true
@@ -316,6 +334,10 @@
upgradeChild = false
}
+ if (commitMsg ==~ /(?s).*\[aio-cluster\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*aio-cluster.*/) {
+ aioCluster = true
+ }
+
// multiregional tests
def multiRegionalMatches = (commitMsg =~ /(\[multiregion\s*.*?\])/)
if (multiRegionalMatches.size() > 0) {
@@ -388,6 +410,10 @@
bootstrapV2Scenario = true
}
+ if (commitMsg ==~ /(?s).*\[enable-fips\].*/ || env.GERRIT_EVENT_COMMENT_TEXT ==~ /(?s).*enable-fips\.*/) {
+ enableFips = true
+ }
+
// parse equinixmetalv2-metro trigger
def equinixMetalV2MetroMatcher = (commitMsg =~ /\[equinixmetalv2-metro(\s+.*)?\]/)
if (equinixMetalV2OnDemandDemo && equinixMetalV2MetroMatcher.size() > 0) {
@@ -423,9 +449,11 @@
Maintenance test: ${runMaintenanceTest}
Container Registry test: ${runContainerregistryTest}
Child proxy test: ${runProxyChildTest}
+ Graceful reboot test: ${runGracefulRebootTest}
Delete mgmt master node test: ${runMgmtDeleteMasterTest}
Delete rgnl master node test: ${runRgnlDeleteMasterTest}
Delete child master node test: ${runChildDeleteMasterTest}
+ Child machine deletion policy test: ${runChildMachineDeletionPolicyTest}
AWS provider deployment scheduled: ${awsOnDemandDemo}
Equinix provider deployment scheduled: ${equinixOnDemandDemo}
EquinixmetalV2 provider deployment scheduled: ${equinixMetalV2OnDemandDemo}
@@ -444,57 +472,65 @@
Service binaries fetching scheduled: ${fetchServiceBinaries}
Current weight of the demo run: ${demoWeight} (Used to manage lockable resources)
Bootstrap v2 scenario enabled: ${bootstrapV2Scenario}
+ FIPS enabled: ${enableFips}
+ Pause for debug enabled: ${pauseForDebug}
+ AIO cluster: ${aioCluster}
Triggers: https://gerrit.mcp.mirantis.com/plugins/gitiles/kaas/core/+/refs/heads/master/hack/ci-gerrit-keywords.md""")
return [
- osCloudLocation : openstackIMC,
- cdnConfig : cdnConfig,
- proxyConfig : proxyConfig,
- useMacOsSeedNode : seedMacOs,
- deployChildEnabled : deployChild,
- childDeployCustomRelease : customChildRelease,
- upgradeChildEnabled : upgradeChild,
- fullUpgradeChildEnabled : fullUpgradeChild,
- mosDeployChildEnabled : mosDeployChild,
- mosUpgradeChildEnabled : mosUpgradeChild,
- runChildConformanceEnabled : runChildConformance,
- attachBYOEnabled : attachBYO,
- upgradeBYOEnabled : upgradeBYO,
- runBYOMatrixEnabled : runBYOMatrix,
- defaultBYOOs : defaultBYOOs,
- upgradeMgmtEnabled : upgradeMgmt,
- autoUpgradeMgmtEnabled : autoUpgradeMgmt,
- enableLMALoggingEnabled : enableLMALogging,
- deployOsOnMosEnabled : deployOsOnMos,
- runUie2eEnabled : runUie2e,
- runUie2eNewEnabled : runUie2eNew,
- runMgmtConformanceEnabled : runMgmtConformance,
- runMaintenanceTestEnabled : runMaintenanceTest,
- runContainerregistryTestEnabled : runContainerregistryTest,
- runMgmtDeleteMasterTestEnabled : runMgmtDeleteMasterTest,
- runRgnlDeleteMasterTestEnabled : runRgnlDeleteMasterTest,
- runChildDeleteMasterTestEnabled : runChildDeleteMasterTest,
- runLMATestEnabled : runLMATest,
- runMgmtUserControllerTestEnabled : runMgmtUserControllerTest,
- runProxyChildTestEnabled : runProxyChildTest,
- fetchServiceBinariesEnabled : fetchServiceBinaries,
- awsOnDemandDemoEnabled : awsOnDemandDemo,
- equinixOnDemandDemoEnabled : equinixOnDemandDemo,
- equinixMetalV2OnDemandDemoEnabled : equinixMetalV2OnDemandDemo,
- equinixMetalV2ChildDiffMetroEnabled : equinixMetalV2ChildDiffMetro,
- equinixOnAwsDemoEnabled : equinixOnAwsDemo,
- azureOnDemandDemoEnabled : azureOnDemandDemo,
- azureOnAwsDemoEnabled : azureOnAwsDemo,
- vsphereDemoEnabled : enableVsphereDemo,
- vsphereOnDemandDemoEnabled : enableVsphereDemo, // TODO: remove after MCC 2.7 is out
- bmDemoEnabled : enableBMDemo,
- osDemoEnabled : enableOSDemo,
- vsphereUbuntuEnabled : enableVsphereUbuntu,
- artifactsBuildEnabled : enableArtifactsBuild,
- childOsBootFromVolume : childOsBootFromVolume,
- multiregionalConfiguration : multiregionalMappings,
- demoWeight : demoWeight,
- bootstrapV2Scenario : bootstrapV2Scenario,
- equinixMetalV2Metro : equinixMetalV2Metro]
+ osCloudLocation : openstackIMC,
+ cdnConfig : cdnConfig,
+ proxyConfig : proxyConfig,
+ useMacOsSeedNode : seedMacOs,
+ deployChildEnabled : deployChild,
+ childDeployCustomRelease : customChildRelease,
+ upgradeChildEnabled : upgradeChild,
+ fullUpgradeChildEnabled : fullUpgradeChild,
+ mosDeployChildEnabled : mosDeployChild,
+ mosUpgradeChildEnabled : mosUpgradeChild,
+ runChildConformanceEnabled : runChildConformance,
+ attachBYOEnabled : attachBYO,
+ upgradeBYOEnabled : upgradeBYO,
+ runBYOMatrixEnabled : runBYOMatrix,
+ defaultBYOOs : defaultBYOOs,
+ upgradeMgmtEnabled : upgradeMgmt,
+ autoUpgradeMgmtEnabled : autoUpgradeMgmt,
+ enableLMALoggingEnabled : enableLMALogging,
+ deployOsOnMosEnabled : deployOsOnMos,
+ runUie2eEnabled : runUie2e,
+ runUie2eNewEnabled : runUie2eNew,
+ runMgmtConformanceEnabled : runMgmtConformance,
+ runMaintenanceTestEnabled : runMaintenanceTest,
+ runContainerregistryTestEnabled : runContainerregistryTest,
+ runGracefulRebootTestEnabled : runGracefulRebootTest,
+ pauseForDebugEnabled : pauseForDebug,
+ runMgmtDeleteMasterTestEnabled : runMgmtDeleteMasterTest,
+ runRgnlDeleteMasterTestEnabled : runRgnlDeleteMasterTest,
+ runChildDeleteMasterTestEnabled : runChildDeleteMasterTest,
+ runChildMachineDeletionPolicyTestEnabled : runChildMachineDeletionPolicyTest,
+ runLMATestEnabled : runLMATest,
+ runMgmtUserControllerTestEnabled : runMgmtUserControllerTest,
+ runProxyChildTestEnabled : runProxyChildTest,
+ fetchServiceBinariesEnabled : fetchServiceBinaries,
+ awsOnDemandDemoEnabled : awsOnDemandDemo,
+ equinixOnDemandDemoEnabled : equinixOnDemandDemo,
+ equinixMetalV2OnDemandDemoEnabled : equinixMetalV2OnDemandDemo,
+ equinixMetalV2ChildDiffMetroEnabled : equinixMetalV2ChildDiffMetro,
+ equinixOnAwsDemoEnabled : equinixOnAwsDemo,
+ azureOnDemandDemoEnabled : azureOnDemandDemo,
+ azureOnAwsDemoEnabled : azureOnAwsDemo,
+ vsphereDemoEnabled : enableVsphereDemo,
+ vsphereOnDemandDemoEnabled : enableVsphereDemo, // TODO: remove after MCC 2.7 is out
+ bmDemoEnabled : enableBMDemo,
+ osDemoEnabled : enableOSDemo,
+ vsphereUbuntuEnabled : enableVsphereUbuntu,
+ artifactsBuildEnabled : enableArtifactsBuild,
+ childOsBootFromVolume : childOsBootFromVolume,
+ multiregionalConfiguration : multiregionalMappings,
+ demoWeight : demoWeight,
+ bootstrapV2Scenario : bootstrapV2Scenario,
+ equinixMetalV2Metro : equinixMetalV2Metro,
+ enableFips : enableFips,
+ aioCluster : aioCluster]
}
/**
@@ -771,9 +807,11 @@
booleanParam(name: 'RUN_MGMT_CFM', value: triggers.runMgmtConformanceEnabled),
booleanParam(name: 'RUN_MAINTENANCE_TEST', value: triggers.runMaintenanceTestEnabled),
booleanParam(name: 'RUN_CONTAINER_REGISTRY_TEST', value: triggers.runContainerregistryTestEnabled),
+ booleanParam(name: 'RUN_GRACEFUL_REBOOT_TEST', value: triggers.runGracefulRebootTestEnabled),
booleanParam(name: 'RUN_MGMT_DELETE_MASTER_TEST', value: triggers.runMgmtDeleteMasterTestEnabled),
booleanParam(name: 'RUN_RGNL_DELETE_MASTER_TEST', value: triggers.runRgnlDeleteMasterTestEnabled),
booleanParam(name: 'RUN_CHILD_DELETE_MASTER_TEST', value: triggers.runChildDeleteMasterTestEnabled),
+ booleanParam(name: 'RUN_CHILD_MACHINE_DELETION_POLICY_TEST', value: triggers.runChildMachineDeletionPolicyTestEnabled),
booleanParam(name: 'RUN_LMA_TEST', value: triggers.runLMATestEnabled),
booleanParam(name: 'RUN_MGMT_USER_CONTROLLER_TEST', value: triggers.runMgmtUserControllerTestEnabled),
booleanParam(name: 'DEPLOY_CHILD_CLUSTER', value: triggers.deployChildEnabled),
@@ -792,6 +830,9 @@
booleanParam(name: 'ALLOW_AZURE_ON_DEMAND', value: triggers.azureOnDemandDemoEnabled),
booleanParam(name: 'AZURE_ON_AWS_DEMO', value: triggers.azureOnAwsDemoEnabled),
booleanParam(name: 'VSPHERE_DEPLOY_UBUNTU', value: triggers.vsphereUbuntuEnabled),
+ booleanParam(name: 'PAUSE_FOR_DEBUG', value: triggers.pauseForDebugEnabled),
+ booleanParam(name: 'ENABLE_FIPS', value: triggers.enableFips),
+ booleanParam(name: 'AIO_CLUSTER', value: triggers.aioCluster),
]
// customize multiregional demo
@@ -1273,7 +1314,8 @@
while (!freeNodes) {
freeNodes = jenkins.model.Jenkins.instance.computers.findAll { node ->
label in node.getAssignedLabels().collect { it.name } &&
- node.isPartiallyIdle()
+ node.isPartiallyIdle() &&
+ node.isOnline()
}
if (!freeNodes) {
echo 'No nodes available for scheduling, retrying...'
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index f552fc3..74a1ce9 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -60,6 +60,7 @@
'warlock>1.3.1;python_version=="3.5"',
// NOTE: pin client packages to current latest to prevent
// downloading packages which are not support Python 2.7
+ 'keystoneauth1<=5.0.0',
'python-openstackclient==4.0.0',
'python-ironicclient==3.1.2',
'openstacksdk<0.44.0',
diff --git a/src/com/mirantis/mk/ReleaseWorkflow.groovy b/src/com/mirantis/mk/ReleaseWorkflow.groovy
index da17fb9..63a3675 100644
--- a/src/com/mirantis/mk/ReleaseWorkflow.groovy
+++ b/src/com/mirantis/mk/ReleaseWorkflow.groovy
@@ -62,6 +62,10 @@
checkoutReleaseMetadataRepo(params)
docker.image(appDockerImage).inside("--volume ${repoDir}:/workspace") {
+ checkResult = sh(script: "metadata-app --path /workspace/metadata ${opts} validate --structure", returnStatus: true)
+ if (checkResult != 0) {
+ throw new Exception("Invalid metadata structure, see errors in the metadata.log file")
+ }
result = sh(script: "metadata-app --path /workspace/metadata ${opts} get --key ${key}", returnStdout: true).trim()
}
common.infoMsg("""
@@ -106,6 +110,7 @@
String changeAuthorName = params.get('crAuthorName', 'MCP-CI')
String changeAuthorEmail = params.get('crAuthorEmail', 'mcp-ci-jenkins@ci.mcp.mirantis.net')
Boolean valuesFromFile = params.get('valuesFromFile', false)
+ Boolean catMetadataLog = params.get('catMetadataLog', false)
def cred = common.getCredentials(gitCredentialsId, 'key')
String gerritUser = cred.username
@@ -161,9 +166,19 @@
}
try {
sh "metadata-app --path /workspace/metadata update --create --key '${keyArr[i]}' ${valueExpression}"
+ checkResult = sh(script: "metadata-app --path /workspace/metadata validate --structure", returnStatus: true)
+ def MetadataLogFile = readFile("metadata.log")
+ if (MetadataLogFile.size() > 0) {
+ if (catMetadataLog) {
+ common.errorMsg("Invalid metadata structure, metadata.log file content:\n${MetadataLogFile}")
+ }
+ }
+ if (checkResult != 0) {
+ throw new Exception("Invalid metadata structure, see errors in the metadata.log file")
+ }
} finally {
- if (valuesFromFile){
- sh "rm ${tmpFile}"
+ if (valuesFromFile) {
+ sh "rm -v ${tmpFile}"
}
}
}
diff --git a/src/com/mirantis/mk/Workflow.groovy b/src/com/mirantis/mk/Workflow.groovy
index 44203ab..f231337 100644
--- a/src/com/mirantis/mk/Workflow.groovy
+++ b/src/com/mirantis/mk/Workflow.groovy
@@ -542,6 +542,24 @@
list_id += 1
}
+ def pause_step_id = list_id
+ for (step in scenario['pause']) {
+ def display_name = step['job']
+ if (step['description'] != null && step['description'].toString() != "") {
+ display_name = step['description']
+ }
+ jobs_data.add([list_id : "$list_id",
+ type : "pause",
+ name : "$display_name",
+ build_url : "0",
+ build_id : "-",
+ status : "-",
+ desc : "",
+ child_desc: "",
+ duration : '-'])
+ list_id += 1
+ }
+
def finally_step_id = list_id
for (step in scenario['finally']) {
def display_name = step['job']
@@ -559,18 +577,31 @@
duration : '-'])
list_id += 1
}
-
+ def job_failed_flag = false
try {
// Run the 'workflow' jobs
runSteps(scenario['workflow'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
} catch (InterruptedException x) {
+ job_failed_flag = true
error "The job was aborted"
} catch (e) {
+ job_failed_flag = true
error("Build failed: " + e.toString())
+
} finally {
- // Switching to 'finally' step index
+ flag_pause_variable = (env.PAUSE_FOR_DEBUG) != null
+ // Run the 'finally' or 'pause' jobs
+ common.infoMsg(failed_jobs)
+ if (flag_pause_variable && (PAUSE_FOR_DEBUG && (job_failed_flag || failed_jobs))) {
+ // Switching to 'pause' step index
+ common.infoMsg("FINALLY BLOCK - PAUSE")
+ step_id = pause_step_id
+ runSteps(scenario['pause'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
+
+ }
+ // Switching to 'finally' step index
+ common.infoMsg("FINALLY BLOCK - CLEAR")
step_id = finally_step_id
- // Run the 'finally' jobs
runSteps(scenario['finally'], global_variables, failed_jobs, jobs_data, step_id, false, artifactoryBaseUrl)
if (failed_jobs) {