0change autoident
Change-Id: Ib16dd609654b5fadad65669c8e38dce9fee54f0f
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index aaba422..90640cb 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -15,7 +15,7 @@
def reclassVersion = 'v1.5.4'
if (common.validInputParam('RECLASS_VERSION')) {
- reclassVersion = RECLASS_VERSION
+ reclassVersion = RECLASS_VERSION
}
slaveNode = (env.SLAVE_NODE ?: 'python&&docker')
@@ -25,153 +25,153 @@
// options from CC contexts
// currently, just mix them together in one set
def testCfg01ExtraFormulas = 'glusterfs jenkins logrotate maas ntp rsyslog fluentd telegraf prometheus ' +
- 'grafana backupninja'
+ 'grafana backupninja'
timeout(time: 2, unit: 'HOURS') {
- node(slaveNode) {
- def templateEnv = "${env.WORKSPACE}/template"
- def modelEnv = "${env.WORKSPACE}/model"
- def testEnv = "${env.WORKSPACE}/test"
- def pipelineEnv = "${env.WORKSPACE}/pipelines"
+ node(slaveNode) {
+ def templateEnv = "${env.WORKSPACE}/template"
+ def modelEnv = "${env.WORKSPACE}/model"
+ def testEnv = "${env.WORKSPACE}/test"
+ def pipelineEnv = "${env.WORKSPACE}/pipelines"
- try {
- def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
- def mcpVersion = templateContext.default_context.mcp_version
- def sharedReclassUrl = templateContext.default_context.shared_reclass_url
- def clusterDomain = templateContext.default_context.cluster_domain
- def clusterName = templateContext.default_context.cluster_name
- def saltMaster = templateContext.default_context.salt_master_hostname
- def localRepositories = templateContext.default_context.local_repositories.toBoolean()
- def offlineDeployment = templateContext.default_context.offline_deployment.toBoolean()
- def cutterEnv = "${env.WORKSPACE}/cutter"
- def jinjaEnv = "${env.WORKSPACE}/jinja"
- def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
- def systemEnv = "${modelEnv}/classes/system"
- def targetBranch = "feature/${clusterName}"
- def templateBaseDir = "${env.WORKSPACE}/template"
- def templateDir = "${templateEnv}/template/dir"
- def templateOutputDir = templateBaseDir
- def user
- def testResult = false
- wrap([$class: 'BuildUser']) {
- user = env.BUILD_USER_ID
- }
-
- if (mcpVersion != '2018.4.0') {
- testCfg01ExtraFormulas += ' auditd'
- }
-
- currentBuild.description = clusterName
- print("Using context:\n" + COOKIECUTTER_TEMPLATE_CONTEXT)
-
- stage('Download Cookiecutter template') {
- sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
- def cookiecutterTemplateUrl = templateContext.default_context.cookiecutter_template_url
- def cookiecutterTemplateBranch = templateContext.default_context.cookiecutter_template_branch
- git.checkoutGitRepository(templateEnv, cookiecutterTemplateUrl, 'master')
- // Use refspec if exists first of all
- if (cookiecutterTemplateBranch.toString().startsWith('refs/')) {
- dir(templateEnv) {
- ssh.agentSh("git fetch ${cookiecutterTemplateUrl} ${cookiecutterTemplateBranch} && git checkout FETCH_HEAD")
- }
- } else {
- // Use mcpVersion git tag if not specified branch for cookiecutter-templates
- if (cookiecutterTemplateBranch == '') {
- cookiecutterTemplateBranch = mcpVersion
- // Don't have nightly/testing/stable for cookiecutter-templates repo, therefore use master
- if ([ "nightly" , "testing", "stable" ].contains(mcpVersion)) {
- cookiecutterTemplateBranch = 'master'
+ try {
+ def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
+ def mcpVersion = templateContext.default_context.mcp_version
+ def sharedReclassUrl = templateContext.default_context.shared_reclass_url
+ def clusterDomain = templateContext.default_context.cluster_domain
+ def clusterName = templateContext.default_context.cluster_name
+ def saltMaster = templateContext.default_context.salt_master_hostname
+ def localRepositories = templateContext.default_context.local_repositories.toBoolean()
+ def offlineDeployment = templateContext.default_context.offline_deployment.toBoolean()
+ def cutterEnv = "${env.WORKSPACE}/cutter"
+ def jinjaEnv = "${env.WORKSPACE}/jinja"
+ def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
+ def systemEnv = "${modelEnv}/classes/system"
+ def targetBranch = "feature/${clusterName}"
+ def templateBaseDir = "${env.WORKSPACE}/template"
+ def templateDir = "${templateEnv}/template/dir"
+ def templateOutputDir = templateBaseDir
+ def user
+ def testResult = false
+ wrap([$class: 'BuildUser']) {
+ user = env.BUILD_USER_ID
}
- }
- git.changeGitBranch(templateEnv, cookiecutterTemplateBranch)
- }
- }
- stage('Create empty reclass model') {
- dir(path: modelEnv) {
- sh "rm -rfv .git"
- sh "git init"
- ssh.agentSh("git submodule add ${sharedReclassUrl} 'classes/system'")
- }
-
- def sharedReclassBranch = templateContext.default_context.shared_reclass_branch
- // Use refspec if exists first of all
- if (sharedReclassBranch.toString().startsWith('refs/')) {
- dir(systemEnv) {
- ssh.agentSh("git fetch ${sharedReclassUrl} ${sharedReclassBranch} && git checkout FETCH_HEAD")
- }
- } else {
- // Use mcpVersion git tag if not specified branch for reclass-system
- if (sharedReclassBranch == '') {
- sharedReclassBranch = mcpVersion
- // Don't have nightly/testing for reclass-system repo, therefore use master
- if ([ "nightly" , "testing", "stable" ].contains(mcpVersion)) {
- common.warningMsg("Fetching reclass-system from master!")
- sharedReclassBranch = 'master'
+ if (mcpVersion != '2018.4.0') {
+ testCfg01ExtraFormulas += ' auditd'
}
- }
- git.changeGitBranch(systemEnv, sharedReclassBranch)
- }
- git.commitGitChanges(modelEnv, "Added new shared reclass submodule", "${user}@localhost", "${user}")
- }
- def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
- for (product in productList) {
+ currentBuild.description = clusterName
+ print("Using context:\n" + COOKIECUTTER_TEMPLATE_CONTEXT)
- // get templateOutputDir and productDir
- if (product.startsWith("stacklight")) {
- templateOutputDir = "${env.WORKSPACE}/output/stacklight"
+ stage('Download Cookiecutter template') {
+ sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
+ def cookiecutterTemplateUrl = templateContext.default_context.cookiecutter_template_url
+ def cookiecutterTemplateBranch = templateContext.default_context.cookiecutter_template_branch
+ git.checkoutGitRepository(templateEnv, cookiecutterTemplateUrl, 'master')
+ // Use refspec if exists first of all
+ if (cookiecutterTemplateBranch.toString().startsWith('refs/')) {
+ dir(templateEnv) {
+ ssh.agentSh("git fetch ${cookiecutterTemplateUrl} ${cookiecutterTemplateBranch} && git checkout FETCH_HEAD")
+ }
+ } else {
+ // Use mcpVersion git tag if not specified branch for cookiecutter-templates
+ if (cookiecutterTemplateBranch == '') {
+ cookiecutterTemplateBranch = mcpVersion
+ // Don't have nightly/testing/stable for cookiecutter-templates repo, therefore use master
+ if (["nightly", "testing", "stable"].contains(mcpVersion)) {
+ cookiecutterTemplateBranch = 'master'
+ }
+ }
+ git.changeGitBranch(templateEnv, cookiecutterTemplateBranch)
+ }
+ }
- def stacklightVersion
- try {
- stacklightVersion = templateContext.default_context['stacklight_version']
- } catch (Throwable e) {
- common.warningMsg('Stacklight version loading failed')
- }
+ stage('Create empty reclass model') {
+ dir(path: modelEnv) {
+ sh "rm -rfv .git"
+ sh "git init"
+ ssh.agentSh("git submodule add ${sharedReclassUrl} 'classes/system'")
+ }
- if (stacklightVersion) {
- productDir = "stacklight" + stacklightVersion
- } else {
- productDir = "stacklight1"
- }
+ def sharedReclassBranch = templateContext.default_context.shared_reclass_branch
+ // Use refspec if exists first of all
+ if (sharedReclassBranch.toString().startsWith('refs/')) {
+ dir(systemEnv) {
+ ssh.agentSh("git fetch ${sharedReclassUrl} ${sharedReclassBranch} && git checkout FETCH_HEAD")
+ }
+ } else {
+ // Use mcpVersion git tag if not specified branch for reclass-system
+ if (sharedReclassBranch == '') {
+ sharedReclassBranch = mcpVersion
+ // Don't have nightly/testing for reclass-system repo, therefore use master
+ if (["nightly", "testing", "stable"].contains(mcpVersion)) {
+ common.warningMsg("Fetching reclass-system from master!")
+ sharedReclassBranch = 'master'
+ }
+ }
+ git.changeGitBranch(systemEnv, sharedReclassBranch)
+ }
+ git.commitGitChanges(modelEnv, "Added new shared reclass submodule", "${user}@localhost", "${user}")
+ }
- } else {
- templateOutputDir = "${env.WORKSPACE}/output/${product}"
- productDir = product
- }
+ def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
+ for (product in productList) {
- if (product == "infra" || (templateContext.default_context["${product}_enabled"]
- && templateContext.default_context["${product}_enabled"].toBoolean())) {
+ // get templateOutputDir and productDir
+ if (product.startsWith("stacklight")) {
+ templateOutputDir = "${env.WORKSPACE}/output/stacklight"
- templateDir = "${templateEnv}/cluster_product/${productDir}"
- common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
+ def stacklightVersion
+ try {
+ stacklightVersion = templateContext.default_context['stacklight_version']
+ } catch (Throwable e) {
+ common.warningMsg('Stacklight version loading failed')
+ }
- sh "rm -rf ${templateOutputDir} || true"
- sh "mkdir -p ${templateOutputDir}"
- sh "mkdir -p ${outputDestination}"
+ if (stacklightVersion) {
+ productDir = "stacklight" + stacklightVersion
+ } else {
+ productDir = "stacklight1"
+ }
- python.setupCookiecutterVirtualenv(cutterEnv)
- python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
- sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
- } else {
- common.warningMsg("Product " + product + " is disabled")
- }
- }
+ } else {
+ templateOutputDir = "${env.WORKSPACE}/output/${product}"
+ productDir = product
+ }
- if (localRepositories && !offlineDeployment) {
- def aptlyModelUrl = templateContext.default_context.local_model_url
- dir(path: modelEnv) {
- ssh.agentSh "git submodule add \"${aptlyModelUrl}\" \"classes/cluster/${clusterName}/cicd/aptly\""
- if (!(mcpVersion in ["nightly", "testing", "stable"])) {
- ssh.agentSh "cd \"classes/cluster/${clusterName}/cicd/aptly\";git fetch --tags;git checkout ${mcpVersion}"
- }
- }
- }
+ if (product == "infra" || (templateContext.default_context["${product}_enabled"]
+ && templateContext.default_context["${product}_enabled"].toBoolean())) {
- stage('Generate new SaltMaster node') {
- def nodeFile = "${modelEnv}/nodes/${saltMaster}.${clusterDomain}.yml"
- def nodeString = """classes:
+ templateDir = "${templateEnv}/cluster_product/${productDir}"
+ common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
+
+ sh "rm -rf ${templateOutputDir} || true"
+ sh "mkdir -p ${templateOutputDir}"
+ sh "mkdir -p ${outputDestination}"
+
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
+ sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
+ } else {
+ common.warningMsg("Product " + product + " is disabled")
+ }
+ }
+
+ if (localRepositories && !offlineDeployment) {
+ def aptlyModelUrl = templateContext.default_context.local_model_url
+ dir(path: modelEnv) {
+ ssh.agentSh "git submodule add \"${aptlyModelUrl}\" \"classes/cluster/${clusterName}/cicd/aptly\""
+ if (!(mcpVersion in ["nightly", "testing", "stable"])) {
+ ssh.agentSh "cd \"classes/cluster/${clusterName}/cicd/aptly\";git fetch --tags;git checkout ${mcpVersion}"
+ }
+ }
+ }
+
+ stage('Generate new SaltMaster node') {
+ def nodeFile = "${modelEnv}/nodes/${saltMaster}.${clusterDomain}.yml"
+ def nodeString = """classes:
- cluster.${clusterName}.infra.config
parameters:
_param:
@@ -182,160 +182,160 @@
name: ${saltMaster}
domain: ${clusterDomain}
"""
- sh "mkdir -p ${modelEnv}/nodes/"
- writeFile(file: nodeFile, text: nodeString)
+ sh "mkdir -p ${modelEnv}/nodes/"
+ writeFile(file: nodeFile, text: nodeString)
- git.commitGitChanges(modelEnv, "Create model ${clusterName}", "${user}@localhost", "${user}")
- }
+ git.commitGitChanges(modelEnv, "Create model ${clusterName}", "${user}@localhost", "${user}")
+ }
- stage("Test") {
- if (TEST_MODEL.toBoolean() && sharedReclassUrl != '') {
- sh("cp -r ${modelEnv} ${testEnv}")
- def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
- common.infoMsg("Attempt to run test against formula-version: ${mcpVersion}")
- testResult = saltModelTesting.setupAndTestNode(
- "${saltMaster}.${clusterDomain}",
- "",
- testCfg01ExtraFormulas,
- testEnv,
- 'pkg',
- mcpVersion,
- reclassVersion,
- 0,
- false,
- false,
- '',
- '',
- DockerCName)
- if (testResult) {
- common.infoMsg("Test finished: SUCCESS")
- } else {
- common.warningMsg('Test finished: FAILURE')
- }
- } else {
- common.warningMsg("Test stage has been skipped!")
+ stage("Test") {
+ if (TEST_MODEL.toBoolean() && sharedReclassUrl != '') {
+ sh("cp -r ${modelEnv} ${testEnv}")
+ def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
+ common.infoMsg("Attempt to run test against formula-version: ${mcpVersion}")
+ testResult = saltModelTesting.setupAndTestNode(
+ "${saltMaster}.${clusterDomain}",
+ "",
+ testCfg01ExtraFormulas,
+ testEnv,
+ 'pkg',
+ mcpVersion,
+ reclassVersion,
+ 0,
+ false,
+ false,
+ '',
+ '',
+ DockerCName)
+ if (testResult) {
+ common.infoMsg("Test finished: SUCCESS")
+ } else {
+ common.warningMsg('Test finished: FAILURE')
+ }
+ } else {
+ common.warningMsg("Test stage has been skipped!")
+ }
+ }
+ stage("Generate config drives") {
+ // apt package genisoimage is required for this stage
+
+ // download create-config-drive
+ // FIXME: that should be refactored, to use git clone - to be able download it from custom repo.
+ def mcpCommonScriptsBranch = templateContext.default_context.mcp_common_scripts_branch
+ if (mcpCommonScriptsBranch == '') {
+ mcpCommonScriptsBranch = mcpVersion
+ // Don't have n/t/s for mcp-common-scripts repo, therefore use master
+ if (["nightly", "testing", "stable"].contains(mcpVersion)) {
+ common.warningMsg("Fetching mcp-common-scripts from master!")
+ mcpCommonScriptsBranch = 'master'
+ }
+ }
+ def config_drive_script_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/${mcpCommonScriptsBranch}/config-drive/create_config_drive.sh"
+ def user_data_script_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/${mcpCommonScriptsBranch}/config-drive/master_config.sh"
+ common.retry(3, 5) {
+ sh "wget -O create-config-drive ${config_drive_script_url} && chmod +x create-config-drive"
+ sh "wget -O user_data.sh ${user_data_script_url}"
+ }
+
+ sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
+ sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
+ args = "--user-data user_data.sh --hostname ${saltMaster} --model ${modelEnv} --mk-pipelines ${pipelineEnv}/mk-pipelines/ --pipeline-library ${pipelineEnv}/pipeline-library/ ${saltMaster}.${clusterDomain}-config.iso"
+
+ // load data from model
+ def smc = [:]
+ smc['SALT_MASTER_MINION_ID'] = "${saltMaster}.${clusterDomain}"
+ smc['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
+ smc['DEPLOY_NETWORK_GW'] = templateContext['default_context']['deploy_network_gateway']
+ smc['DEPLOY_NETWORK_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
+ if (templateContext['default_context'].get('deploy_network_mtu')) {
+ smc['DEPLOY_NETWORK_MTU'] = templateContext['default_context']['deploy_network_mtu']
+ }
+ smc['DNS_SERVERS'] = templateContext['default_context']['dns_server01']
+ smc['MCP_VERSION'] = "${mcpVersion}"
+ if (templateContext['default_context']['local_repositories'] == 'True') {
+ def localRepoIP = templateContext['default_context']['local_repo_url']
+ smc['MCP_SALT_REPO_KEY'] = "http://${localRepoIP}/public.gpg"
+ smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}/ubuntu-xenial"
+ smc['PIPELINES_FROM_ISO'] = 'false'
+ smc['PIPELINE_REPO_URL'] = "http://${localRepoIP}:8088"
+ smc['LOCAL_REPOS'] = 'true'
+ }
+ if (templateContext['default_context']['upstream_proxy_enabled'] == 'True') {
+ if (templateContext['default_context']['upstream_proxy_auth_enabled'] == 'True') {
+ smc['http_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_user'] + ':' + templateContext['default_context']['upstream_proxy_password'] + '@' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
+ smc['https_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_user'] + ':' + templateContext['default_context']['upstream_proxy_password'] + '@' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
+ } else {
+ smc['http_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
+ smc['https_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
+ }
+ }
+
+ for (i in common.entries(smc)) {
+ sh "sed -i 's,export ${i[0]}=.*,export ${i[0]}=${i[1]},' user_data.sh"
+ }
+
+ // create cfg config-drive
+ sh "./create-config-drive ${args}"
+ sh("mkdir output-${clusterName} && mv ${saltMaster}.${clusterDomain}-config.iso output-${clusterName}/")
+
+ // save cfg iso to artifacts
+ archiveArtifacts artifacts: "output-${clusterName}/${saltMaster}.${clusterDomain}-config.iso"
+
+ if (templateContext['default_context']['local_repositories'] == 'True') {
+ def aptlyServerHostname = templateContext.default_context.aptly_server_hostname
+ def user_data_script_apt_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/master/config-drive/mirror_config.sh"
+ sh "wget -O mirror_config.sh ${user_data_script_apt_url}"
+
+ def smc_apt = [:]
+ smc_apt['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
+ smc_apt['APTLY_DEPLOY_IP'] = templateContext['default_context']['aptly_server_deploy_address']
+ smc_apt['APTLY_DEPLOY_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
+ smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${clusterDomain}"
+
+ for (i in common.entries(smc_apt)) {
+ sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config.sh"
+ }
+
+ // create apt config-drive
+ sh "./create-config-drive --user-data mirror_config.sh --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${clusterDomain}-config.iso"
+ sh("mv ${aptlyServerHostname}.${clusterDomain}-config.iso output-${clusterName}/")
+
+ // save apt iso to artifacts
+ archiveArtifacts artifacts: "output-${clusterName}/${aptlyServerHostname}.${clusterDomain}-config.iso"
+ }
+ }
+
+ stage('Save changes reclass model') {
+ sh(returnStatus: true, script: "tar -czf output-${clusterName}/${clusterName}.tar.gz --exclude='*@tmp' -C ${modelEnv} .")
+ archiveArtifacts artifacts: "output-${clusterName}/${clusterName}.tar.gz"
+
+
+ if (EMAIL_ADDRESS != null && EMAIL_ADDRESS != "") {
+ emailext(to: EMAIL_ADDRESS,
+ attachmentsPattern: "output-${clusterName}/*",
+ body: "Mirantis Jenkins\n\nRequested reclass model ${clusterName} has been created and attached to this email.\nEnjoy!\n\nMirantis",
+ subject: "Your Salt model ${clusterName}")
+ }
+ dir("output-${clusterName}") {
+ deleteDir()
+ }
+ }
+
+ // Fail, but leave possibility to get failed artifacts
+ if (!testResult && TEST_MODEL.toBoolean()) {
+ common.warningMsg('Test finished: FAILURE. Please check logs and\\or debug failed model manually!')
+ error('Test stage finished: FAILURE')
+ }
+
+ } catch (Throwable e) {
+ currentBuild.result = "FAILURE"
+ currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+ throw e
+ } finally {
+ stage('Clean workspace directories') {
+ sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
+ }
+ // common.sendNotification(currentBuild.result,"",["slack"])
}
- }
- stage("Generate config drives") {
- // apt package genisoimage is required for this stage
-
- // download create-config-drive
- // FIXME: that should be refactored, to use git clone - to be able download it from custom repo.
- def mcpCommonScriptsBranch = templateContext.default_context.mcp_common_scripts_branch
- if (mcpCommonScriptsBranch == '') {
- mcpCommonScriptsBranch = mcpVersion
- // Don't have n/t/s for mcp-common-scripts repo, therefore use master
- if ([ "nightly" , "testing", "stable" ].contains(mcpVersion)) {
- common.warningMsg("Fetching mcp-common-scripts from master!")
- mcpCommonScriptsBranch = 'master'
- }
- }
- def config_drive_script_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/${mcpCommonScriptsBranch}/config-drive/create_config_drive.sh"
- def user_data_script_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/${mcpCommonScriptsBranch}/config-drive/master_config.sh"
- common.retry(3, 5) {
- sh "wget -O create-config-drive ${config_drive_script_url} && chmod +x create-config-drive"
- sh "wget -O user_data.sh ${user_data_script_url}"
- }
-
- sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
- sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
- args = "--user-data user_data.sh --hostname ${saltMaster} --model ${modelEnv} --mk-pipelines ${pipelineEnv}/mk-pipelines/ --pipeline-library ${pipelineEnv}/pipeline-library/ ${saltMaster}.${clusterDomain}-config.iso"
-
- // load data from model
- def smc = [:]
- smc['SALT_MASTER_MINION_ID'] = "${saltMaster}.${clusterDomain}"
- smc['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
- smc['DEPLOY_NETWORK_GW'] = templateContext['default_context']['deploy_network_gateway']
- smc['DEPLOY_NETWORK_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
- if (templateContext['default_context'].get('deploy_network_mtu')) {
- smc['DEPLOY_NETWORK_MTU'] = templateContext['default_context']['deploy_network_mtu']
- }
- smc['DNS_SERVERS'] = templateContext['default_context']['dns_server01']
- smc['MCP_VERSION'] = "${mcpVersion}"
- if (templateContext['default_context']['local_repositories'] == 'True') {
- def localRepoIP = templateContext['default_context']['local_repo_url']
- smc['MCP_SALT_REPO_KEY'] = "http://${localRepoIP}/public.gpg"
- smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}/ubuntu-xenial"
- smc['PIPELINES_FROM_ISO'] = 'false'
- smc['PIPELINE_REPO_URL'] = "http://${localRepoIP}:8088"
- smc['LOCAL_REPOS'] = 'true'
- }
- if (templateContext['default_context']['upstream_proxy_enabled'] == 'True') {
- if (templateContext['default_context']['upstream_proxy_auth_enabled'] == 'True') {
- smc['http_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_user'] + ':' + templateContext['default_context']['upstream_proxy_password'] + '@' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
- smc['https_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_user'] + ':' + templateContext['default_context']['upstream_proxy_password'] + '@' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
- } else {
- smc['http_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
- smc['https_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
- }
- }
-
- for (i in common.entries(smc)) {
- sh "sed -i 's,export ${i[0]}=.*,export ${i[0]}=${i[1]},' user_data.sh"
- }
-
- // create cfg config-drive
- sh "./create-config-drive ${args}"
- sh("mkdir output-${clusterName} && mv ${saltMaster}.${clusterDomain}-config.iso output-${clusterName}/")
-
- // save cfg iso to artifacts
- archiveArtifacts artifacts: "output-${clusterName}/${saltMaster}.${clusterDomain}-config.iso"
-
- if (templateContext['default_context']['local_repositories'] == 'True') {
- def aptlyServerHostname = templateContext.default_context.aptly_server_hostname
- def user_data_script_apt_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/master/config-drive/mirror_config.sh"
- sh "wget -O mirror_config.sh ${user_data_script_apt_url}"
-
- def smc_apt = [:]
- smc_apt['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
- smc_apt['APTLY_DEPLOY_IP'] = templateContext['default_context']['aptly_server_deploy_address']
- smc_apt['APTLY_DEPLOY_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
- smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${clusterDomain}"
-
- for (i in common.entries(smc_apt)) {
- sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config.sh"
- }
-
- // create apt config-drive
- sh "./create-config-drive --user-data mirror_config.sh --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${clusterDomain}-config.iso"
- sh("mv ${aptlyServerHostname}.${clusterDomain}-config.iso output-${clusterName}/")
-
- // save apt iso to artifacts
- archiveArtifacts artifacts: "output-${clusterName}/${aptlyServerHostname}.${clusterDomain}-config.iso"
- }
- }
-
- stage('Save changes reclass model') {
- sh(returnStatus: true, script: "tar -czf output-${clusterName}/${clusterName}.tar.gz --exclude='*@tmp' -C ${modelEnv} .")
- archiveArtifacts artifacts: "output-${clusterName}/${clusterName}.tar.gz"
-
-
- if (EMAIL_ADDRESS != null && EMAIL_ADDRESS != "") {
- emailext(to: EMAIL_ADDRESS,
- attachmentsPattern: "output-${clusterName}/*",
- body: "Mirantis Jenkins\n\nRequested reclass model ${clusterName} has been created and attached to this email.\nEnjoy!\n\nMirantis",
- subject: "Your Salt model ${clusterName}")
- }
- dir("output-${clusterName}") {
- deleteDir()
- }
- }
-
- // Fail, but leave possibility to get failed artifacts
- if (!testResult && TEST_MODEL.toBoolean()) {
- common.warningMsg('Test finished: FAILURE. Please check logs and\\or debug failed model manually!')
- error('Test stage finished: FAILURE')
- }
-
- } catch (Throwable e) {
- currentBuild.result = "FAILURE"
- currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
- throw e
- } finally {
- stage('Clean workspace directories') {
- sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
- }
- // common.sendNotification(currentBuild.result,"",["slack"])
}
- }
}
diff --git a/generate-salt-model-docs-pipeline.groovy b/generate-salt-model-docs-pipeline.groovy
index 4a36f0e..59dd3eb 100644
--- a/generate-salt-model-docs-pipeline.groovy
+++ b/generate-salt-model-docs-pipeline.groovy
@@ -14,83 +14,83 @@
salt = new com.mirantis.mk.Salt()
timeout(time: 12, unit: 'HOURS') {
- node("python") {
- try {
- def workspace = common.getWorkspace()
- def masterName = "cfg01." + CLUSTER_NAME.replace("-","_") + ".lab"
- def jenkinsUserIds = common.getJenkinsUserIds()
- def img = docker.image("tcpcloud/salt-models-testing:nightly")
- img.pull()
- img.inside("-u root:root --hostname ${masterName} --ulimit nofile=4096:8192 --cpus=2") {
- stage("Prepare salt env") {
- if(MODEL_GIT_REF != "" && MODEL_GIT_URL != "") {
- checkouted = gerrit.gerritPatchsetCheckout(MODEL_GIT_URL, MODEL_GIT_REF, "HEAD", CREDENTIALS_ID)
- } else {
- throw new Exception("Cannot checkout gerrit patchset, MODEL_GIT_URL or MODEL_GIT_REF is null")
- }
- if(checkouted) {
- if (fileExists('classes/system')) {
- ssh.prepareSshAgentKey(CREDENTIALS_ID)
- dir('classes/system') {
- // XXX: JENKINS-33510 dir step not work properly inside containers, so let's taky reclass system model directly
- //remoteUrl = git.getGitRemote()
- ssh.ensureKnownHosts("https://github.com/Mirantis/reclass-system-salt-model")
+ node("python") {
+ try {
+ def workspace = common.getWorkspace()
+ def masterName = "cfg01." + CLUSTER_NAME.replace("-", "_") + ".lab"
+ def jenkinsUserIds = common.getJenkinsUserIds()
+ def img = docker.image("tcpcloud/salt-models-testing:nightly")
+ img.pull()
+ img.inside("-u root:root --hostname ${masterName} --ulimit nofile=4096:8192 --cpus=2") {
+ stage("Prepare salt env") {
+ if (MODEL_GIT_REF != "" && MODEL_GIT_URL != "") {
+ checkouted = gerrit.gerritPatchsetCheckout(MODEL_GIT_URL, MODEL_GIT_REF, "HEAD", CREDENTIALS_ID)
+ } else {
+ throw new Exception("Cannot checkout gerrit patchset, MODEL_GIT_URL or MODEL_GIT_REF is null")
}
- ssh.agentSh("git submodule init; git submodule sync; git submodule update --recursive")
- }
- }
- withEnv(["MASTER_HOSTNAME=${masterName}", "CLUSTER_NAME=${CLUSTER_NAME}", "MINION_ID=${masterName}"]){
- sh("cp -r ${workspace}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts")
- sh("""bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts \
+ if (checkouted) {
+ if (fileExists('classes/system')) {
+ ssh.prepareSshAgentKey(CREDENTIALS_ID)
+ dir('classes/system') {
+ // XXX: JENKINS-33510 dir step not work properly inside containers, so let's taky reclass system model directly
+ //remoteUrl = git.getGitRemote()
+ ssh.ensureKnownHosts("https://github.com/Mirantis/reclass-system-salt-model")
+ }
+ ssh.agentSh("git submodule init; git submodule sync; git submodule update --recursive")
+ }
+ }
+ withEnv(["MASTER_HOSTNAME=${masterName}", "CLUSTER_NAME=${CLUSTER_NAME}", "MINION_ID=${masterName}"]) {
+ sh("cp -r ${workspace}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts")
+ sh("""bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts \
&& source_local_envs \
&& configure_salt_master \
&& configure_salt_minion \
&& install_salt_formula_pkg; \
saltservice_restart; \
saltmaster_init'""")
- }
- }
- stage("Generate documentation"){
- def saltResult = sh(script:"salt-call state.sls salt.minion,sphinx.server,nginx", returnStatus:true)
- if(saltResult > 0){
- common.warnMsg("Salt call salt.minion,sphinx.server,nginx failed but continuing")
+ }
}
- }
- stage("Publish outputs"){
- try {
- // /srv/static/sites/reclass_doc will be used for publishHTML step
- // /srv/static/extern will be used as tar artifact
- def outputPresent = sh(script:"ls /srv/static/sites/reclass_doc > /dev/null 2>&1 && ls /srv/static/extern > /dev/null 2>&1", returnStatus: true) == 0
- if(outputPresent){
- sh("""mkdir ${workspace}/output && \
+ stage("Generate documentation") {
+ def saltResult = sh(script: "salt-call state.sls salt.minion,sphinx.server,nginx", returnStatus: true)
+ if (saltResult > 0) {
+ common.warnMsg("Salt call salt.minion,sphinx.server,nginx failed but continuing")
+ }
+ }
+ stage("Publish outputs") {
+ try {
+ // /srv/static/sites/reclass_doc will be used for publishHTML step
+ // /srv/static/extern will be used as tar artifact
+ def outputPresent = sh(script: "ls /srv/static/sites/reclass_doc > /dev/null 2>&1 && ls /srv/static/extern > /dev/null 2>&1", returnStatus: true) == 0
+ if (outputPresent) {
+ sh("""mkdir ${workspace}/output && \
tar -zcf ${workspace}/output/docs-html.tar.gz /srv/static/sites/reclass_doc && \
tar -zcf ${workspace}/output/docs-src.tar.gz /srv/static/extern && \
cp -R /srv/static/sites/reclass_doc ${workspace}/output && \
chown -R ${jenkinsUserIds[0]}:${jenkinsUserIds[1]} ${workspace}/output""")
- publishHTML (target: [
- alwaysLinkToLastBuild: true,
- keepAll: true,
- reportDir: 'output/reclass_doc',
- reportFiles: 'index.html',
- reportName: "Reclass-documentation"
- ])
- archiveArtifacts artifacts: "output/*"
- } else {
- common.errorMsg("Documentation publish failed, one of output directories /srv/static/sites/reclass_doc or /srv/static/extern not exists!")
- }
- } catch(Exception e) {
- common.errorMsg("Documentation publish stage failed!")
+ publishHTML(target: [
+ alwaysLinkToLastBuild: true,
+ keepAll : true,
+ reportDir : 'output/reclass_doc',
+ reportFiles : 'index.html',
+ reportName : "Reclass-documentation"
+ ])
+ archiveArtifacts artifacts: "output/*"
+ } else {
+ common.errorMsg("Documentation publish failed, one of output directories /srv/static/sites/reclass_doc or /srv/static/extern not exists!")
+ }
+ } catch (Exception e) {
+ common.errorMsg("Documentation publish stage failed!")
+ }
}
- }
- }
- } catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
- currentBuild.result = "FAILURE"
- currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
- throw e
- } finally {
- common.sendNotification(currentBuild.result, "", ["slack"])
+ }
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+ throw e
+ } finally {
+ common.sendNotification(currentBuild.result, "", ["slack"])
+ }
}
- }
}