blob: 9e48ec8df03edd50e2d19bf602acdc053fa36d60 [file] [log] [blame]
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02001/**
2 * Generate cookiecutter cluster by individual products
3 *
4 * Expected parameters:
5 * COOKIECUTTER_TEMPLATE_CREDENTIALS Credentials to the Cookiecutter template repo.
6 * COOKIECUTTER_TEMPLATE_URL Cookiecutter template repo address.
7 * COOKIECUTTER_TEMPLATE_BRANCH Branch for the template.
8 * COOKIECUTTER_TEMPLATE_CONTEXT Context parameters for the template generation.
Tomáš Kukrál91e49252017-05-09 14:40:26 +02009 * EMAIL_ADDRESS Email to send a created tar file
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020010 *
11**/
12
13common = new com.mirantis.mk.Common()
14git = new com.mirantis.mk.Git()
15python = new com.mirantis.mk.Python()
chnyda89191012017-05-29 15:38:35 +020016saltModelTesting = new com.mirantis.mk.SaltModelTesting()
Tomáš Kukrálcebc1a02017-07-25 16:10:37 +020017ssh = new com.mirantis.mk.Ssh()
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020018
Jakub Josefa63f9862018-01-11 17:58:38 +010019timeout(time: 12, unit: 'HOURS') {
20 node("python&&docker") {
21 def templateEnv = "${env.WORKSPACE}/template"
22 def modelEnv = "${env.WORKSPACE}/model"
23 def testEnv = "${env.WORKSPACE}/test"
24 def pipelineEnv = "${env.WORKSPACE}/pipelines"
Tomáš Kukrál9f6260f2017-03-29 23:58:26 +020025
Jakub Josefa63f9862018-01-11 17:58:38 +010026 try {
27 def templateContext = readYaml text: COOKIECUTTER_TEMPLATE_CONTEXT
28 def clusterDomain = templateContext.default_context.cluster_domain
29 def clusterName = templateContext.default_context.cluster_name
30 def saltMaster = templateContext.default_context.salt_master_hostname
31 def cutterEnv = "${env.WORKSPACE}/cutter"
32 def jinjaEnv = "${env.WORKSPACE}/jinja"
33 def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
34 def targetBranch = "feature/${clusterName}"
35 def templateBaseDir = "${env.WORKSPACE}/template"
36 def templateDir = "${templateEnv}/template/dir"
37 def templateOutputDir = templateBaseDir
38 def user
39 wrap([$class: 'BuildUser']) {
40 user = env.BUILD_USER_ID
Tomáš Kukrála3f4ba72017-08-03 15:40:22 +020041 }
42
Jakub Josefa63f9862018-01-11 17:58:38 +010043 currentBuild.description = clusterName
44 print("Using context:\n" + COOKIECUTTER_TEMPLATE_CONTEXT)
Tomáš Kukrál0c6835d2017-07-18 16:00:27 +020045
Jakub Josefa63f9862018-01-11 17:58:38 +010046 stage ('Download Cookiecutter template') {
Richard Felklc73f2ee2018-01-18 11:09:48 +010047 if (COOKIECUTTER_TEMPLATE_BRANCH.startsWith('refs/')) {
Jakub Josefa63f9862018-01-11 17:58:38 +010048 git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, 'master', COOKIECUTTER_TEMPLATE_CREDENTIALS)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040049
Jakub Josefa63f9862018-01-11 17:58:38 +010050 dir(templateEnv) {
51 ssh.agentSh("git fetch ${COOKIECUTTER_TEMPLATE_URL} ${COOKIECUTTER_TEMPLATE_BRANCH} && git checkout FETCH_HEAD")
52 }
53 } else {
54 git.checkoutGitRepository(templateEnv, COOKIECUTTER_TEMPLATE_URL, COOKIECUTTER_TEMPLATE_BRANCH, COOKIECUTTER_TEMPLATE_CREDENTIALS)
55 }
Richard Felkl9543faa2018-01-11 09:47:35 +010056
Jakub Josefa63f9862018-01-11 17:58:38 +010057 }
58
59 stage ('Create empty reclass model') {
60 dir(path: modelEnv) {
61 sh "rm -rfv .git"
62 sh "git init"
63
Leontii Istomine7b10332018-02-20 17:01:58 +010064 def sharedReclassUrl = templateContext['default_context']['shared_reclass_url']
65 if (sharedReclassUrl != '') {
66 ssh.agentSh "git submodule add \"${sharedReclassUtl}\" \"classes/system\""
Jakub Josefa63f9862018-01-11 17:58:38 +010067
Leontii Istomine7b10332018-02-20 17:01:58 +010068 def sharedReclassRefspec = templateContext['default_context']['shared_reclass_refspec']
69 if(sharedReclassRefspec != '') {
70 ssh.agentSh "cd \"classes/system\";git fetch ${sharedReclassUrl} ${sharedReclassRefspec}; git checkout FETCH_HEAD"
71 } else {
72 def mcpVersion = templateContext['default_context']['mcp_version']
73 if(mcpVersion != "stable" && mcpVersion != "nightly" && mcpVersion != "testing"){
74 ssh.agentSh "cd \"classes/system\";git fetch --tags;git checkout ${mcpVersion}"
75 }
Jakub Josefa63f9862018-01-11 17:58:38 +010076 }
77
78 git.commitGitChanges(modelEnv, "Added new shared reclass submodule", "${user}@localhost", "${user}")
79 }
80 }
81 }
82
83 def productList = ["infra", "cicd", "opencontrail", "kubernetes", "openstack", "oss", "stacklight", "ceph"]
84 for (product in productList) {
85
86 // get templateOutputDir and productDir
87 if (product.startsWith("stacklight")) {
88 templateOutputDir = "${env.WORKSPACE}/output/stacklight"
89
90 def stacklightVersion
91 try {
92 stacklightVersion = templateContext.default_context['stacklight_version']
93 } catch (Throwable e) {
94 common.warningMsg('Stacklight version loading failed')
Richard Felkl9543faa2018-01-11 09:47:35 +010095 }
96
Jakub Josefa63f9862018-01-11 17:58:38 +010097 if (stacklightVersion) {
98 productDir = "stacklight" + stacklightVersion
99 } else {
100 productDir = "stacklight1"
101 }
Tomáš Kukrál7ded3642017-03-27 15:52:51 +0200102
Tomáš Kukrála3f4ba72017-08-03 15:40:22 +0200103 } else {
Jakub Josefa63f9862018-01-11 17:58:38 +0100104 templateOutputDir = "${env.WORKSPACE}/output/${product}"
105 productDir = product
Tomáš Kukrál9a6821e2017-07-24 11:07:01 +0200106 }
Tomáš Kukrála3f4ba72017-08-03 15:40:22 +0200107
Jakub Josefa63f9862018-01-11 17:58:38 +0100108 if (product == "infra" || (templateContext.default_context["${product}_enabled"]
109 && templateContext.default_context["${product}_enabled"].toBoolean())) {
Tomáš Kukrál7ded3642017-03-27 15:52:51 +0200110
Jakub Josefa63f9862018-01-11 17:58:38 +0100111 templateDir = "${templateEnv}/cluster_product/${productDir}"
112 common.infoMsg("Generating product " + product + " from " + templateDir + " to " + templateOutputDir)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400113
Jakub Josefa63f9862018-01-11 17:58:38 +0100114 sh "rm -rf ${templateOutputDir} || true"
115 sh "mkdir -p ${templateOutputDir}"
116 sh "mkdir -p ${outputDestination}"
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400117
Jakub Josefa63f9862018-01-11 17:58:38 +0100118 python.setupCookiecutterVirtualenv(cutterEnv)
119 python.buildCookiecutterTemplate(templateDir, COOKIECUTTER_TEMPLATE_CONTEXT, templateOutputDir, cutterEnv, templateBaseDir)
120 sh "mv -v ${templateOutputDir}/${clusterName}/* ${outputDestination}"
Jiri Broulikd1375f72018-01-09 10:14:59 +0100121 } else {
Jakub Josefa63f9862018-01-11 17:58:38 +0100122 common.warningMsg("Product " + product + " is disabled")
Jiri Broulikd1375f72018-01-09 10:14:59 +0100123 }
124 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400125
Jakub Josefa63f9862018-01-11 17:58:38 +0100126 stage('Generate new SaltMaster node') {
127 def nodeFile = "${modelEnv}/nodes/${saltMaster}.${clusterDomain}.yml"
128 def nodeString = """classes:
Leontii Istomin56b6b112018-01-15 12:14:24 +0300129- cluster.${clusterName}.infra.config
130parameters:
131 _param:
132 linux_system_codename: xenial
133 reclass_data_revision: master
134 linux:
135 system:
136 name: ${saltMaster}
137 domain: ${clusterDomain}
Jakub Josefa63f9862018-01-11 17:58:38 +0100138 """
139 sh "mkdir -p ${modelEnv}/nodes/"
140 writeFile(file: nodeFile, text: nodeString)
141
142 git.commitGitChanges(modelEnv, "Create model ${clusterName}", "${user}@localhost", "${user}")
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400143 }
144
Jakub Josefa63f9862018-01-11 17:58:38 +0100145 stage("Test") {
146 if (SHARED_RECLASS_URL != "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
147 sh("cp -r ${modelEnv} ${testEnv}")
148 saltModelTesting.setupAndTestNode("${saltMaster}.${clusterDomain}", "", testEnv)
149 }
150 }
Richard Felkl9fd47942017-10-20 16:23:29 +0200151
Jakub Josefa63f9862018-01-11 17:58:38 +0100152 stage("Generate config drives") {
153 // apt package genisoimage is required for this stage
Richard Felkl9fd47942017-10-20 16:23:29 +0200154
Jakub Josefa63f9862018-01-11 17:58:38 +0100155 // download create-config-drive
156 // FIXME: that should be refactored, to use git clone - to be able download it from custom repo.
157 def config_drive_script_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/master/config-drive/create_config_drive.sh"
158 def user_data_script_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/master/config-drive/master_config.sh"
Richard Felkl9fd47942017-10-20 16:23:29 +0200159
Jakub Josefa63f9862018-01-11 17:58:38 +0100160 sh "wget -O create-config-drive ${config_drive_script_url} && chmod +x create-config-drive"
161 sh "wget -O user_data.sh ${user_data_script_url}"
Richard Felkl9fd47942017-10-20 16:23:29 +0200162
Jakub Josefa63f9862018-01-11 17:58:38 +0100163 sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
164 sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
165 args = "--user-data user_data.sh --hostname ${saltMaster} --model ${modelEnv} --mk-pipelines ${pipelineEnv}/mk-pipelines/ --pipeline-library ${pipelineEnv}/pipeline-library/ ${saltMaster}.${clusterDomain}-config.iso"
166
167 // load data from model
168 def smc = [:]
169 smc['SALT_MASTER_MINION_ID'] = "${saltMaster}.${clusterDomain}"
170 smc['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
171 smc['DEPLOY_NETWORK_GW'] = templateContext['default_context']['deploy_network_gateway']
172 smc['DEPLOY_NETWORK_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
173 smc['DNS_SERVERS'] = templateContext['default_context']['dns_server01']
174 if (templateContext['default_context']['local_repositories'] == 'True'){
175 smc['PIPELINES_FROM_ISO'] = 'false'
176 smc['PIPELINE_REPO_URL'] = 'http://' + templateContext['default_context']['aptly_server_deploy_address'] + ':8088'
177 }
178 if (templateContext['default_context']['upstream_proxy_enabled'] == 'True'){
179 if (templateContext['default_context']['upstream_proxy_auth_enabled'] == 'True'){
180 smc['http_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_user'] + ':' + templateContext['default_context']['upstream_proxy_password'] + '@' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
181 smc['https_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_user'] + ':' + templateContext['default_context']['upstream_proxy_password'] + '@' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
182 } else {
183 smc['http_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
184 smc['https_proxy'] = 'http://' + templateContext['default_context']['upstream_proxy_address'] + ':' + templateContext['default_context']['upstream_proxy_port']
185 }
Richard Felkl9fd47942017-10-20 16:23:29 +0200186 }
187
Jakub Josefa63f9862018-01-11 17:58:38 +0100188 for (i in common.entries(smc)) {
189 sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" user_data.sh"
190 }
Richard Felkl9fd47942017-10-20 16:23:29 +0200191
Jakub Josefa63f9862018-01-11 17:58:38 +0100192 // create cfg config-drive
193 sh "./create-config-drive ${args}"
194 sh("mkdir output-${clusterName} && mv ${saltMaster}.${clusterDomain}-config.iso output-${clusterName}/")
195
196 // save cfg iso to artifacts
197 archiveArtifacts artifacts: "output-${clusterName}/${saltMaster}.${clusterDomain}-config.iso"
198
199 if (templateContext['default_context']['local_repositories'] == 'True'){
200 def aptlyServerHostname = templateContext.default_context.aptly_server_hostname
201 def user_data_script_apt_url = "https://raw.githubusercontent.com/Mirantis/mcp-common-scripts/master/config-drive/mirror_config.sh"
202 sh "wget -O mirror_config.sh ${user_data_script_apt_url}"
203
204 def smc_apt = [:]
205 smc_apt['SALT_MASTER_DEPLOY_IP'] = templateContext['default_context']['salt_master_management_address']
206 smc_apt['APTLY_DEPLOY_IP'] = templateContext['default_context']['aptly_server_deploy_address']
207 smc_apt['APTLY_DEPLOY_NETMASK'] = templateContext['default_context']['deploy_network_netmask']
208 smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${clusterDomain}"
209
210 for (i in common.entries(smc_apt)) {
211 sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config.sh"
212 }
213
214 // create apt config-drive
215 sh "./create-config-drive --user-data mirror_config.sh --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${clusterDomain}-config.iso"
216 sh("mv ${aptlyServerHostname}.${clusterDomain}-config.iso output-${clusterName}/")
217
218 // save apt iso to artifacts
219 archiveArtifacts artifacts: "output-${clusterName}/${aptlyServerHostname}.${clusterDomain}-config.iso"
220 }
Richard Felkl9fd47942017-10-20 16:23:29 +0200221 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400222
Jakub Josefa63f9862018-01-11 17:58:38 +0100223 stage ('Save changes reclass model') {
224 sh(returnStatus: true, script: "tar -zcf output-${clusterName}/${clusterName}.tar.gz -C ${modelEnv} .")
225 archiveArtifacts artifacts: "output-${clusterName}/${clusterName}.tar.gz"
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400226
227
Jakub Josefa63f9862018-01-11 17:58:38 +0100228 if (EMAIL_ADDRESS != null && EMAIL_ADDRESS != "") {
229 emailext(to: EMAIL_ADDRESS,
230 attachmentsPattern: "output-${clusterName}/*",
231 body: "Mirantis Jenkins\n\nRequested reclass model ${clusterName} has been created and attached to this email.\nEnjoy!\n\nMirantis",
232 subject: "Your Salt model ${clusterName}")
233 }
234 dir("output-${clusterName}"){
235 deleteDir()
236 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400237 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400238
Jakub Josefa63f9862018-01-11 17:58:38 +0100239 } catch (Throwable e) {
240 // If there was an error or exception thrown, the build failed
241 currentBuild.result = "FAILURE"
242 currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
243 throw e
244 } finally {
245 stage ('Clean workspace directories') {
246 sh(returnStatus: true, script: "rm -rf ${templateEnv}")
247 sh(returnStatus: true, script: "rm -rf ${modelEnv}")
248 sh(returnStatus: true, script: "rm -rf ${pipelineEnv}")
249 }
250 // common.sendNotification(currentBuild.result,"",["slack"])
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400251 }
Tomáš Kukrál7ded3642017-03-27 15:52:51 +0200252 }
Mikhail Ivanov9f812922017-11-07 18:52:02 +0400253}