blob: 806534b23d9680e405b4ee783cb27288bfadc4e3 [file] [log] [blame]
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02001/**
2 * Generate cookiecutter cluster by individual products
3 *
4 * Expected parameters:
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02005 * COOKIECUTTER_TEMPLATE_CONTEXT Context parameters for the template generation.
Sergey Galkin8b87f6e2018-10-24 18:40:13 +04006 * CREDENTIALS_ID Credentials id for git
azvyagintsev6d678da2018-11-28 21:19:06 +02007 * TEST_MODEL Run syntax tests for model
azvyagintsev3ed704f2018-07-09 15:49:27 +03008 **/
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +00009import static groovy.json.JsonOutput.toJson
10import static groovy.json.JsonOutput.prettyPrint
Denis Egorenko032a2b72019-04-03 14:56:52 +040011import org.apache.commons.net.util.SubnetUtils
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020012
13common = new com.mirantis.mk.Common()
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +000014common2 = new com.mirantis.mcp.Common()
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020015git = new com.mirantis.mk.Git()
16python = new com.mirantis.mk.Python()
chnyda89191012017-05-29 15:38:35 +020017saltModelTesting = new com.mirantis.mk.SaltModelTesting()
azvyagintsev5b7ec892019-05-15 16:37:34 +030018updateSaltFormulasDuringTest = true
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020019
Aleksey Zvyagintsev4a804212019-02-07 13:02:31 +000020slaveNode = env.getProperty('SLAVE_NODE') ?: 'virtual'
azvyagintsev6d678da2018-11-28 21:19:06 +020021gerritCredentials = env.getProperty('CREDENTIALS_ID') ?: 'gerrit'
22runTestModel = (env.getProperty('TEST_MODEL') ?: true).toBoolean()
azvyagintsev866b19a2018-11-20 18:21:43 +020023distribRevision = 'proposed'
24gitGuessedVersion = false
25
Denis Egorenkoce93af32019-05-20 16:57:33 +040026def GenerateModelToxDocker(Map params) {
27 def ccRoot = params['ccRoot']
28 def context = params['context']
29 def outDir = params['outDir']
30 def envOpts = params['envOpts']
31 def tempContextFile = new File(ccRoot, 'tempContext.yaml_' + UUID.randomUUID().toString()).toString()
32 writeFile file: tempContextFile, text: context
33 // Get Jenkins user UID and GID
34 def jenkinsUID = sh(script: 'id -u', returnStdout: true).trim()
35 def jenkinsGID = sh(script: 'id -g', returnStdout: true).trim()
36 /*
37 by default, process in image operates via root user
38 Otherwise, gpg key for model and all files managed by jenkins user
39 To make it compatible, install rrequirementfrom user, but generate model via jenkins
40 for build use upstream Ubuntu Bionic image
41 */
42 def configRun = ['distribRevision': 'nightly',
43 'envOpts' : envOpts + ["CONFIG_FILE=$tempContextFile",
44 "OUTPUT_DIR=${outDir}"
45 ],
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000046 'image': 'docker-prod-local.artifactory.mirantis.com/mirantis/cicd/jnlp-slave',
Denis Egorenkoce93af32019-05-20 16:57:33 +040047 'runCommands' : [
48 '001_prepare_generate_auto_reqs': {
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000049 sh('''
Denis Egorenkoce93af32019-05-20 16:57:33 +040050 pip install tox
51 ''')
52 },
53 // user & group can be different on host and in docker
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000054 '002_set_jenkins_id': {
55 sh("""
Denis Egorenkoce93af32019-05-20 16:57:33 +040056 usermod -u ${jenkinsUID} jenkins
57 groupmod -g ${jenkinsUID} jenkins
58 """)
59 },
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000060 '003_run_generate_auto': {
Denis Egorenkoce93af32019-05-20 16:57:33 +040061 print('[Cookiecutter build] Result:\n' +
62 sh(returnStdout: true, script: 'cd ' + ccRoot + '; su jenkins -c "tox -ve generate_auto" '))
63 }
64 ]
65 ]
66
67 saltModelTesting.setupDockerAndTest(configRun)
68}
69
azvyagintsev866b19a2018-11-20 18:21:43 +020070def globalVariatorsUpdate() {
71 def templateContext = readYaml text: env.COOKIECUTTER_TEMPLATE_CONTEXT
72 def context = templateContext['default_context']
73 // TODO add more check's for critical var's
74 // Since we can't pin to any '_branch' variable from context, to identify 'default git revision' -
75 // because each of them, might be 'refs/' variable, we need to add some tricky trigger of using
76 // 'release/XXX' logic. This is totall guess - so,if even those one failed, to definitely must pass
77 // correct variable finally!
Denis Egorenkof0d213d2019-06-03 19:05:40 +040078 [ context.get('cookiecutter_template_branch'), context.get('shared_reclass_branch'), context.get('mcp_common_scripts_branch') ].any { branch ->
azvyagintsev866b19a2018-11-20 18:21:43 +020079 if (branch.toString().startsWith('release/')) {
80 gitGuessedVersion = branch
81 return true
82 }
83 }
Denis Egorenkof0d213d2019-06-03 19:05:40 +040084
85 [ 'cookiecutter_template_branch', 'shared_reclass_branch', 'mcp_common_scripts_branch' ].each { repoName ->
Denis Egorenko0b283902019-06-04 13:09:15 +040086 if (context['mcp_version'] in [ "nightly", "testing", "stable" ] && ! context.get(repoName)) {
Denis Egorenkof0d213d2019-06-03 19:05:40 +040087 context[repoName] = 'master'
88 } else if (! context.get(repoName)) {
89 context[repoName] = gitGuessedVersion ?: "release/${context['mcp_version']}".toString()
90 }
azvyagintsev866b19a2018-11-20 18:21:43 +020091 }
92 //
93 distribRevision = context['mcp_version']
94 if (['master'].contains(context['mcp_version'])) {
95 distribRevision = 'nightly'
96 }
97 if (distribRevision.contains('/')) {
98 distribRevision = distribRevision.split('/')[-1]
99 }
100 // Check if we are going to test bleeding-edge release, which doesn't have binary release yet
azvyagintsev8e081642019-02-03 19:15:22 +0200101 // After 2018q4 releases, need to also check 'static' repo, for example ubuntu.
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200102 def binTest = common.checkRemoteBinary(['mcp_version': distribRevision])
azvyagintsev8e081642019-02-03 19:15:22 +0200103 if (!binTest.linux_system_repo_url || !binTest.linux_system_repo_ubuntu_url) {
104 common.errorMsg("Binary release: ${distribRevision} not exist or not full. Fallback to 'proposed'! ")
azvyagintsev866b19a2018-11-20 18:21:43 +0200105 distribRevision = 'proposed'
106 }
azvyagintsev8e081642019-02-03 19:15:22 +0200107
azvyagintsev5400d1d2018-12-11 13:19:29 +0200108 // (azvyagintsev) WA for PROD-25732
109 if (context.cookiecutter_template_url.contains('gerrit.mcp.mirantis.com/mk/cookiecutter-templates')) {
110 common.warningMsg('Apply WA for PROD-25732')
111 context.cookiecutter_template_url = 'ssh://gerrit.mcp.mirantis.com:29418/mk/cookiecutter-templates.git'
112 }
azvyagintsev5b7ec892019-05-15 16:37:34 +0300113 // check, if we are going to test clear release version, w\o any updates and patches
114 if (!gitGuessedVersion && (distribRevision == context.mcp_version)) {
115 updateSaltFormulasDuringTest = false
116 }
117
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400118 if (gitGuessedVersion == 'release/proposed/2019.2.0') {
Denis Egorenko83dfe212019-09-11 16:19:01 +0400119 def mcpSaltRepoUpdateVar = 'deb [arch=amd64] http://mirror.mirantis.com/update/proposed/salt-formulas/xenial xenial main'
120 if (context.get('offline_deployment', 'False').toBoolean()) {
Denis Egorenkoa43c66e2019-09-11 16:40:56 +0400121 mcpSaltRepoUpdateVar = "deb [arch=amd64] http://${context.get('aptly_server_deploy_address')}/update/proposed/salt-formulas/xenial xenial main".toString()
Denis Egorenko83dfe212019-09-11 16:19:01 +0400122 }
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400123 // CFG node in 2019.2.X update has to be bootstrapped with update/proposed repository for salt formulas
124 context['cloudinit_master_config'] = context.get('cloudinit_master_config', false) ?: [:]
Denis Egorenko83dfe212019-09-11 16:19:01 +0400125 context['cloudinit_master_config']['MCP_SALT_REPO_UPDATES'] = context['cloudinit_master_config'].get('MCP_SALT_REPO_UPDATES', false) ?: mcpSaltRepoUpdateVar
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400126 }
127
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000128 common.infoMsg("Using context:\n" + context)
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000129 print prettyPrint(toJson(context))
azvyagintsev866b19a2018-11-20 18:21:43 +0200130 return context
131
132}
azvyagintsevf252b592018-08-13 18:39:14 +0300133
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000134timeout(time: 1, unit: 'HOURS') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000135 node(slaveNode) {
azvyagintsev866b19a2018-11-20 18:21:43 +0200136 def context = globalVariatorsUpdate()
azvyagintsev6d678da2018-11-28 21:19:06 +0200137 def RequesterEmail = context.get('email_address', '')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000138 def templateEnv = "${env.WORKSPACE}/template"
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200139 // modelEnv - this is reclass root, aka /srv/salt/reclass
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000140 def modelEnv = "${env.WORKSPACE}/model"
141 def testEnv = "${env.WORKSPACE}/test"
142 def pipelineEnv = "${env.WORKSPACE}/pipelines"
Tomáš Kukrál9f6260f2017-03-29 23:58:26 +0200143
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000144 try {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000145 //
146 def cutterEnv = "${env.WORKSPACE}/cutter"
147 def systemEnv = "${modelEnv}/classes/system"
148 def testResult = false
149 def user
150 wrap([$class: 'BuildUser']) {
151 user = env.BUILD_USER_ID
152 }
azvyagintsev6d678da2018-11-28 21:19:06 +0200153 currentBuild.description = "${context['cluster_name']} ${RequesterEmail}"
azvyagintsev866b19a2018-11-20 18:21:43 +0200154
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000155 stage('Download Cookiecutter template') {
156 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
157 checkout([
158 $class : 'GitSCM',
159 branches : [[name: 'FETCH_HEAD'],],
160 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: templateEnv]],
161 userRemoteConfigs: [[url: context['cookiecutter_template_url'], refspec: context['cookiecutter_template_branch'], credentialsId: gerritCredentials],],
162 ])
163 }
164 stage('Create empty reclass model') {
165 dir(path: modelEnv) {
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200166 sh 'rm -rfv .git; git init'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000167 sshagent(credentials: [gerritCredentials]) {
168 sh "git submodule add ${context['shared_reclass_url']} 'classes/system'"
169 }
170 }
171 checkout([
172 $class : 'GitSCM',
173 branches : [[name: 'FETCH_HEAD'],],
174 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: systemEnv]],
175 userRemoteConfigs: [[url: context['shared_reclass_url'], refspec: context['shared_reclass_branch'], credentialsId: gerritCredentials],],
176 ])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200177 git.commitGitChanges(modelEnv, 'Added new shared reclass submodule', "${user}@localhost", "${user}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000178 }
179
180 stage('Generate model') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000181 // GNUPGHOME environment variable is required for all gpg commands
182 // and for python.generateModel execution
Denis Egorenkoce93af32019-05-20 16:57:33 +0400183 def envOpts = ["GNUPGHOME=${env.WORKSPACE}/gpghome"]
184 withEnv(envOpts) {
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300185 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000186 sh "mkdir gpghome; chmod 700 gpghome"
Dmitry Pyzhovb5c74c72018-12-17 22:08:50 +0300187 def secretKeyID = RequesterEmail ?: "salt@${context['cluster_domain']}".toString()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300188 if (!context.get('secrets_encryption_private_key')) {
189 def batchData = """
Denis Egorenko131de5f2019-05-16 14:25:40 +0400190 %echo Generating a basic OpenPGP key for Salt-Master
191 %no-protection
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300192 Key-Type: 1
193 Key-Length: 4096
194 Expire-Date: 0
195 Name-Real: ${context['salt_master_hostname']}.${context['cluster_domain']}
196 Name-Email: ${secretKeyID}
Denis Egorenko131de5f2019-05-16 14:25:40 +0400197 %commit
198 %echo done
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300199 """.stripIndent()
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200200 writeFile file: 'gpg-batch.txt', text: batchData
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300201 sh "gpg --gen-key --batch < gpg-batch.txt"
202 sh "gpg --export-secret-key -a ${secretKeyID} > gpgkey.asc"
203 } else {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200204 writeFile file: 'gpgkey.asc', text: context['secrets_encryption_private_key']
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300205 sh "gpg --import gpgkey.asc"
Denis Egorenko131de5f2019-05-16 14:25:40 +0400206 secretKeyID = sh(returnStdout: true, script: 'gpg --list-secret-keys --with-colons | grep -E "^sec" | awk -F: \'{print \$5}\'').trim()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300207 }
208 context['secrets_encryption_key_id'] = secretKeyID
209 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400210 if (context.get('cfg_failsafe_ssh_public_key')) {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200211 writeFile file: 'failsafe-ssh-key.pub', text: context['cfg_failsafe_ssh_public_key']
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400212 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200213 if (!fileExists(new File(templateEnv, 'tox.ini').toString())) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200214 reqs = new File(templateEnv, 'requirements.txt').toString()
215 if (fileExists(reqs)) {
216 python.setupVirtualenv(cutterEnv, 'python2', [], reqs)
217 } else {
218 python.setupCookiecutterVirtualenv(cutterEnv)
219 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200220 python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, modelEnv, templateEnv, false)
221 } else {
222 // tox-based CC generated structure of reclass,from the root. Otherwise for bw compat, modelEnv
223 // still expect only lower lvl of project, aka model/classes/cluster/XXX/. So,lets dump result into
224 // temp dir, and then copy it over initial structure.
225 reclassTempRootDir = sh(script: "mktemp -d -p ${env.WORKSPACE}", returnStdout: true).trim()
Denis Egorenkoce93af32019-05-20 16:57:33 +0400226 GenerateModelToxDocker(['context': common2.dumpYAML(['default_context': context]),
227 'ccRoot' : templateEnv,
228 'outDir' : reclassTempRootDir,
229 'envOpts': envOpts])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200230 dir(modelEnv) {
231 common.warningMsg('Forming reclass-root structure...')
232 sh("cp -ra ${reclassTempRootDir}/reclass/* .")
233 }
234 }
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300235 git.commitGitChanges(modelEnv, "Create model ${context['cluster_name']}", "${user}@localhost", "${user}")
236 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000237 }
238
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000239 stage("Test") {
azvyagintsev6d678da2018-11-28 21:19:06 +0200240 if (runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000241 sh("cp -r ${modelEnv} ${testEnv}")
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200242 if (fileExists('gpgkey.asc')) {
243 common.infoMsg('gpgkey.asc found!Copy it into reclass folder for tests..')
244 sh("cp -v gpgkey.asc ${testEnv}/salt_master_pillar.asc")
245 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000246 def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
azvyagintsev5b7ec892019-05-15 16:37:34 +0300247 common.warningMsg("Attempt to run test against:\n" +
248 "DISTRIB_REVISION from ${distribRevision}\n" +
249 "updateSaltFormulasDuringTest = ${updateSaltFormulasDuringTest}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000250 try {
251 def config = [
Denis Egorenko926bc7d2019-02-19 14:27:34 +0400252 'dockerHostname' : "${context['salt_master_hostname']}",
253 'domain' : "${context['cluster_domain']}",
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000254 'reclassEnv' : testEnv,
255 'distribRevision' : distribRevision,
256 'dockerContainerName': DockerCName,
Denis Egorenkod87490e2019-02-26 20:00:41 +0400257 'testContext' : 'salt-model-node',
azvyagintsev5b7ec892019-05-15 16:37:34 +0300258 'dockerExtraOpts' : ['--memory=3g'],
259 'updateSaltFormulas' : updateSaltFormulasDuringTest
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000260 ]
Denis Egorenko19a793d2019-11-18 20:50:59 +0400261 if (gitGuessedVersion == 'release/proposed/2019.2.0') {
262 config['updateSaltFormulasRev'] = 'proposed'
263 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000264 testResult = saltModelTesting.testNode(config)
265 common.infoMsg("Test finished: SUCCESS")
266 } catch (Exception ex) {
267 common.warningMsg("Test finished: FAILED")
268 testResult = false
269 }
270 } else {
271 common.warningMsg("Test stage has been skipped!")
272 }
273 }
274 stage("Generate config drives") {
275 // apt package genisoimage is required for this stage
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000276 // download create-config-drive
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000277 def commonScriptsRepoUrl = context['mcp_common_scripts_repo'] ?: 'ssh://gerrit.mcp.mirantis.com:29418/mcp/mcp-common-scripts'
278 checkout([
279 $class : 'GitSCM',
280 branches : [[name: 'FETCH_HEAD'],],
281 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'mcp-common-scripts']],
azvyagintsev866b19a2018-11-20 18:21:43 +0200282 userRemoteConfigs: [[url: commonScriptsRepoUrl, refspec: context['mcp_common_scripts_branch'], credentialsId: gerritCredentials],],
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000283 ])
284
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400285 def outdateGeneration = false
286 if (fileExists('mcp-common-scripts/config-drive/create_config_drive.py')) {
287 sh 'cp mcp-common-scripts/config-drive/create_config_drive.py create-config-drive.py'
288 } else {
289 outdateGeneration = true
290 sh 'cp mcp-common-scripts/config-drive/create_config_drive.sh create-config-drive && chmod +x create-config-drive'
291 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000292 sh '[ -f mcp-common-scripts/config-drive/master_config.sh ] && cp mcp-common-scripts/config-drive/master_config.sh user_data || cp mcp-common-scripts/config-drive/master_config.yaml user_data'
293
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000294 sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
295 sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400296 args = [
azvyagintsev1385db92019-03-22 16:05:10 +0200297 "--user-data user_data", "--model ${modelEnv}",
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400298 "--mk-pipelines ${pipelineEnv}/mk-pipelines/", "--pipeline-library ${pipelineEnv}/pipeline-library/"
299 ]
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300300 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400301 args.add('--gpg-key gpgkey.asc')
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300302 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400303 if (context.get('cfg_failsafe_ssh_public_key')) {
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400304 if (outdateGeneration) {
305 args.add('--ssh-key failsafe-ssh-key.pub')
306 } else {
Denis Egorenko85ddf0d2019-03-18 18:11:19 +0400307 if (context.get('cfg_failsafe_user')) {
308 args.add('--ssh-keys failsafe-ssh-key.pub')
309 args.add("--cloud-user-name ${context.get('cfg_failsafe_user')}")
310 }
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400311 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400312 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000313 // load data from model
314 def smc = [:]
315 smc['SALT_MASTER_MINION_ID'] = "${context['salt_master_hostname']}.${context['cluster_domain']}"
316 smc['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400317 if (context.get('cloudinit_master_config', false)) {
318 context['cloudinit_master_config'].each { k, v ->
319 smc[k] = v
320 }
321 }
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400322 if (outdateGeneration) {
323 smc['DEPLOY_NETWORK_GW'] = context['deploy_network_gateway']
324 smc['DEPLOY_NETWORK_NETMASK'] = context['deploy_network_netmask']
325 if (context.get('deploy_network_mtu')) {
326 smc['DEPLOY_NETWORK_MTU'] = context['deploy_network_mtu']
327 }
328 smc['DNS_SERVERS'] = context['dns_server01']
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000329 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000330 smc['MCP_VERSION'] = "${context['mcp_version']}"
331 if (context['local_repositories'] == 'True') {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400332 def localRepoIP = ''
Denis Egorenkof97aec22019-02-05 14:57:33 +0400333 if (context['mcp_version'] in ['2018.4.0', '2018.8.0', '2018.8.0-milestone1', '2018.11.0']) {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400334 localRepoIP = context['local_repo_url']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400335 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}/ubuntu-xenial"
336 } else {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400337 localRepoIP = context['aptly_server_deploy_address']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400338 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}"
339 }
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400340 smc['MCP_SALT_REPO_KEY'] = "http://${localRepoIP}/public.gpg"
Denis Egorenko5f1c0d92019-09-26 09:03:57 +0000341 smc['PIPELINES_FROM_ISO'] = 'false'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000342 smc['PIPELINE_REPO_URL'] = "http://${localRepoIP}:8088"
343 smc['LOCAL_REPOS'] = 'true'
344 }
345 if (context['upstream_proxy_enabled'] == 'True') {
346 if (context['upstream_proxy_auth_enabled'] == 'True') {
347 smc['http_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
348 smc['https_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
349 } else {
350 smc['http_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
351 smc['https_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
352 }
353 }
354
355 for (i in common.entries(smc)) {
Denis Egorenko815d5022019-07-15 16:01:04 +0400356 sh "sed -i 's,export ${i[0]}=.*,export ${i[0]}=\${${i[0]}:-\"${i[1]}\"},' user_data"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000357 }
358
Denis Egorenko032a2b72019-04-03 14:56:52 +0400359 // calculate netmask
Denis Egorenko1f131b72019-04-05 14:42:48 +0400360 def deployNetworkSubnet = ''
361 if (context.get('deploy_network_subnet')) {
362 def subnet = new SubnetUtils(context['deploy_network_subnet'])
363 deployNetworkSubnet = subnet.getInfo().getNetmask()
364 } else if (context.get('deploy_network_netmask')) { // case for 2018.4.0
365 deployNetworkSubnet = context['deploy_network_netmask']
366 } else {
367 error('Neither context parameter deploy_network_subnet or deploy_network_netmask should be set!')
368 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000369 // create cfg config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400370 if (outdateGeneration) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200371 args += ["--hostname ${context['salt_master_hostname']}", "${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"]
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400372 sh "./create-config-drive ${args.join(' ')}"
373 } else {
374 args += [
375 "--name ${context['salt_master_hostname']}", "--hostname ${context['salt_master_hostname']}.${context['cluster_domain']}", "--clean-up",
Denis Egorenko032a2b72019-04-03 14:56:52 +0400376 "--ip ${context['salt_master_management_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400377 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400378 ]
azvyagintsevc4c047a2019-04-04 17:42:48 +0300379 sh "chmod 0755 create-config-drive.py ; ./create-config-drive.py ${args.join(' ')}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400380 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000381 sh("mkdir output-${context['cluster_name']} && mv ${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000382
383 // save cfg iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000384 archiveArtifacts artifacts: "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000385
386 if (context['local_repositories'] == 'True') {
387 def aptlyServerHostname = context.aptly_server_hostname
388 sh "[ -f mcp-common-scripts/config-drive/mirror_config.yaml ] && cp mcp-common-scripts/config-drive/mirror_config.yaml mirror_config || cp mcp-common-scripts/config-drive/mirror_config.sh mirror_config"
389
390 def smc_apt = [:]
391 smc_apt['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400392 if (outdateGeneration) {
393 smc_apt['APTLY_DEPLOY_IP'] = context['aptly_server_deploy_address']
394 smc_apt['APTLY_DEPLOY_NETMASK'] = context['deploy_network_netmask']
395 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000396 smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${context['cluster_domain']}"
397
398 for (i in common.entries(smc_apt)) {
399 sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config"
400 }
401
402 // create apt config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400403 if (outdateGeneration) {
404 sh "./create-config-drive --user-data mirror_config --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
405 } else {
406 args = [
Denis Egorenko032a2b72019-04-03 14:56:52 +0400407 "--ip ${context['aptly_server_deploy_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400408 "--user-data mirror_config", "--hostname ${aptlyServerHostname}.${context['cluster_domain']}", "--name ${aptlyServerHostname}", "--clean-up",
409 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400410 ]
411 sh "python ./create-config-drive.py ${args.join(' ')}"
412 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000413 sh("mv ${aptlyServerHostname}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000414
415 // save apt iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000416 archiveArtifacts artifacts: "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000417 }
418 }
419
420 stage('Save changes reclass model') {
Denis Egorenkod9865252019-04-24 15:41:57 +0400421 sh(returnStatus: true, script: "tar -czf ${context['cluster_name']}.tar.gz --exclude='*@tmp' -C ${modelEnv} .")
422 archiveArtifacts artifacts: "${context['cluster_name']}.tar.gz"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000423
azvyagintsev5400d1d2018-12-11 13:19:29 +0200424 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
Denis Egorenkof9729842019-07-08 15:53:01 +0400425 def mailSubject = "Your Salt model ${context['cluster_name']}"
426 if (context.get('send_method') == 'gcs') {
427 def gcs = new com.mirantis.mk.GoogleCloudStorage()
428 def uploadIsos = [ "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso" ]
429 if (context['local_repositories'] == 'True') {
430 uploadIsos << "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
431 }
432 // generate random hash to have uniq and unpredictable link to file
433 def randHash = common.generateRandomHashString(64)
434 def config = [
435 'creds': context['gcs_creds'],
436 'project': context['gcs_project'],
437 'dest': "gs://${context['gcs_bucket']}/${randHash}",
438 'sources': uploadIsos
439 ]
440 def fileURLs = gcs.uploadArtifactToGoogleStorageBucket(config).join(' ').replace('gs://', 'https://storage.googleapis.com/')
441 emailext(to: RequesterEmail,
442 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and available to download via next URL: ${fileURLs} within 7 days.\nEnjoy!\n\nMirantis",
443 subject: mailSubject)
444 } else {
445 emailext(to: RequesterEmail,
446 attachmentsPattern: "output-${context['cluster_name']}/*",
447 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and attached to this email.\nEnjoy!\n\nMirantis",
448 subject: mailSubject)
449 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000450 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000451 dir("output-${context['cluster_name']}") {
452 deleteDir()
453 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000454 }
455
456 // Fail, but leave possibility to get failed artifacts
azvyagintsev6d678da2018-11-28 21:19:06 +0200457 if (!testResult && runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000458 common.warningMsg('Test finished: FAILURE. Please check logs and\\or debug failed model manually!')
459 error('Test stage finished: FAILURE')
460 }
461
462 } catch (Throwable e) {
463 currentBuild.result = "FAILURE"
464 currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
465 throw e
466 } finally {
467 stage('Clean workspace directories') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000468 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000469 }
470 // common.sendNotification(currentBuild.result,"",["slack"])
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400471 stage('Save artifacts to Artifactory') {
472 def artifactory = new com.mirantis.mcp.MCPArtifactory()
azvyagintsev06dfe402019-03-22 17:58:53 +0200473 def buildProps = ["context=${context['cluster_name']}"]
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400474 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
475 buildProps.add("emailTo=${RequesterEmail}")
476 }
477 def artifactoryLink = artifactory.uploadJobArtifactsToArtifactory([
azvyagintsev06dfe402019-03-22 17:58:53 +0200478 'artifactory' : 'mcp-ci',
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400479 'artifactoryRepo': "drivetrain-local/${JOB_NAME}/${context['cluster_name']}-${BUILD_NUMBER}",
azvyagintsev06dfe402019-03-22 17:58:53 +0200480 'buildProps' : buildProps,
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400481 ])
482 currentBuild.description += "<br/>${artifactoryLink}"
483 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400484 }
Tomáš Kukrál7ded3642017-03-27 15:52:51 +0200485 }
Mikhail Ivanov9f812922017-11-07 18:52:02 +0400486}