blob: d8bfe3a896ae6bb3022c7d1db6f76b0e033e9ab6 [file] [log] [blame]
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02001/**
2 * Generate cookiecutter cluster by individual products
3 *
4 * Expected parameters:
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02005 * COOKIECUTTER_TEMPLATE_CONTEXT Context parameters for the template generation.
Sergey Galkin8b87f6e2018-10-24 18:40:13 +04006 * CREDENTIALS_ID Credentials id for git
azvyagintsev6d678da2018-11-28 21:19:06 +02007 * TEST_MODEL Run syntax tests for model
azvyagintsev3ed704f2018-07-09 15:49:27 +03008 **/
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +00009import static groovy.json.JsonOutput.toJson
10import static groovy.json.JsonOutput.prettyPrint
Denis Egorenko032a2b72019-04-03 14:56:52 +040011import org.apache.commons.net.util.SubnetUtils
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020012
13common = new com.mirantis.mk.Common()
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +000014common2 = new com.mirantis.mcp.Common()
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020015git = new com.mirantis.mk.Git()
16python = new com.mirantis.mk.Python()
chnyda89191012017-05-29 15:38:35 +020017saltModelTesting = new com.mirantis.mk.SaltModelTesting()
azvyagintsev5b7ec892019-05-15 16:37:34 +030018updateSaltFormulasDuringTest = true
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020019
Aleksey Zvyagintsev4a804212019-02-07 13:02:31 +000020slaveNode = env.getProperty('SLAVE_NODE') ?: 'virtual'
azvyagintsev6d678da2018-11-28 21:19:06 +020021gerritCredentials = env.getProperty('CREDENTIALS_ID') ?: 'gerrit'
22runTestModel = (env.getProperty('TEST_MODEL') ?: true).toBoolean()
azvyagintsev866b19a2018-11-20 18:21:43 +020023distribRevision = 'proposed'
24gitGuessedVersion = false
25
Denis Egorenkoce93af32019-05-20 16:57:33 +040026def GenerateModelToxDocker(Map params) {
27 def ccRoot = params['ccRoot']
28 def context = params['context']
29 def outDir = params['outDir']
30 def envOpts = params['envOpts']
31 def tempContextFile = new File(ccRoot, 'tempContext.yaml_' + UUID.randomUUID().toString()).toString()
32 writeFile file: tempContextFile, text: context
33 // Get Jenkins user UID and GID
34 def jenkinsUID = sh(script: 'id -u', returnStdout: true).trim()
35 def jenkinsGID = sh(script: 'id -g', returnStdout: true).trim()
36 /*
37 by default, process in image operates via root user
38 Otherwise, gpg key for model and all files managed by jenkins user
39 To make it compatible, install rrequirementfrom user, but generate model via jenkins
40 for build use upstream Ubuntu Bionic image
41 */
42 def configRun = ['distribRevision': 'nightly',
43 'envOpts' : envOpts + ["CONFIG_FILE=$tempContextFile",
44 "OUTPUT_DIR=${outDir}"
45 ],
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000046 'image': 'docker-prod-local.artifactory.mirantis.com/mirantis/cicd/jnlp-slave',
Denis Egorenkoce93af32019-05-20 16:57:33 +040047 'runCommands' : [
48 '001_prepare_generate_auto_reqs': {
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000049 sh('''
Denis Egorenkoce93af32019-05-20 16:57:33 +040050 pip install tox
51 ''')
52 },
53 // user & group can be different on host and in docker
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000054 '002_set_jenkins_id': {
55 sh("""
Denis Egorenkoce93af32019-05-20 16:57:33 +040056 usermod -u ${jenkinsUID} jenkins
57 groupmod -g ${jenkinsUID} jenkins
58 """)
59 },
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000060 '003_run_generate_auto': {
Denis Egorenkoce93af32019-05-20 16:57:33 +040061 print('[Cookiecutter build] Result:\n' +
62 sh(returnStdout: true, script: 'cd ' + ccRoot + '; su jenkins -c "tox -ve generate_auto" '))
63 }
64 ]
65 ]
66
67 saltModelTesting.setupDockerAndTest(configRun)
68}
69
azvyagintsev866b19a2018-11-20 18:21:43 +020070def globalVariatorsUpdate() {
71 def templateContext = readYaml text: env.COOKIECUTTER_TEMPLATE_CONTEXT
72 def context = templateContext['default_context']
73 // TODO add more check's for critical var's
74 // Since we can't pin to any '_branch' variable from context, to identify 'default git revision' -
75 // because each of them, might be 'refs/' variable, we need to add some tricky trigger of using
76 // 'release/XXX' logic. This is totall guess - so,if even those one failed, to definitely must pass
77 // correct variable finally!
Denis Egorenkof0d213d2019-06-03 19:05:40 +040078 [ context.get('cookiecutter_template_branch'), context.get('shared_reclass_branch'), context.get('mcp_common_scripts_branch') ].any { branch ->
azvyagintsev866b19a2018-11-20 18:21:43 +020079 if (branch.toString().startsWith('release/')) {
80 gitGuessedVersion = branch
81 return true
82 }
83 }
Denis Egorenkof0d213d2019-06-03 19:05:40 +040084
85 [ 'cookiecutter_template_branch', 'shared_reclass_branch', 'mcp_common_scripts_branch' ].each { repoName ->
Denis Egorenko0b283902019-06-04 13:09:15 +040086 if (context['mcp_version'] in [ "nightly", "testing", "stable" ] && ! context.get(repoName)) {
Denis Egorenkof0d213d2019-06-03 19:05:40 +040087 context[repoName] = 'master'
88 } else if (! context.get(repoName)) {
89 context[repoName] = gitGuessedVersion ?: "release/${context['mcp_version']}".toString()
90 }
azvyagintsev866b19a2018-11-20 18:21:43 +020091 }
92 //
93 distribRevision = context['mcp_version']
94 if (['master'].contains(context['mcp_version'])) {
95 distribRevision = 'nightly'
96 }
97 if (distribRevision.contains('/')) {
98 distribRevision = distribRevision.split('/')[-1]
99 }
100 // Check if we are going to test bleeding-edge release, which doesn't have binary release yet
azvyagintsev8e081642019-02-03 19:15:22 +0200101 // After 2018q4 releases, need to also check 'static' repo, for example ubuntu.
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200102 def binTest = common.checkRemoteBinary(['mcp_version': distribRevision])
azvyagintsev8e081642019-02-03 19:15:22 +0200103 if (!binTest.linux_system_repo_url || !binTest.linux_system_repo_ubuntu_url) {
104 common.errorMsg("Binary release: ${distribRevision} not exist or not full. Fallback to 'proposed'! ")
azvyagintsev866b19a2018-11-20 18:21:43 +0200105 distribRevision = 'proposed'
106 }
azvyagintsev8e081642019-02-03 19:15:22 +0200107
azvyagintsev5400d1d2018-12-11 13:19:29 +0200108 // (azvyagintsev) WA for PROD-25732
109 if (context.cookiecutter_template_url.contains('gerrit.mcp.mirantis.com/mk/cookiecutter-templates')) {
110 common.warningMsg('Apply WA for PROD-25732')
111 context.cookiecutter_template_url = 'ssh://gerrit.mcp.mirantis.com:29418/mk/cookiecutter-templates.git'
112 }
azvyagintsev5b7ec892019-05-15 16:37:34 +0300113 // check, if we are going to test clear release version, w\o any updates and patches
114 if (!gitGuessedVersion && (distribRevision == context.mcp_version)) {
115 updateSaltFormulasDuringTest = false
116 }
117
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400118 if (gitGuessedVersion == 'release/proposed/2019.2.0') {
Denis Egorenko83dfe212019-09-11 16:19:01 +0400119 def mcpSaltRepoUpdateVar = 'deb [arch=amd64] http://mirror.mirantis.com/update/proposed/salt-formulas/xenial xenial main'
120 if (context.get('offline_deployment', 'False').toBoolean()) {
Denis Egorenkoa43c66e2019-09-11 16:40:56 +0400121 mcpSaltRepoUpdateVar = "deb [arch=amd64] http://${context.get('aptly_server_deploy_address')}/update/proposed/salt-formulas/xenial xenial main".toString()
Denis Egorenko83dfe212019-09-11 16:19:01 +0400122 }
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400123 // CFG node in 2019.2.X update has to be bootstrapped with update/proposed repository for salt formulas
124 context['cloudinit_master_config'] = context.get('cloudinit_master_config', false) ?: [:]
Denis Egorenko83dfe212019-09-11 16:19:01 +0400125 context['cloudinit_master_config']['MCP_SALT_REPO_UPDATES'] = context['cloudinit_master_config'].get('MCP_SALT_REPO_UPDATES', false) ?: mcpSaltRepoUpdateVar
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400126 }
127
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000128 common.infoMsg("Using context:\n" + context)
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000129 print prettyPrint(toJson(context))
azvyagintsev866b19a2018-11-20 18:21:43 +0200130 return context
131
132}
azvyagintsevf252b592018-08-13 18:39:14 +0300133
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000134timeout(time: 1, unit: 'HOURS') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000135 node(slaveNode) {
azvyagintsev866b19a2018-11-20 18:21:43 +0200136 def context = globalVariatorsUpdate()
azvyagintsev6d678da2018-11-28 21:19:06 +0200137 def RequesterEmail = context.get('email_address', '')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000138 def templateEnv = "${env.WORKSPACE}/template"
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200139 // modelEnv - this is reclass root, aka /srv/salt/reclass
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000140 def modelEnv = "${env.WORKSPACE}/model"
141 def testEnv = "${env.WORKSPACE}/test"
142 def pipelineEnv = "${env.WORKSPACE}/pipelines"
Tomáš Kukrál9f6260f2017-03-29 23:58:26 +0200143
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000144 try {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000145 //
146 def cutterEnv = "${env.WORKSPACE}/cutter"
147 def systemEnv = "${modelEnv}/classes/system"
148 def testResult = false
149 def user
150 wrap([$class: 'BuildUser']) {
151 user = env.BUILD_USER_ID
152 }
azvyagintsev6d678da2018-11-28 21:19:06 +0200153 currentBuild.description = "${context['cluster_name']} ${RequesterEmail}"
azvyagintsev866b19a2018-11-20 18:21:43 +0200154
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000155 stage('Download Cookiecutter template') {
156 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
157 checkout([
158 $class : 'GitSCM',
159 branches : [[name: 'FETCH_HEAD'],],
160 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: templateEnv]],
161 userRemoteConfigs: [[url: context['cookiecutter_template_url'], refspec: context['cookiecutter_template_branch'], credentialsId: gerritCredentials],],
162 ])
163 }
164 stage('Create empty reclass model') {
165 dir(path: modelEnv) {
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200166 sh 'rm -rfv .git; git init'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000167 sshagent(credentials: [gerritCredentials]) {
168 sh "git submodule add ${context['shared_reclass_url']} 'classes/system'"
169 }
170 }
171 checkout([
172 $class : 'GitSCM',
173 branches : [[name: 'FETCH_HEAD'],],
174 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: systemEnv]],
175 userRemoteConfigs: [[url: context['shared_reclass_url'], refspec: context['shared_reclass_branch'], credentialsId: gerritCredentials],],
176 ])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200177 git.commitGitChanges(modelEnv, 'Added new shared reclass submodule', "${user}@localhost", "${user}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000178 }
179
180 stage('Generate model') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000181 // GNUPGHOME environment variable is required for all gpg commands
182 // and for python.generateModel execution
Denis Egorenkoce93af32019-05-20 16:57:33 +0400183 def envOpts = ["GNUPGHOME=${env.WORKSPACE}/gpghome"]
184 withEnv(envOpts) {
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300185 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000186 sh "mkdir gpghome; chmod 700 gpghome"
Dmitry Pyzhovb5c74c72018-12-17 22:08:50 +0300187 def secretKeyID = RequesterEmail ?: "salt@${context['cluster_domain']}".toString()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300188 if (!context.get('secrets_encryption_private_key')) {
189 def batchData = """
Denis Egorenko131de5f2019-05-16 14:25:40 +0400190 %echo Generating a basic OpenPGP key for Salt-Master
191 %no-protection
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300192 Key-Type: 1
193 Key-Length: 4096
194 Expire-Date: 0
195 Name-Real: ${context['salt_master_hostname']}.${context['cluster_domain']}
196 Name-Email: ${secretKeyID}
Denis Egorenko131de5f2019-05-16 14:25:40 +0400197 %commit
198 %echo done
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300199 """.stripIndent()
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200200 writeFile file: 'gpg-batch.txt', text: batchData
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300201 sh "gpg --gen-key --batch < gpg-batch.txt"
202 sh "gpg --export-secret-key -a ${secretKeyID} > gpgkey.asc"
203 } else {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200204 writeFile file: 'gpgkey.asc', text: context['secrets_encryption_private_key']
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300205 sh "gpg --import gpgkey.asc"
Denis Egorenko131de5f2019-05-16 14:25:40 +0400206 secretKeyID = sh(returnStdout: true, script: 'gpg --list-secret-keys --with-colons | grep -E "^sec" | awk -F: \'{print \$5}\'').trim()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300207 }
208 context['secrets_encryption_key_id'] = secretKeyID
209 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400210 if (context.get('cfg_failsafe_ssh_public_key')) {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200211 writeFile file: 'failsafe-ssh-key.pub', text: context['cfg_failsafe_ssh_public_key']
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400212 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200213 if (!fileExists(new File(templateEnv, 'tox.ini').toString())) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200214 reqs = new File(templateEnv, 'requirements.txt').toString()
215 if (fileExists(reqs)) {
216 python.setupVirtualenv(cutterEnv, 'python2', [], reqs)
217 } else {
218 python.setupCookiecutterVirtualenv(cutterEnv)
219 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200220 python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, modelEnv, templateEnv, false)
221 } else {
222 // tox-based CC generated structure of reclass,from the root. Otherwise for bw compat, modelEnv
223 // still expect only lower lvl of project, aka model/classes/cluster/XXX/. So,lets dump result into
224 // temp dir, and then copy it over initial structure.
225 reclassTempRootDir = sh(script: "mktemp -d -p ${env.WORKSPACE}", returnStdout: true).trim()
Denis Egorenkoce93af32019-05-20 16:57:33 +0400226 GenerateModelToxDocker(['context': common2.dumpYAML(['default_context': context]),
227 'ccRoot' : templateEnv,
228 'outDir' : reclassTempRootDir,
229 'envOpts': envOpts])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200230 dir(modelEnv) {
231 common.warningMsg('Forming reclass-root structure...')
232 sh("cp -ra ${reclassTempRootDir}/reclass/* .")
233 }
234 }
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300235 git.commitGitChanges(modelEnv, "Create model ${context['cluster_name']}", "${user}@localhost", "${user}")
236 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000237 }
238
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000239 stage("Test") {
azvyagintsev6d678da2018-11-28 21:19:06 +0200240 if (runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000241 sh("cp -r ${modelEnv} ${testEnv}")
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200242 if (fileExists('gpgkey.asc')) {
243 common.infoMsg('gpgkey.asc found!Copy it into reclass folder for tests..')
244 sh("cp -v gpgkey.asc ${testEnv}/salt_master_pillar.asc")
245 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000246 def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
azvyagintsev5b7ec892019-05-15 16:37:34 +0300247 common.warningMsg("Attempt to run test against:\n" +
248 "DISTRIB_REVISION from ${distribRevision}\n" +
249 "updateSaltFormulasDuringTest = ${updateSaltFormulasDuringTest}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000250 try {
251 def config = [
Denis Egorenko926bc7d2019-02-19 14:27:34 +0400252 'dockerHostname' : "${context['salt_master_hostname']}",
253 'domain' : "${context['cluster_domain']}",
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000254 'reclassEnv' : testEnv,
255 'distribRevision' : distribRevision,
256 'dockerContainerName': DockerCName,
Denis Egorenkod87490e2019-02-26 20:00:41 +0400257 'testContext' : 'salt-model-node',
azvyagintsev5b7ec892019-05-15 16:37:34 +0300258 'dockerExtraOpts' : ['--memory=3g'],
259 'updateSaltFormulas' : updateSaltFormulasDuringTest
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000260 ]
261 testResult = saltModelTesting.testNode(config)
262 common.infoMsg("Test finished: SUCCESS")
263 } catch (Exception ex) {
264 common.warningMsg("Test finished: FAILED")
265 testResult = false
266 }
267 } else {
268 common.warningMsg("Test stage has been skipped!")
269 }
270 }
271 stage("Generate config drives") {
272 // apt package genisoimage is required for this stage
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000273 // download create-config-drive
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000274 def commonScriptsRepoUrl = context['mcp_common_scripts_repo'] ?: 'ssh://gerrit.mcp.mirantis.com:29418/mcp/mcp-common-scripts'
275 checkout([
276 $class : 'GitSCM',
277 branches : [[name: 'FETCH_HEAD'],],
278 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'mcp-common-scripts']],
azvyagintsev866b19a2018-11-20 18:21:43 +0200279 userRemoteConfigs: [[url: commonScriptsRepoUrl, refspec: context['mcp_common_scripts_branch'], credentialsId: gerritCredentials],],
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000280 ])
281
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400282 def outdateGeneration = false
283 if (fileExists('mcp-common-scripts/config-drive/create_config_drive.py')) {
284 sh 'cp mcp-common-scripts/config-drive/create_config_drive.py create-config-drive.py'
285 } else {
286 outdateGeneration = true
287 sh 'cp mcp-common-scripts/config-drive/create_config_drive.sh create-config-drive && chmod +x create-config-drive'
288 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000289 sh '[ -f mcp-common-scripts/config-drive/master_config.sh ] && cp mcp-common-scripts/config-drive/master_config.sh user_data || cp mcp-common-scripts/config-drive/master_config.yaml user_data'
290
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000291 sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
292 sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400293 args = [
azvyagintsev1385db92019-03-22 16:05:10 +0200294 "--user-data user_data", "--model ${modelEnv}",
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400295 "--mk-pipelines ${pipelineEnv}/mk-pipelines/", "--pipeline-library ${pipelineEnv}/pipeline-library/"
296 ]
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300297 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400298 args.add('--gpg-key gpgkey.asc')
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300299 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400300 if (context.get('cfg_failsafe_ssh_public_key')) {
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400301 if (outdateGeneration) {
302 args.add('--ssh-key failsafe-ssh-key.pub')
303 } else {
Denis Egorenko85ddf0d2019-03-18 18:11:19 +0400304 if (context.get('cfg_failsafe_user')) {
305 args.add('--ssh-keys failsafe-ssh-key.pub')
306 args.add("--cloud-user-name ${context.get('cfg_failsafe_user')}")
307 }
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400308 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400309 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000310 // load data from model
311 def smc = [:]
312 smc['SALT_MASTER_MINION_ID'] = "${context['salt_master_hostname']}.${context['cluster_domain']}"
313 smc['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400314 if (context.get('cloudinit_master_config', false)) {
315 context['cloudinit_master_config'].each { k, v ->
316 smc[k] = v
317 }
318 }
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400319 if (outdateGeneration) {
320 smc['DEPLOY_NETWORK_GW'] = context['deploy_network_gateway']
321 smc['DEPLOY_NETWORK_NETMASK'] = context['deploy_network_netmask']
322 if (context.get('deploy_network_mtu')) {
323 smc['DEPLOY_NETWORK_MTU'] = context['deploy_network_mtu']
324 }
325 smc['DNS_SERVERS'] = context['dns_server01']
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000326 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000327 smc['MCP_VERSION'] = "${context['mcp_version']}"
328 if (context['local_repositories'] == 'True') {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400329 def localRepoIP = ''
Denis Egorenkof97aec22019-02-05 14:57:33 +0400330 if (context['mcp_version'] in ['2018.4.0', '2018.8.0', '2018.8.0-milestone1', '2018.11.0']) {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400331 localRepoIP = context['local_repo_url']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400332 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}/ubuntu-xenial"
333 } else {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400334 localRepoIP = context['aptly_server_deploy_address']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400335 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}"
336 }
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400337 smc['MCP_SALT_REPO_KEY'] = "http://${localRepoIP}/public.gpg"
Denis Egorenko5f1c0d92019-09-26 09:03:57 +0000338 smc['PIPELINES_FROM_ISO'] = 'false'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000339 smc['PIPELINE_REPO_URL'] = "http://${localRepoIP}:8088"
340 smc['LOCAL_REPOS'] = 'true'
341 }
342 if (context['upstream_proxy_enabled'] == 'True') {
343 if (context['upstream_proxy_auth_enabled'] == 'True') {
344 smc['http_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
345 smc['https_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
346 } else {
347 smc['http_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
348 smc['https_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
349 }
350 }
351
352 for (i in common.entries(smc)) {
Denis Egorenko815d5022019-07-15 16:01:04 +0400353 sh "sed -i 's,export ${i[0]}=.*,export ${i[0]}=\${${i[0]}:-\"${i[1]}\"},' user_data"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000354 }
355
Denis Egorenko032a2b72019-04-03 14:56:52 +0400356 // calculate netmask
Denis Egorenko1f131b72019-04-05 14:42:48 +0400357 def deployNetworkSubnet = ''
358 if (context.get('deploy_network_subnet')) {
359 def subnet = new SubnetUtils(context['deploy_network_subnet'])
360 deployNetworkSubnet = subnet.getInfo().getNetmask()
361 } else if (context.get('deploy_network_netmask')) { // case for 2018.4.0
362 deployNetworkSubnet = context['deploy_network_netmask']
363 } else {
364 error('Neither context parameter deploy_network_subnet or deploy_network_netmask should be set!')
365 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000366 // create cfg config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400367 if (outdateGeneration) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200368 args += ["--hostname ${context['salt_master_hostname']}", "${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"]
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400369 sh "./create-config-drive ${args.join(' ')}"
370 } else {
371 args += [
372 "--name ${context['salt_master_hostname']}", "--hostname ${context['salt_master_hostname']}.${context['cluster_domain']}", "--clean-up",
Denis Egorenko032a2b72019-04-03 14:56:52 +0400373 "--ip ${context['salt_master_management_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400374 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400375 ]
azvyagintsevc4c047a2019-04-04 17:42:48 +0300376 sh "chmod 0755 create-config-drive.py ; ./create-config-drive.py ${args.join(' ')}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400377 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000378 sh("mkdir output-${context['cluster_name']} && mv ${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000379
380 // save cfg iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000381 archiveArtifacts artifacts: "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000382
383 if (context['local_repositories'] == 'True') {
384 def aptlyServerHostname = context.aptly_server_hostname
385 sh "[ -f mcp-common-scripts/config-drive/mirror_config.yaml ] && cp mcp-common-scripts/config-drive/mirror_config.yaml mirror_config || cp mcp-common-scripts/config-drive/mirror_config.sh mirror_config"
386
387 def smc_apt = [:]
388 smc_apt['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400389 if (outdateGeneration) {
390 smc_apt['APTLY_DEPLOY_IP'] = context['aptly_server_deploy_address']
391 smc_apt['APTLY_DEPLOY_NETMASK'] = context['deploy_network_netmask']
392 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000393 smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${context['cluster_domain']}"
394
395 for (i in common.entries(smc_apt)) {
396 sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config"
397 }
398
399 // create apt config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400400 if (outdateGeneration) {
401 sh "./create-config-drive --user-data mirror_config --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
402 } else {
403 args = [
Denis Egorenko032a2b72019-04-03 14:56:52 +0400404 "--ip ${context['aptly_server_deploy_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400405 "--user-data mirror_config", "--hostname ${aptlyServerHostname}.${context['cluster_domain']}", "--name ${aptlyServerHostname}", "--clean-up",
406 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400407 ]
408 sh "python ./create-config-drive.py ${args.join(' ')}"
409 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000410 sh("mv ${aptlyServerHostname}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000411
412 // save apt iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000413 archiveArtifacts artifacts: "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000414 }
415 }
416
417 stage('Save changes reclass model') {
Denis Egorenkod9865252019-04-24 15:41:57 +0400418 sh(returnStatus: true, script: "tar -czf ${context['cluster_name']}.tar.gz --exclude='*@tmp' -C ${modelEnv} .")
419 archiveArtifacts artifacts: "${context['cluster_name']}.tar.gz"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000420
azvyagintsev5400d1d2018-12-11 13:19:29 +0200421 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
Denis Egorenkof9729842019-07-08 15:53:01 +0400422 def mailSubject = "Your Salt model ${context['cluster_name']}"
423 if (context.get('send_method') == 'gcs') {
424 def gcs = new com.mirantis.mk.GoogleCloudStorage()
425 def uploadIsos = [ "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso" ]
426 if (context['local_repositories'] == 'True') {
427 uploadIsos << "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
428 }
429 // generate random hash to have uniq and unpredictable link to file
430 def randHash = common.generateRandomHashString(64)
431 def config = [
432 'creds': context['gcs_creds'],
433 'project': context['gcs_project'],
434 'dest': "gs://${context['gcs_bucket']}/${randHash}",
435 'sources': uploadIsos
436 ]
437 def fileURLs = gcs.uploadArtifactToGoogleStorageBucket(config).join(' ').replace('gs://', 'https://storage.googleapis.com/')
438 emailext(to: RequesterEmail,
439 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and available to download via next URL: ${fileURLs} within 7 days.\nEnjoy!\n\nMirantis",
440 subject: mailSubject)
441 } else {
442 emailext(to: RequesterEmail,
443 attachmentsPattern: "output-${context['cluster_name']}/*",
444 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and attached to this email.\nEnjoy!\n\nMirantis",
445 subject: mailSubject)
446 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000447 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000448 dir("output-${context['cluster_name']}") {
449 deleteDir()
450 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000451 }
452
453 // Fail, but leave possibility to get failed artifacts
azvyagintsev6d678da2018-11-28 21:19:06 +0200454 if (!testResult && runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000455 common.warningMsg('Test finished: FAILURE. Please check logs and\\or debug failed model manually!')
456 error('Test stage finished: FAILURE')
457 }
458
459 } catch (Throwable e) {
460 currentBuild.result = "FAILURE"
461 currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
462 throw e
463 } finally {
464 stage('Clean workspace directories') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000465 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000466 }
467 // common.sendNotification(currentBuild.result,"",["slack"])
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400468 stage('Save artifacts to Artifactory') {
469 def artifactory = new com.mirantis.mcp.MCPArtifactory()
azvyagintsev06dfe402019-03-22 17:58:53 +0200470 def buildProps = ["context=${context['cluster_name']}"]
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400471 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
472 buildProps.add("emailTo=${RequesterEmail}")
473 }
474 def artifactoryLink = artifactory.uploadJobArtifactsToArtifactory([
azvyagintsev06dfe402019-03-22 17:58:53 +0200475 'artifactory' : 'mcp-ci',
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400476 'artifactoryRepo': "drivetrain-local/${JOB_NAME}/${context['cluster_name']}-${BUILD_NUMBER}",
azvyagintsev06dfe402019-03-22 17:58:53 +0200477 'buildProps' : buildProps,
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400478 ])
479 currentBuild.description += "<br/>${artifactoryLink}"
480 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400481 }
Tomáš Kukrál7ded3642017-03-27 15:52:51 +0200482 }
Mikhail Ivanov9f812922017-11-07 18:52:02 +0400483}