blob: 17fbdca81f78d55adf7a4c2933215ae175843d03 [file] [log] [blame]
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02001/**
2 * Generate cookiecutter cluster by individual products
3 *
4 * Expected parameters:
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02005 * COOKIECUTTER_TEMPLATE_CONTEXT Context parameters for the template generation.
Sergey Galkin8b87f6e2018-10-24 18:40:13 +04006 * CREDENTIALS_ID Credentials id for git
azvyagintsev6d678da2018-11-28 21:19:06 +02007 * TEST_MODEL Run syntax tests for model
azvyagintsev3ed704f2018-07-09 15:49:27 +03008 **/
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +00009import static groovy.json.JsonOutput.toJson
10import static groovy.json.JsonOutput.prettyPrint
Denis Egorenko032a2b72019-04-03 14:56:52 +040011import org.apache.commons.net.util.SubnetUtils
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020012
13common = new com.mirantis.mk.Common()
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +000014common2 = new com.mirantis.mcp.Common()
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020015git = new com.mirantis.mk.Git()
16python = new com.mirantis.mk.Python()
chnyda89191012017-05-29 15:38:35 +020017saltModelTesting = new com.mirantis.mk.SaltModelTesting()
azvyagintsev5b7ec892019-05-15 16:37:34 +030018updateSaltFormulasDuringTest = true
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020019
Aleksey Zvyagintsev4a804212019-02-07 13:02:31 +000020slaveNode = env.getProperty('SLAVE_NODE') ?: 'virtual'
azvyagintsev6d678da2018-11-28 21:19:06 +020021gerritCredentials = env.getProperty('CREDENTIALS_ID') ?: 'gerrit'
22runTestModel = (env.getProperty('TEST_MODEL') ?: true).toBoolean()
azvyagintsev866b19a2018-11-20 18:21:43 +020023distribRevision = 'proposed'
24gitGuessedVersion = false
25
Denis Egorenkoce93af32019-05-20 16:57:33 +040026def GenerateModelToxDocker(Map params) {
27 def ccRoot = params['ccRoot']
28 def context = params['context']
29 def outDir = params['outDir']
30 def envOpts = params['envOpts']
31 def tempContextFile = new File(ccRoot, 'tempContext.yaml_' + UUID.randomUUID().toString()).toString()
32 writeFile file: tempContextFile, text: context
33 // Get Jenkins user UID and GID
34 def jenkinsUID = sh(script: 'id -u', returnStdout: true).trim()
35 def jenkinsGID = sh(script: 'id -g', returnStdout: true).trim()
36 /*
37 by default, process in image operates via root user
38 Otherwise, gpg key for model and all files managed by jenkins user
39 To make it compatible, install rrequirementfrom user, but generate model via jenkins
40 for build use upstream Ubuntu Bionic image
41 */
42 def configRun = ['distribRevision': 'nightly',
43 'envOpts' : envOpts + ["CONFIG_FILE=$tempContextFile",
44 "OUTPUT_DIR=${outDir}"
45 ],
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000046 'image': 'docker-prod-local.artifactory.mirantis.com/mirantis/cicd/jnlp-slave',
Denis Egorenkoce93af32019-05-20 16:57:33 +040047 'runCommands' : [
48 '001_prepare_generate_auto_reqs': {
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000049 sh('''
Denis Egorenkoce93af32019-05-20 16:57:33 +040050 pip install tox
51 ''')
52 },
53 // user & group can be different on host and in docker
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000054 '002_set_jenkins_id': {
55 sh("""
Denis Egorenkoce93af32019-05-20 16:57:33 +040056 usermod -u ${jenkinsUID} jenkins
57 groupmod -g ${jenkinsUID} jenkins
58 """)
59 },
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000060 '003_run_generate_auto': {
Denis Egorenkoce93af32019-05-20 16:57:33 +040061 print('[Cookiecutter build] Result:\n' +
62 sh(returnStdout: true, script: 'cd ' + ccRoot + '; su jenkins -c "tox -ve generate_auto" '))
63 }
64 ]
65 ]
66
67 saltModelTesting.setupDockerAndTest(configRun)
68}
69
azvyagintsev866b19a2018-11-20 18:21:43 +020070def globalVariatorsUpdate() {
71 def templateContext = readYaml text: env.COOKIECUTTER_TEMPLATE_CONTEXT
72 def context = templateContext['default_context']
73 // TODO add more check's for critical var's
74 // Since we can't pin to any '_branch' variable from context, to identify 'default git revision' -
75 // because each of them, might be 'refs/' variable, we need to add some tricky trigger of using
76 // 'release/XXX' logic. This is totall guess - so,if even those one failed, to definitely must pass
77 // correct variable finally!
Denis Egorenkof0d213d2019-06-03 19:05:40 +040078 [ context.get('cookiecutter_template_branch'), context.get('shared_reclass_branch'), context.get('mcp_common_scripts_branch') ].any { branch ->
azvyagintsev866b19a2018-11-20 18:21:43 +020079 if (branch.toString().startsWith('release/')) {
80 gitGuessedVersion = branch
81 return true
82 }
83 }
Denis Egorenkof0d213d2019-06-03 19:05:40 +040084
85 [ 'cookiecutter_template_branch', 'shared_reclass_branch', 'mcp_common_scripts_branch' ].each { repoName ->
Denis Egorenko0b283902019-06-04 13:09:15 +040086 if (context['mcp_version'] in [ "nightly", "testing", "stable" ] && ! context.get(repoName)) {
Denis Egorenkof0d213d2019-06-03 19:05:40 +040087 context[repoName] = 'master'
88 } else if (! context.get(repoName)) {
89 context[repoName] = gitGuessedVersion ?: "release/${context['mcp_version']}".toString()
90 }
azvyagintsev866b19a2018-11-20 18:21:43 +020091 }
92 //
93 distribRevision = context['mcp_version']
94 if (['master'].contains(context['mcp_version'])) {
95 distribRevision = 'nightly'
96 }
97 if (distribRevision.contains('/')) {
98 distribRevision = distribRevision.split('/')[-1]
99 }
100 // Check if we are going to test bleeding-edge release, which doesn't have binary release yet
azvyagintsev8e081642019-02-03 19:15:22 +0200101 // After 2018q4 releases, need to also check 'static' repo, for example ubuntu.
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200102 def binTest = common.checkRemoteBinary(['mcp_version': distribRevision])
azvyagintsev8e081642019-02-03 19:15:22 +0200103 if (!binTest.linux_system_repo_url || !binTest.linux_system_repo_ubuntu_url) {
104 common.errorMsg("Binary release: ${distribRevision} not exist or not full. Fallback to 'proposed'! ")
azvyagintsev866b19a2018-11-20 18:21:43 +0200105 distribRevision = 'proposed'
106 }
azvyagintsev8e081642019-02-03 19:15:22 +0200107
azvyagintsev5400d1d2018-12-11 13:19:29 +0200108 // (azvyagintsev) WA for PROD-25732
109 if (context.cookiecutter_template_url.contains('gerrit.mcp.mirantis.com/mk/cookiecutter-templates')) {
110 common.warningMsg('Apply WA for PROD-25732')
111 context.cookiecutter_template_url = 'ssh://gerrit.mcp.mirantis.com:29418/mk/cookiecutter-templates.git'
112 }
azvyagintsev5b7ec892019-05-15 16:37:34 +0300113 // check, if we are going to test clear release version, w\o any updates and patches
114 if (!gitGuessedVersion && (distribRevision == context.mcp_version)) {
115 updateSaltFormulasDuringTest = false
116 }
117
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400118 if (gitGuessedVersion == 'release/proposed/2019.2.0') {
Denis Egorenko57714502019-11-18 16:42:33 +0400119 distribRevision = 'proposed'
Denis Egorenko83dfe212019-09-11 16:19:01 +0400120 def mcpSaltRepoUpdateVar = 'deb [arch=amd64] http://mirror.mirantis.com/update/proposed/salt-formulas/xenial xenial main'
121 if (context.get('offline_deployment', 'False').toBoolean()) {
Denis Egorenkoa43c66e2019-09-11 16:40:56 +0400122 mcpSaltRepoUpdateVar = "deb [arch=amd64] http://${context.get('aptly_server_deploy_address')}/update/proposed/salt-formulas/xenial xenial main".toString()
Denis Egorenko83dfe212019-09-11 16:19:01 +0400123 }
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400124 // CFG node in 2019.2.X update has to be bootstrapped with update/proposed repository for salt formulas
125 context['cloudinit_master_config'] = context.get('cloudinit_master_config', false) ?: [:]
Denis Egorenko83dfe212019-09-11 16:19:01 +0400126 context['cloudinit_master_config']['MCP_SALT_REPO_UPDATES'] = context['cloudinit_master_config'].get('MCP_SALT_REPO_UPDATES', false) ?: mcpSaltRepoUpdateVar
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400127 }
128
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000129 common.infoMsg("Using context:\n" + context)
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000130 print prettyPrint(toJson(context))
azvyagintsev866b19a2018-11-20 18:21:43 +0200131 return context
132
133}
azvyagintsevf252b592018-08-13 18:39:14 +0300134
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000135timeout(time: 1, unit: 'HOURS') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000136 node(slaveNode) {
azvyagintsev866b19a2018-11-20 18:21:43 +0200137 def context = globalVariatorsUpdate()
azvyagintsev6d678da2018-11-28 21:19:06 +0200138 def RequesterEmail = context.get('email_address', '')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000139 def templateEnv = "${env.WORKSPACE}/template"
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200140 // modelEnv - this is reclass root, aka /srv/salt/reclass
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000141 def modelEnv = "${env.WORKSPACE}/model"
142 def testEnv = "${env.WORKSPACE}/test"
143 def pipelineEnv = "${env.WORKSPACE}/pipelines"
Tomáš Kukrál9f6260f2017-03-29 23:58:26 +0200144
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000145 try {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000146 //
147 def cutterEnv = "${env.WORKSPACE}/cutter"
148 def systemEnv = "${modelEnv}/classes/system"
149 def testResult = false
150 def user
151 wrap([$class: 'BuildUser']) {
152 user = env.BUILD_USER_ID
153 }
azvyagintsev6d678da2018-11-28 21:19:06 +0200154 currentBuild.description = "${context['cluster_name']} ${RequesterEmail}"
azvyagintsev866b19a2018-11-20 18:21:43 +0200155
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000156 stage('Download Cookiecutter template') {
157 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
158 checkout([
159 $class : 'GitSCM',
160 branches : [[name: 'FETCH_HEAD'],],
161 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: templateEnv]],
162 userRemoteConfigs: [[url: context['cookiecutter_template_url'], refspec: context['cookiecutter_template_branch'], credentialsId: gerritCredentials],],
163 ])
164 }
165 stage('Create empty reclass model') {
166 dir(path: modelEnv) {
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200167 sh 'rm -rfv .git; git init'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000168 sshagent(credentials: [gerritCredentials]) {
169 sh "git submodule add ${context['shared_reclass_url']} 'classes/system'"
170 }
171 }
172 checkout([
173 $class : 'GitSCM',
174 branches : [[name: 'FETCH_HEAD'],],
175 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: systemEnv]],
176 userRemoteConfigs: [[url: context['shared_reclass_url'], refspec: context['shared_reclass_branch'], credentialsId: gerritCredentials],],
177 ])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200178 git.commitGitChanges(modelEnv, 'Added new shared reclass submodule', "${user}@localhost", "${user}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000179 }
180
181 stage('Generate model') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000182 // GNUPGHOME environment variable is required for all gpg commands
183 // and for python.generateModel execution
Denis Egorenkoce93af32019-05-20 16:57:33 +0400184 def envOpts = ["GNUPGHOME=${env.WORKSPACE}/gpghome"]
185 withEnv(envOpts) {
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300186 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000187 sh "mkdir gpghome; chmod 700 gpghome"
Dmitry Pyzhovb5c74c72018-12-17 22:08:50 +0300188 def secretKeyID = RequesterEmail ?: "salt@${context['cluster_domain']}".toString()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300189 if (!context.get('secrets_encryption_private_key')) {
190 def batchData = """
Denis Egorenko131de5f2019-05-16 14:25:40 +0400191 %echo Generating a basic OpenPGP key for Salt-Master
192 %no-protection
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300193 Key-Type: 1
194 Key-Length: 4096
195 Expire-Date: 0
196 Name-Real: ${context['salt_master_hostname']}.${context['cluster_domain']}
197 Name-Email: ${secretKeyID}
Denis Egorenko131de5f2019-05-16 14:25:40 +0400198 %commit
199 %echo done
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300200 """.stripIndent()
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200201 writeFile file: 'gpg-batch.txt', text: batchData
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300202 sh "gpg --gen-key --batch < gpg-batch.txt"
203 sh "gpg --export-secret-key -a ${secretKeyID} > gpgkey.asc"
204 } else {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200205 writeFile file: 'gpgkey.asc', text: context['secrets_encryption_private_key']
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300206 sh "gpg --import gpgkey.asc"
Denis Egorenko131de5f2019-05-16 14:25:40 +0400207 secretKeyID = sh(returnStdout: true, script: 'gpg --list-secret-keys --with-colons | grep -E "^sec" | awk -F: \'{print \$5}\'').trim()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300208 }
209 context['secrets_encryption_key_id'] = secretKeyID
210 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400211 if (context.get('cfg_failsafe_ssh_public_key')) {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200212 writeFile file: 'failsafe-ssh-key.pub', text: context['cfg_failsafe_ssh_public_key']
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400213 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200214 if (!fileExists(new File(templateEnv, 'tox.ini').toString())) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200215 reqs = new File(templateEnv, 'requirements.txt').toString()
216 if (fileExists(reqs)) {
217 python.setupVirtualenv(cutterEnv, 'python2', [], reqs)
218 } else {
219 python.setupCookiecutterVirtualenv(cutterEnv)
220 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200221 python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, modelEnv, templateEnv, false)
222 } else {
223 // tox-based CC generated structure of reclass,from the root. Otherwise for bw compat, modelEnv
224 // still expect only lower lvl of project, aka model/classes/cluster/XXX/. So,lets dump result into
225 // temp dir, and then copy it over initial structure.
226 reclassTempRootDir = sh(script: "mktemp -d -p ${env.WORKSPACE}", returnStdout: true).trim()
Denis Egorenkoce93af32019-05-20 16:57:33 +0400227 GenerateModelToxDocker(['context': common2.dumpYAML(['default_context': context]),
228 'ccRoot' : templateEnv,
229 'outDir' : reclassTempRootDir,
230 'envOpts': envOpts])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200231 dir(modelEnv) {
232 common.warningMsg('Forming reclass-root structure...')
233 sh("cp -ra ${reclassTempRootDir}/reclass/* .")
234 }
235 }
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300236 git.commitGitChanges(modelEnv, "Create model ${context['cluster_name']}", "${user}@localhost", "${user}")
237 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000238 }
239
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000240 stage("Test") {
azvyagintsev6d678da2018-11-28 21:19:06 +0200241 if (runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000242 sh("cp -r ${modelEnv} ${testEnv}")
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200243 if (fileExists('gpgkey.asc')) {
244 common.infoMsg('gpgkey.asc found!Copy it into reclass folder for tests..')
245 sh("cp -v gpgkey.asc ${testEnv}/salt_master_pillar.asc")
246 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000247 def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
azvyagintsev5b7ec892019-05-15 16:37:34 +0300248 common.warningMsg("Attempt to run test against:\n" +
249 "DISTRIB_REVISION from ${distribRevision}\n" +
250 "updateSaltFormulasDuringTest = ${updateSaltFormulasDuringTest}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000251 try {
252 def config = [
Denis Egorenko926bc7d2019-02-19 14:27:34 +0400253 'dockerHostname' : "${context['salt_master_hostname']}",
254 'domain' : "${context['cluster_domain']}",
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000255 'reclassEnv' : testEnv,
256 'distribRevision' : distribRevision,
257 'dockerContainerName': DockerCName,
Denis Egorenkod87490e2019-02-26 20:00:41 +0400258 'testContext' : 'salt-model-node',
azvyagintsev5b7ec892019-05-15 16:37:34 +0300259 'dockerExtraOpts' : ['--memory=3g'],
260 'updateSaltFormulas' : updateSaltFormulasDuringTest
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000261 ]
262 testResult = saltModelTesting.testNode(config)
263 common.infoMsg("Test finished: SUCCESS")
264 } catch (Exception ex) {
265 common.warningMsg("Test finished: FAILED")
266 testResult = false
267 }
268 } else {
269 common.warningMsg("Test stage has been skipped!")
270 }
271 }
272 stage("Generate config drives") {
273 // apt package genisoimage is required for this stage
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000274 // download create-config-drive
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000275 def commonScriptsRepoUrl = context['mcp_common_scripts_repo'] ?: 'ssh://gerrit.mcp.mirantis.com:29418/mcp/mcp-common-scripts'
276 checkout([
277 $class : 'GitSCM',
278 branches : [[name: 'FETCH_HEAD'],],
279 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'mcp-common-scripts']],
azvyagintsev866b19a2018-11-20 18:21:43 +0200280 userRemoteConfigs: [[url: commonScriptsRepoUrl, refspec: context['mcp_common_scripts_branch'], credentialsId: gerritCredentials],],
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000281 ])
282
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400283 def outdateGeneration = false
284 if (fileExists('mcp-common-scripts/config-drive/create_config_drive.py')) {
285 sh 'cp mcp-common-scripts/config-drive/create_config_drive.py create-config-drive.py'
286 } else {
287 outdateGeneration = true
288 sh 'cp mcp-common-scripts/config-drive/create_config_drive.sh create-config-drive && chmod +x create-config-drive'
289 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000290 sh '[ -f mcp-common-scripts/config-drive/master_config.sh ] && cp mcp-common-scripts/config-drive/master_config.sh user_data || cp mcp-common-scripts/config-drive/master_config.yaml user_data'
291
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000292 sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
293 sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400294 args = [
azvyagintsev1385db92019-03-22 16:05:10 +0200295 "--user-data user_data", "--model ${modelEnv}",
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400296 "--mk-pipelines ${pipelineEnv}/mk-pipelines/", "--pipeline-library ${pipelineEnv}/pipeline-library/"
297 ]
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300298 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400299 args.add('--gpg-key gpgkey.asc')
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300300 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400301 if (context.get('cfg_failsafe_ssh_public_key')) {
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400302 if (outdateGeneration) {
303 args.add('--ssh-key failsafe-ssh-key.pub')
304 } else {
Denis Egorenko85ddf0d2019-03-18 18:11:19 +0400305 if (context.get('cfg_failsafe_user')) {
306 args.add('--ssh-keys failsafe-ssh-key.pub')
307 args.add("--cloud-user-name ${context.get('cfg_failsafe_user')}")
308 }
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400309 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400310 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000311 // load data from model
312 def smc = [:]
313 smc['SALT_MASTER_MINION_ID'] = "${context['salt_master_hostname']}.${context['cluster_domain']}"
314 smc['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400315 if (context.get('cloudinit_master_config', false)) {
316 context['cloudinit_master_config'].each { k, v ->
317 smc[k] = v
318 }
319 }
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400320 if (outdateGeneration) {
321 smc['DEPLOY_NETWORK_GW'] = context['deploy_network_gateway']
322 smc['DEPLOY_NETWORK_NETMASK'] = context['deploy_network_netmask']
323 if (context.get('deploy_network_mtu')) {
324 smc['DEPLOY_NETWORK_MTU'] = context['deploy_network_mtu']
325 }
326 smc['DNS_SERVERS'] = context['dns_server01']
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000327 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000328 smc['MCP_VERSION'] = "${context['mcp_version']}"
329 if (context['local_repositories'] == 'True') {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400330 def localRepoIP = ''
Denis Egorenkof97aec22019-02-05 14:57:33 +0400331 if (context['mcp_version'] in ['2018.4.0', '2018.8.0', '2018.8.0-milestone1', '2018.11.0']) {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400332 localRepoIP = context['local_repo_url']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400333 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}/ubuntu-xenial"
334 } else {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400335 localRepoIP = context['aptly_server_deploy_address']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400336 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}"
337 }
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400338 smc['MCP_SALT_REPO_KEY'] = "http://${localRepoIP}/public.gpg"
Denis Egorenko5f1c0d92019-09-26 09:03:57 +0000339 smc['PIPELINES_FROM_ISO'] = 'false'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000340 smc['PIPELINE_REPO_URL'] = "http://${localRepoIP}:8088"
341 smc['LOCAL_REPOS'] = 'true'
342 }
343 if (context['upstream_proxy_enabled'] == 'True') {
344 if (context['upstream_proxy_auth_enabled'] == 'True') {
345 smc['http_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
346 smc['https_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
347 } else {
348 smc['http_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
349 smc['https_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
350 }
351 }
352
353 for (i in common.entries(smc)) {
Denis Egorenko815d5022019-07-15 16:01:04 +0400354 sh "sed -i 's,export ${i[0]}=.*,export ${i[0]}=\${${i[0]}:-\"${i[1]}\"},' user_data"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000355 }
356
Denis Egorenko032a2b72019-04-03 14:56:52 +0400357 // calculate netmask
Denis Egorenko1f131b72019-04-05 14:42:48 +0400358 def deployNetworkSubnet = ''
359 if (context.get('deploy_network_subnet')) {
360 def subnet = new SubnetUtils(context['deploy_network_subnet'])
361 deployNetworkSubnet = subnet.getInfo().getNetmask()
362 } else if (context.get('deploy_network_netmask')) { // case for 2018.4.0
363 deployNetworkSubnet = context['deploy_network_netmask']
364 } else {
365 error('Neither context parameter deploy_network_subnet or deploy_network_netmask should be set!')
366 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000367 // create cfg config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400368 if (outdateGeneration) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200369 args += ["--hostname ${context['salt_master_hostname']}", "${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"]
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400370 sh "./create-config-drive ${args.join(' ')}"
371 } else {
372 args += [
373 "--name ${context['salt_master_hostname']}", "--hostname ${context['salt_master_hostname']}.${context['cluster_domain']}", "--clean-up",
Denis Egorenko032a2b72019-04-03 14:56:52 +0400374 "--ip ${context['salt_master_management_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400375 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400376 ]
azvyagintsevc4c047a2019-04-04 17:42:48 +0300377 sh "chmod 0755 create-config-drive.py ; ./create-config-drive.py ${args.join(' ')}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400378 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000379 sh("mkdir output-${context['cluster_name']} && mv ${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000380
381 // save cfg iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000382 archiveArtifacts artifacts: "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000383
384 if (context['local_repositories'] == 'True') {
385 def aptlyServerHostname = context.aptly_server_hostname
386 sh "[ -f mcp-common-scripts/config-drive/mirror_config.yaml ] && cp mcp-common-scripts/config-drive/mirror_config.yaml mirror_config || cp mcp-common-scripts/config-drive/mirror_config.sh mirror_config"
387
388 def smc_apt = [:]
389 smc_apt['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400390 if (outdateGeneration) {
391 smc_apt['APTLY_DEPLOY_IP'] = context['aptly_server_deploy_address']
392 smc_apt['APTLY_DEPLOY_NETMASK'] = context['deploy_network_netmask']
393 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000394 smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${context['cluster_domain']}"
395
396 for (i in common.entries(smc_apt)) {
397 sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config"
398 }
399
400 // create apt config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400401 if (outdateGeneration) {
402 sh "./create-config-drive --user-data mirror_config --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
403 } else {
404 args = [
Denis Egorenko032a2b72019-04-03 14:56:52 +0400405 "--ip ${context['aptly_server_deploy_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400406 "--user-data mirror_config", "--hostname ${aptlyServerHostname}.${context['cluster_domain']}", "--name ${aptlyServerHostname}", "--clean-up",
407 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400408 ]
409 sh "python ./create-config-drive.py ${args.join(' ')}"
410 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000411 sh("mv ${aptlyServerHostname}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000412
413 // save apt iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000414 archiveArtifacts artifacts: "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000415 }
416 }
417
418 stage('Save changes reclass model') {
Denis Egorenkod9865252019-04-24 15:41:57 +0400419 sh(returnStatus: true, script: "tar -czf ${context['cluster_name']}.tar.gz --exclude='*@tmp' -C ${modelEnv} .")
420 archiveArtifacts artifacts: "${context['cluster_name']}.tar.gz"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000421
azvyagintsev5400d1d2018-12-11 13:19:29 +0200422 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
Denis Egorenkof9729842019-07-08 15:53:01 +0400423 def mailSubject = "Your Salt model ${context['cluster_name']}"
424 if (context.get('send_method') == 'gcs') {
425 def gcs = new com.mirantis.mk.GoogleCloudStorage()
426 def uploadIsos = [ "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso" ]
427 if (context['local_repositories'] == 'True') {
428 uploadIsos << "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
429 }
430 // generate random hash to have uniq and unpredictable link to file
431 def randHash = common.generateRandomHashString(64)
432 def config = [
433 'creds': context['gcs_creds'],
434 'project': context['gcs_project'],
435 'dest': "gs://${context['gcs_bucket']}/${randHash}",
436 'sources': uploadIsos
437 ]
438 def fileURLs = gcs.uploadArtifactToGoogleStorageBucket(config).join(' ').replace('gs://', 'https://storage.googleapis.com/')
439 emailext(to: RequesterEmail,
440 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and available to download via next URL: ${fileURLs} within 7 days.\nEnjoy!\n\nMirantis",
441 subject: mailSubject)
442 } else {
443 emailext(to: RequesterEmail,
444 attachmentsPattern: "output-${context['cluster_name']}/*",
445 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and attached to this email.\nEnjoy!\n\nMirantis",
446 subject: mailSubject)
447 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000448 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000449 dir("output-${context['cluster_name']}") {
450 deleteDir()
451 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000452 }
453
454 // Fail, but leave possibility to get failed artifacts
azvyagintsev6d678da2018-11-28 21:19:06 +0200455 if (!testResult && runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000456 common.warningMsg('Test finished: FAILURE. Please check logs and\\or debug failed model manually!')
457 error('Test stage finished: FAILURE')
458 }
459
460 } catch (Throwable e) {
461 currentBuild.result = "FAILURE"
462 currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
463 throw e
464 } finally {
465 stage('Clean workspace directories') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000466 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000467 }
468 // common.sendNotification(currentBuild.result,"",["slack"])
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400469 stage('Save artifacts to Artifactory') {
470 def artifactory = new com.mirantis.mcp.MCPArtifactory()
azvyagintsev06dfe402019-03-22 17:58:53 +0200471 def buildProps = ["context=${context['cluster_name']}"]
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400472 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
473 buildProps.add("emailTo=${RequesterEmail}")
474 }
475 def artifactoryLink = artifactory.uploadJobArtifactsToArtifactory([
azvyagintsev06dfe402019-03-22 17:58:53 +0200476 'artifactory' : 'mcp-ci',
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400477 'artifactoryRepo': "drivetrain-local/${JOB_NAME}/${context['cluster_name']}-${BUILD_NUMBER}",
azvyagintsev06dfe402019-03-22 17:58:53 +0200478 'buildProps' : buildProps,
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400479 ])
480 currentBuild.description += "<br/>${artifactoryLink}"
481 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400482 }
Tomáš Kukrál7ded3642017-03-27 15:52:51 +0200483 }
Mikhail Ivanov9f812922017-11-07 18:52:02 +0400484}