blob: 98107b8f0d8b1eb9fad34e0832998e5460f1b433 [file] [log] [blame]
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02001/**
2 * Generate cookiecutter cluster by individual products
3 *
4 * Expected parameters:
Tomáš Kukrál7ded3642017-03-27 15:52:51 +02005 * COOKIECUTTER_TEMPLATE_CONTEXT Context parameters for the template generation.
Sergey Galkin8b87f6e2018-10-24 18:40:13 +04006 * CREDENTIALS_ID Credentials id for git
azvyagintsev6d678da2018-11-28 21:19:06 +02007 * TEST_MODEL Run syntax tests for model
azvyagintsev3ed704f2018-07-09 15:49:27 +03008 **/
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +00009import static groovy.json.JsonOutput.toJson
10import static groovy.json.JsonOutput.prettyPrint
Denis Egorenko032a2b72019-04-03 14:56:52 +040011import org.apache.commons.net.util.SubnetUtils
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020012
13common = new com.mirantis.mk.Common()
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +000014common2 = new com.mirantis.mcp.Common()
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020015git = new com.mirantis.mk.Git()
16python = new com.mirantis.mk.Python()
chnyda89191012017-05-29 15:38:35 +020017saltModelTesting = new com.mirantis.mk.SaltModelTesting()
azvyagintsev5b7ec892019-05-15 16:37:34 +030018updateSaltFormulasDuringTest = true
Tomáš Kukrál7ded3642017-03-27 15:52:51 +020019
Aleksey Zvyagintsev4a804212019-02-07 13:02:31 +000020slaveNode = env.getProperty('SLAVE_NODE') ?: 'virtual'
azvyagintsev6d678da2018-11-28 21:19:06 +020021gerritCredentials = env.getProperty('CREDENTIALS_ID') ?: 'gerrit'
22runTestModel = (env.getProperty('TEST_MODEL') ?: true).toBoolean()
azvyagintsev866b19a2018-11-20 18:21:43 +020023distribRevision = 'proposed'
24gitGuessedVersion = false
Ivan Berezovskiyf590bc62020-02-28 17:29:17 +040025aptlyServerHostname = ''
azvyagintsev866b19a2018-11-20 18:21:43 +020026
Denis Egorenkoce93af32019-05-20 16:57:33 +040027def GenerateModelToxDocker(Map params) {
28 def ccRoot = params['ccRoot']
29 def context = params['context']
30 def outDir = params['outDir']
31 def envOpts = params['envOpts']
32 def tempContextFile = new File(ccRoot, 'tempContext.yaml_' + UUID.randomUUID().toString()).toString()
33 writeFile file: tempContextFile, text: context
34 // Get Jenkins user UID and GID
35 def jenkinsUID = sh(script: 'id -u', returnStdout: true).trim()
36 def jenkinsGID = sh(script: 'id -g', returnStdout: true).trim()
37 /*
38 by default, process in image operates via root user
39 Otherwise, gpg key for model and all files managed by jenkins user
40 To make it compatible, install rrequirementfrom user, but generate model via jenkins
41 for build use upstream Ubuntu Bionic image
42 */
43 def configRun = ['distribRevision': 'nightly',
44 'envOpts' : envOpts + ["CONFIG_FILE=$tempContextFile",
45 "OUTPUT_DIR=${outDir}"
46 ],
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000047 'image': 'docker-prod-local.artifactory.mirantis.com/mirantis/cicd/jnlp-slave',
Denis Egorenkoce93af32019-05-20 16:57:33 +040048 'runCommands' : [
49 '001_prepare_generate_auto_reqs': {
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000050 sh('''
Denis Egorenkoce93af32019-05-20 16:57:33 +040051 pip install tox
52 ''')
53 },
54 // user & group can be different on host and in docker
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000055 '002_set_jenkins_id': {
56 sh("""
Denis Egorenkoce93af32019-05-20 16:57:33 +040057 usermod -u ${jenkinsUID} jenkins
Ivan Berezovskiy6c6118a2019-12-10 17:21:46 +040058 groupmod -g ${jenkinsGID} jenkins
Denis Egorenkoce93af32019-05-20 16:57:33 +040059 """)
60 },
Aleksey Zvyagintsev1cbd4d72019-05-24 14:08:19 +000061 '003_run_generate_auto': {
Denis Egorenkoce93af32019-05-20 16:57:33 +040062 print('[Cookiecutter build] Result:\n' +
63 sh(returnStdout: true, script: 'cd ' + ccRoot + '; su jenkins -c "tox -ve generate_auto" '))
64 }
65 ]
66 ]
67
68 saltModelTesting.setupDockerAndTest(configRun)
69}
70
azvyagintsev866b19a2018-11-20 18:21:43 +020071def globalVariatorsUpdate() {
72 def templateContext = readYaml text: env.COOKIECUTTER_TEMPLATE_CONTEXT
73 def context = templateContext['default_context']
74 // TODO add more check's for critical var's
75 // Since we can't pin to any '_branch' variable from context, to identify 'default git revision' -
76 // because each of them, might be 'refs/' variable, we need to add some tricky trigger of using
77 // 'release/XXX' logic. This is totall guess - so,if even those one failed, to definitely must pass
78 // correct variable finally!
Denis Egorenkof0d213d2019-06-03 19:05:40 +040079 [ context.get('cookiecutter_template_branch'), context.get('shared_reclass_branch'), context.get('mcp_common_scripts_branch') ].any { branch ->
azvyagintsev866b19a2018-11-20 18:21:43 +020080 if (branch.toString().startsWith('release/')) {
81 gitGuessedVersion = branch
82 return true
83 }
84 }
Denis Egorenkof0d213d2019-06-03 19:05:40 +040085
86 [ 'cookiecutter_template_branch', 'shared_reclass_branch', 'mcp_common_scripts_branch' ].each { repoName ->
Denis Egorenko0b283902019-06-04 13:09:15 +040087 if (context['mcp_version'] in [ "nightly", "testing", "stable" ] && ! context.get(repoName)) {
Denis Egorenkof0d213d2019-06-03 19:05:40 +040088 context[repoName] = 'master'
89 } else if (! context.get(repoName)) {
90 context[repoName] = gitGuessedVersion ?: "release/${context['mcp_version']}".toString()
91 }
azvyagintsev866b19a2018-11-20 18:21:43 +020092 }
93 //
94 distribRevision = context['mcp_version']
95 if (['master'].contains(context['mcp_version'])) {
96 distribRevision = 'nightly'
97 }
98 if (distribRevision.contains('/')) {
99 distribRevision = distribRevision.split('/')[-1]
100 }
101 // Check if we are going to test bleeding-edge release, which doesn't have binary release yet
azvyagintsev8e081642019-02-03 19:15:22 +0200102 // After 2018q4 releases, need to also check 'static' repo, for example ubuntu.
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200103 def binTest = common.checkRemoteBinary(['mcp_version': distribRevision])
azvyagintsev8e081642019-02-03 19:15:22 +0200104 if (!binTest.linux_system_repo_url || !binTest.linux_system_repo_ubuntu_url) {
105 common.errorMsg("Binary release: ${distribRevision} not exist or not full. Fallback to 'proposed'! ")
azvyagintsev866b19a2018-11-20 18:21:43 +0200106 distribRevision = 'proposed'
107 }
azvyagintsev8e081642019-02-03 19:15:22 +0200108
azvyagintsev5400d1d2018-12-11 13:19:29 +0200109 // (azvyagintsev) WA for PROD-25732
110 if (context.cookiecutter_template_url.contains('gerrit.mcp.mirantis.com/mk/cookiecutter-templates')) {
111 common.warningMsg('Apply WA for PROD-25732')
112 context.cookiecutter_template_url = 'ssh://gerrit.mcp.mirantis.com:29418/mk/cookiecutter-templates.git'
113 }
Ivan Berezovskiy6c6118a2019-12-10 17:21:46 +0400114 // check, if we are going to test clear release version, w\o any updates and patches.
115 // All 2019.2.X starting from 2019.2.1 and above have update repo which has to be used for salt-formulas.
116 // update/2019.2.0 repo can't be used b/c it points to latest 2019.2.X
117 if (!gitGuessedVersion && (distribRevision == context.mcp_version) && !(distribRevision ==~ /2019\.2\.[1-9]\d*/)) {
azvyagintsev5b7ec892019-05-15 16:37:34 +0300118 updateSaltFormulasDuringTest = false
119 }
120
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400121 if (gitGuessedVersion == 'release/proposed/2019.2.0') {
Denis Egorenko83dfe212019-09-11 16:19:01 +0400122 def mcpSaltRepoUpdateVar = 'deb [arch=amd64] http://mirror.mirantis.com/update/proposed/salt-formulas/xenial xenial main'
123 if (context.get('offline_deployment', 'False').toBoolean()) {
Denis Egorenkoa43c66e2019-09-11 16:40:56 +0400124 mcpSaltRepoUpdateVar = "deb [arch=amd64] http://${context.get('aptly_server_deploy_address')}/update/proposed/salt-formulas/xenial xenial main".toString()
Denis Egorenko83dfe212019-09-11 16:19:01 +0400125 }
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400126 // CFG node in 2019.2.X update has to be bootstrapped with update/proposed repository for salt formulas
127 context['cloudinit_master_config'] = context.get('cloudinit_master_config', false) ?: [:]
Denis Egorenko83dfe212019-09-11 16:19:01 +0400128 context['cloudinit_master_config']['MCP_SALT_REPO_UPDATES'] = context['cloudinit_master_config'].get('MCP_SALT_REPO_UPDATES', false) ?: mcpSaltRepoUpdateVar
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400129 }
130
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000131 common.infoMsg("Using context:\n" + context)
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000132 print prettyPrint(toJson(context))
azvyagintsev866b19a2018-11-20 18:21:43 +0200133 return context
134
135}
azvyagintsevf252b592018-08-13 18:39:14 +0300136
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000137timeout(time: 1, unit: 'HOURS') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000138 node(slaveNode) {
azvyagintsev866b19a2018-11-20 18:21:43 +0200139 def context = globalVariatorsUpdate()
azvyagintsev6d678da2018-11-28 21:19:06 +0200140 def RequesterEmail = context.get('email_address', '')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000141 def templateEnv = "${env.WORKSPACE}/template"
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200142 // modelEnv - this is reclass root, aka /srv/salt/reclass
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000143 def modelEnv = "${env.WORKSPACE}/model"
144 def testEnv = "${env.WORKSPACE}/test"
145 def pipelineEnv = "${env.WORKSPACE}/pipelines"
Tomáš Kukrál9f6260f2017-03-29 23:58:26 +0200146
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000147 try {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000148 //
149 def cutterEnv = "${env.WORKSPACE}/cutter"
150 def systemEnv = "${modelEnv}/classes/system"
151 def testResult = false
152 def user
153 wrap([$class: 'BuildUser']) {
154 user = env.BUILD_USER_ID
155 }
azvyagintsev6d678da2018-11-28 21:19:06 +0200156 currentBuild.description = "${context['cluster_name']} ${RequesterEmail}"
azvyagintsev866b19a2018-11-20 18:21:43 +0200157
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000158 stage('Download Cookiecutter template') {
159 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
160 checkout([
161 $class : 'GitSCM',
162 branches : [[name: 'FETCH_HEAD'],],
163 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: templateEnv]],
164 userRemoteConfigs: [[url: context['cookiecutter_template_url'], refspec: context['cookiecutter_template_branch'], credentialsId: gerritCredentials],],
165 ])
166 }
167 stage('Create empty reclass model') {
168 dir(path: modelEnv) {
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200169 sh 'rm -rfv .git; git init'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000170 sshagent(credentials: [gerritCredentials]) {
171 sh "git submodule add ${context['shared_reclass_url']} 'classes/system'"
172 }
173 }
174 checkout([
175 $class : 'GitSCM',
176 branches : [[name: 'FETCH_HEAD'],],
177 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: systemEnv]],
178 userRemoteConfigs: [[url: context['shared_reclass_url'], refspec: context['shared_reclass_branch'], credentialsId: gerritCredentials],],
179 ])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200180 git.commitGitChanges(modelEnv, 'Added new shared reclass submodule', "${user}@localhost", "${user}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000181 }
182
183 stage('Generate model') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000184 // GNUPGHOME environment variable is required for all gpg commands
185 // and for python.generateModel execution
Denis Egorenkoce93af32019-05-20 16:57:33 +0400186 def envOpts = ["GNUPGHOME=${env.WORKSPACE}/gpghome"]
187 withEnv(envOpts) {
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300188 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000189 sh "mkdir gpghome; chmod 700 gpghome"
Dmitry Pyzhovb5c74c72018-12-17 22:08:50 +0300190 def secretKeyID = RequesterEmail ?: "salt@${context['cluster_domain']}".toString()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300191 if (!context.get('secrets_encryption_private_key')) {
192 def batchData = """
Denis Egorenko131de5f2019-05-16 14:25:40 +0400193 %echo Generating a basic OpenPGP key for Salt-Master
194 %no-protection
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300195 Key-Type: 1
196 Key-Length: 4096
197 Expire-Date: 0
198 Name-Real: ${context['salt_master_hostname']}.${context['cluster_domain']}
199 Name-Email: ${secretKeyID}
Denis Egorenko131de5f2019-05-16 14:25:40 +0400200 %commit
201 %echo done
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300202 """.stripIndent()
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200203 writeFile file: 'gpg-batch.txt', text: batchData
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300204 sh "gpg --gen-key --batch < gpg-batch.txt"
205 sh "gpg --export-secret-key -a ${secretKeyID} > gpgkey.asc"
206 } else {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200207 writeFile file: 'gpgkey.asc', text: context['secrets_encryption_private_key']
Vladimir Khlyunev90072962021-01-25 19:19:48 +0400208 sh "gpg --batch --import gpgkey.asc"
Denis Egorenko131de5f2019-05-16 14:25:40 +0400209 secretKeyID = sh(returnStdout: true, script: 'gpg --list-secret-keys --with-colons | grep -E "^sec" | awk -F: \'{print \$5}\'').trim()
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300210 }
211 context['secrets_encryption_key_id'] = secretKeyID
212 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400213 if (context.get('cfg_failsafe_ssh_public_key')) {
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200214 writeFile file: 'failsafe-ssh-key.pub', text: context['cfg_failsafe_ssh_public_key']
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400215 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200216 if (!fileExists(new File(templateEnv, 'tox.ini').toString())) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200217 reqs = new File(templateEnv, 'requirements.txt').toString()
218 if (fileExists(reqs)) {
219 python.setupVirtualenv(cutterEnv, 'python2', [], reqs)
220 } else {
221 python.setupCookiecutterVirtualenv(cutterEnv)
222 }
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200223 python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, modelEnv, templateEnv, false)
224 } else {
225 // tox-based CC generated structure of reclass,from the root. Otherwise for bw compat, modelEnv
226 // still expect only lower lvl of project, aka model/classes/cluster/XXX/. So,lets dump result into
227 // temp dir, and then copy it over initial structure.
228 reclassTempRootDir = sh(script: "mktemp -d -p ${env.WORKSPACE}", returnStdout: true).trim()
Denis Egorenkoce93af32019-05-20 16:57:33 +0400229 GenerateModelToxDocker(['context': common2.dumpYAML(['default_context': context]),
230 'ccRoot' : templateEnv,
231 'outDir' : reclassTempRootDir,
232 'envOpts': envOpts])
azvyagintsev4f34f7a2019-02-26 18:47:37 +0200233 dir(modelEnv) {
234 common.warningMsg('Forming reclass-root structure...')
235 sh("cp -ra ${reclassTempRootDir}/reclass/* .")
236 }
237 }
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300238 git.commitGitChanges(modelEnv, "Create model ${context['cluster_name']}", "${user}@localhost", "${user}")
239 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000240 }
241
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000242 stage("Test") {
azvyagintsev6d678da2018-11-28 21:19:06 +0200243 if (runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000244 sh("cp -r ${modelEnv} ${testEnv}")
azvyagintsev7d6d46c2019-02-11 14:25:41 +0200245 if (fileExists('gpgkey.asc')) {
246 common.infoMsg('gpgkey.asc found!Copy it into reclass folder for tests..')
247 sh("cp -v gpgkey.asc ${testEnv}/salt_master_pillar.asc")
248 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000249 def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
azvyagintsev5b7ec892019-05-15 16:37:34 +0300250 common.warningMsg("Attempt to run test against:\n" +
251 "DISTRIB_REVISION from ${distribRevision}\n" +
252 "updateSaltFormulasDuringTest = ${updateSaltFormulasDuringTest}")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000253 try {
254 def config = [
Denis Egorenko926bc7d2019-02-19 14:27:34 +0400255 'dockerHostname' : "${context['salt_master_hostname']}",
256 'domain' : "${context['cluster_domain']}",
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000257 'reclassEnv' : testEnv,
258 'distribRevision' : distribRevision,
259 'dockerContainerName': DockerCName,
Denis Egorenkod87490e2019-02-26 20:00:41 +0400260 'testContext' : 'salt-model-node',
azvyagintsev5b7ec892019-05-15 16:37:34 +0300261 'dockerExtraOpts' : ['--memory=3g'],
262 'updateSaltFormulas' : updateSaltFormulasDuringTest
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000263 ]
Denis Egorenko19a793d2019-11-18 20:50:59 +0400264 if (gitGuessedVersion == 'release/proposed/2019.2.0') {
265 config['updateSaltFormulasRev'] = 'proposed'
266 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000267 testResult = saltModelTesting.testNode(config)
268 common.infoMsg("Test finished: SUCCESS")
269 } catch (Exception ex) {
270 common.warningMsg("Test finished: FAILED")
271 testResult = false
272 }
273 } else {
274 common.warningMsg("Test stage has been skipped!")
275 }
276 }
277 stage("Generate config drives") {
278 // apt package genisoimage is required for this stage
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000279 // download create-config-drive
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000280 def commonScriptsRepoUrl = context['mcp_common_scripts_repo'] ?: 'ssh://gerrit.mcp.mirantis.com:29418/mcp/mcp-common-scripts'
281 checkout([
282 $class : 'GitSCM',
283 branches : [[name: 'FETCH_HEAD'],],
284 extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'mcp-common-scripts']],
azvyagintsev866b19a2018-11-20 18:21:43 +0200285 userRemoteConfigs: [[url: commonScriptsRepoUrl, refspec: context['mcp_common_scripts_branch'], credentialsId: gerritCredentials],],
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000286 ])
287
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400288 def outdateGeneration = false
289 if (fileExists('mcp-common-scripts/config-drive/create_config_drive.py')) {
290 sh 'cp mcp-common-scripts/config-drive/create_config_drive.py create-config-drive.py'
291 } else {
292 outdateGeneration = true
293 sh 'cp mcp-common-scripts/config-drive/create_config_drive.sh create-config-drive && chmod +x create-config-drive'
294 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000295 sh '[ -f mcp-common-scripts/config-drive/master_config.sh ] && cp mcp-common-scripts/config-drive/master_config.sh user_data || cp mcp-common-scripts/config-drive/master_config.yaml user_data'
296
Denis Egorenkoc1e1d042019-04-26 08:55:27 +0000297 sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
298 sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400299 args = [
azvyagintsev1385db92019-03-22 16:05:10 +0200300 "--user-data user_data", "--model ${modelEnv}",
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400301 "--mk-pipelines ${pipelineEnv}/mk-pipelines/", "--pipeline-library ${pipelineEnv}/pipeline-library/"
302 ]
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300303 if (context['secrets_encryption_enabled'] == 'True') {
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400304 args.add('--gpg-key gpgkey.asc')
Dmitry Pyzhov089fb4f2018-12-11 16:58:00 +0300305 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400306 if (context.get('cfg_failsafe_ssh_public_key')) {
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400307 if (outdateGeneration) {
308 args.add('--ssh-key failsafe-ssh-key.pub')
309 } else {
Denis Egorenko85ddf0d2019-03-18 18:11:19 +0400310 if (context.get('cfg_failsafe_user')) {
311 args.add('--ssh-keys failsafe-ssh-key.pub')
312 args.add("--cloud-user-name ${context.get('cfg_failsafe_user')}")
313 }
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400314 }
Stanislav Riazanovda45ea02018-12-21 16:12:50 +0400315 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000316 // load data from model
317 def smc = [:]
318 smc['SALT_MASTER_MINION_ID'] = "${context['salt_master_hostname']}.${context['cluster_domain']}"
319 smc['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Ivan Berezovskiy1e66f502019-05-24 18:11:17 +0400320 if (context.get('cloudinit_master_config', false)) {
321 context['cloudinit_master_config'].each { k, v ->
322 smc[k] = v
323 }
324 }
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400325 if (outdateGeneration) {
326 smc['DEPLOY_NETWORK_GW'] = context['deploy_network_gateway']
327 smc['DEPLOY_NETWORK_NETMASK'] = context['deploy_network_netmask']
328 if (context.get('deploy_network_mtu')) {
329 smc['DEPLOY_NETWORK_MTU'] = context['deploy_network_mtu']
330 }
331 smc['DNS_SERVERS'] = context['dns_server01']
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000332 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000333 smc['MCP_VERSION'] = "${context['mcp_version']}"
334 if (context['local_repositories'] == 'True') {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400335 def localRepoIP = ''
Denis Egorenkof97aec22019-02-05 14:57:33 +0400336 if (context['mcp_version'] in ['2018.4.0', '2018.8.0', '2018.8.0-milestone1', '2018.11.0']) {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400337 localRepoIP = context['local_repo_url']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400338 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}/ubuntu-xenial"
339 } else {
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400340 localRepoIP = context['aptly_server_deploy_address']
Denis Egorenkof97aec22019-02-05 14:57:33 +0400341 smc['MCP_SALT_REPO_URL'] = "http://${localRepoIP}"
342 }
Denis Egorenko6bfa7552019-02-05 19:09:25 +0400343 smc['MCP_SALT_REPO_KEY'] = "http://${localRepoIP}/public.gpg"
Denis Egorenko5f1c0d92019-09-26 09:03:57 +0000344 smc['PIPELINES_FROM_ISO'] = 'false'
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000345 smc['PIPELINE_REPO_URL'] = "http://${localRepoIP}:8088"
346 smc['LOCAL_REPOS'] = 'true'
347 }
348 if (context['upstream_proxy_enabled'] == 'True') {
349 if (context['upstream_proxy_auth_enabled'] == 'True') {
350 smc['http_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
351 smc['https_proxy'] = 'http://' + context['upstream_proxy_user'] + ':' + context['upstream_proxy_password'] + '@' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
352 } else {
353 smc['http_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
354 smc['https_proxy'] = 'http://' + context['upstream_proxy_address'] + ':' + context['upstream_proxy_port']
355 }
356 }
357
358 for (i in common.entries(smc)) {
Denis Egorenko815d5022019-07-15 16:01:04 +0400359 sh "sed -i 's,export ${i[0]}=.*,export ${i[0]}=\${${i[0]}:-\"${i[1]}\"},' user_data"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000360 }
361
Denis Egorenko032a2b72019-04-03 14:56:52 +0400362 // calculate netmask
Denis Egorenko1f131b72019-04-05 14:42:48 +0400363 def deployNetworkSubnet = ''
364 if (context.get('deploy_network_subnet')) {
365 def subnet = new SubnetUtils(context['deploy_network_subnet'])
366 deployNetworkSubnet = subnet.getInfo().getNetmask()
367 } else if (context.get('deploy_network_netmask')) { // case for 2018.4.0
368 deployNetworkSubnet = context['deploy_network_netmask']
369 } else {
370 error('Neither context parameter deploy_network_subnet or deploy_network_netmask should be set!')
371 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000372 // create cfg config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400373 if (outdateGeneration) {
azvyagintsev06dfe402019-03-22 17:58:53 +0200374 args += ["--hostname ${context['salt_master_hostname']}", "${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"]
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400375 sh "./create-config-drive ${args.join(' ')}"
376 } else {
377 args += [
378 "--name ${context['salt_master_hostname']}", "--hostname ${context['salt_master_hostname']}.${context['cluster_domain']}", "--clean-up",
Denis Egorenko032a2b72019-04-03 14:56:52 +0400379 "--ip ${context['salt_master_management_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400380 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400381 ]
azvyagintsevc4c047a2019-04-04 17:42:48 +0300382 sh "chmod 0755 create-config-drive.py ; ./create-config-drive.py ${args.join(' ')}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400383 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000384 sh("mkdir output-${context['cluster_name']} && mv ${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000385
386 // save cfg iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000387 archiveArtifacts artifacts: "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000388
389 if (context['local_repositories'] == 'True') {
Ivan Berezovskiyf590bc62020-02-28 17:29:17 +0400390 aptlyServerHostname = context.aptly_server_hostname
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000391 sh "[ -f mcp-common-scripts/config-drive/mirror_config.yaml ] && cp mcp-common-scripts/config-drive/mirror_config.yaml mirror_config || cp mcp-common-scripts/config-drive/mirror_config.sh mirror_config"
392
393 def smc_apt = [:]
394 smc_apt['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400395 if (outdateGeneration) {
396 smc_apt['APTLY_DEPLOY_IP'] = context['aptly_server_deploy_address']
397 smc_apt['APTLY_DEPLOY_NETMASK'] = context['deploy_network_netmask']
398 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000399 smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${context['cluster_domain']}"
400
401 for (i in common.entries(smc_apt)) {
402 sh "sed -i \"s,export ${i[0]}=.*,export ${i[0]}=${i[1]},\" mirror_config"
403 }
404
405 // create apt config-drive
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400406 if (outdateGeneration) {
407 sh "./create-config-drive --user-data mirror_config --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
408 } else {
409 args = [
Denis Egorenko032a2b72019-04-03 14:56:52 +0400410 "--ip ${context['aptly_server_deploy_address']}", "--netmask ${deployNetworkSubnet}", "--gateway ${context['deploy_network_gateway']}",
Denis Egorenkoaaeda9b2019-02-28 20:55:59 +0400411 "--user-data mirror_config", "--hostname ${aptlyServerHostname}.${context['cluster_domain']}", "--name ${aptlyServerHostname}", "--clean-up",
412 "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
Denis Egorenkoaeaa0132019-02-25 16:55:08 +0400413 ]
414 sh "python ./create-config-drive.py ${args.join(' ')}"
415 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000416 sh("mv ${aptlyServerHostname}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000417
418 // save apt iso to artifacts
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000419 archiveArtifacts artifacts: "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000420 }
421 }
422
423 stage('Save changes reclass model') {
Denis Egorenkod9865252019-04-24 15:41:57 +0400424 sh(returnStatus: true, script: "tar -czf ${context['cluster_name']}.tar.gz --exclude='*@tmp' -C ${modelEnv} .")
425 archiveArtifacts artifacts: "${context['cluster_name']}.tar.gz"
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000426
azvyagintsev5400d1d2018-12-11 13:19:29 +0200427 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
Denis Egorenkof9729842019-07-08 15:53:01 +0400428 def mailSubject = "Your Salt model ${context['cluster_name']}"
429 if (context.get('send_method') == 'gcs') {
430 def gcs = new com.mirantis.mk.GoogleCloudStorage()
431 def uploadIsos = [ "output-${context['cluster_name']}/${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso" ]
432 if (context['local_repositories'] == 'True') {
433 uploadIsos << "output-${context['cluster_name']}/${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
434 }
435 // generate random hash to have uniq and unpredictable link to file
436 def randHash = common.generateRandomHashString(64)
437 def config = [
438 'creds': context['gcs_creds'],
439 'project': context['gcs_project'],
440 'dest': "gs://${context['gcs_bucket']}/${randHash}",
441 'sources': uploadIsos
442 ]
443 def fileURLs = gcs.uploadArtifactToGoogleStorageBucket(config).join(' ').replace('gs://', 'https://storage.googleapis.com/')
444 emailext(to: RequesterEmail,
445 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and available to download via next URL: ${fileURLs} within 7 days.\nEnjoy!\n\nMirantis",
446 subject: mailSubject)
447 } else {
448 emailext(to: RequesterEmail,
449 attachmentsPattern: "output-${context['cluster_name']}/*",
450 body: "Mirantis Jenkins\n\nRequested reclass model ${context['cluster_name']} has been created and attached to this email.\nEnjoy!\n\nMirantis",
451 subject: mailSubject)
452 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000453 }
Aleksey Zvyagintsevaa235f22019-04-24 12:46:42 +0000454 dir("output-${context['cluster_name']}") {
455 deleteDir()
456 }
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000457 }
458
459 // Fail, but leave possibility to get failed artifacts
azvyagintsev6d678da2018-11-28 21:19:06 +0200460 if (!testResult && runTestModel) {
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000461 common.warningMsg('Test finished: FAILURE. Please check logs and\\or debug failed model manually!')
462 error('Test stage finished: FAILURE')
463 }
464
465 } catch (Throwable e) {
466 currentBuild.result = "FAILURE"
467 currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
468 throw e
469 } finally {
470 stage('Clean workspace directories') {
Aleksey Zvyagintsev7ca28d22019-02-25 11:05:49 +0000471 sh(script: 'find . -mindepth 1 -delete > /dev/null || true')
Aleksey Zvyagintsevb16902d2018-10-29 12:33:48 +0000472 }
473 // common.sendNotification(currentBuild.result,"",["slack"])
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400474 stage('Save artifacts to Artifactory') {
475 def artifactory = new com.mirantis.mcp.MCPArtifactory()
azvyagintsev06dfe402019-03-22 17:58:53 +0200476 def buildProps = ["context=${context['cluster_name']}"]
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400477 if (RequesterEmail != '' && !RequesterEmail.contains('example')) {
478 buildProps.add("emailTo=${RequesterEmail}")
479 }
480 def artifactoryLink = artifactory.uploadJobArtifactsToArtifactory([
azvyagintsev06dfe402019-03-22 17:58:53 +0200481 'artifactory' : 'mcp-ci',
Ivan Udovichenkoa6d5e212021-01-18 16:03:47 +0300482 'artifactoryRepo': "artifactory/drivetrain-local/${JOB_NAME}/${context['cluster_name']}-${BUILD_NUMBER}",
azvyagintsev06dfe402019-03-22 17:58:53 +0200483 'buildProps' : buildProps,
Denis Egorenko34c4a3b2019-03-12 12:48:15 +0400484 ])
485 currentBuild.description += "<br/>${artifactoryLink}"
486 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400487 }
Tomáš Kukrál7ded3642017-03-27 15:52:51 +0200488 }
Mikhail Ivanov9f812922017-11-07 18:52:02 +0400489}