blob: 700ba24894d15061e29969d3a664ae338a1804db [file] [log] [blame]
Denis Egorenko358a4fc2018-11-27 17:57:22 +04001/*
2 Global wrapper for testing next projects:
3 - salt-models/reclass-system
4 - mk/cookiecutter-templates
5
6 Can be triggered manually or by gerrit trigger:
7 1) gerrit trigger
8 Automatically switches if GERRIT_PROJECT variable detected
9 Always test GERRIT_REFSPEC VS GERRIT_BRANCH-master version of opposite project
10
11 2) manual run via job-build , possible to pass refspecs
12 - for CC
13 - Reclass
14
15 Example of TEST_PARAMETERS_YAML manual config:
16---
17RECLASS_SYSTEM_URL: ssh://mcp-jenkins@gerrit.mcp.mirantis.net:29418/salt-models/reclass-system
18RECLASS_SYSTEM_GIT_REF: 2018.11.0
19RECLASS_SYSTEM_BRANCH: refs/heads/2018.11.0
20COOKIECUTTER_TEMPLATE_URL: ssh://mcp-jenkins@gerrit.mcp.mirantis.net:29418/mk/cookiecutter-templates
21COOKIECUTTER_TEMPLATE_REF: refs/heads/2018.11.0
22COOKIECUTTER_TEMPLATE_BRANCH: 2018.11.0
23DISTRIB_REVISION: 2018.11.0
24TEST_MODELS: ''
25
26 */
27
28import groovy.json.JsonOutput
Denis Egorenko6c325112018-12-05 19:58:34 +040029gerrit = new com.mirantis.mk.Gerrit()
Denis Egorenko358a4fc2018-11-27 17:57:22 +040030
31cookiecutterTemplatesRepo='mk/cookiecutter-templates'
32reclassSystemRepo='salt-models/reclass-system'
33slaveNode = env.getProperty('SLAVE_NODE') ?: 'python&&docker'
34
Denis Egorenko76a0be82018-12-04 13:36:07 +040035voteMatrix = [
36 'test-mk-cookiecutter-templates': true,
37 'test-drivetrain': true,
38 'oscore-test-cookiecutter-models': false,
39 'test-salt-model-infra': true,
40 'test-salt-model-mcp-virtual-lab': true,
41]
42
Denis Egorenko6c325112018-12-05 19:58:34 +040043baseGerritConfig = [:]
Denis Egorenkof4d3f522018-12-06 16:33:11 +040044jobResultComments = [:]
45commentLock = false
Denis Egorenko6c325112018-12-05 19:58:34 +040046
Denis Egorenko358a4fc2018-11-27 17:57:22 +040047LinkedHashMap getManualRefParams(LinkedHashMap map) {
48 LinkedHashMap manualParams = [:]
Denis Egorenko358a4fc2018-11-27 17:57:22 +040049 if (map.containsKey('RECLASS_SYSTEM_GIT_REF') && map.containsKey('RECLASS_SYSTEM_URL')) {
50 manualParams[reclassSystemRepo] = [
51 'url': map.get('RECLASS_SYSTEM_URL'),
52 'ref': map.get('RECLASS_SYSTEM_GIT_REF'),
53 'branch': map.get('RECLASS_SYSTEM_BRANCH', 'master'),
54 ]
55 }
56 if (map.containsKey('COOKIECUTTER_TEMPLATE_REF') && map.containsKey('COOKIECUTTER_TEMPLATE_URL')) {
57 manualParams[cookiecutterTemplatesRepo] = [
58 'url': map.get('COOKIECUTTER_TEMPLATE_URL'),
59 'ref': map.get('COOKIECUTTER_TEMPLATE_REF'),
60 'branch': map.get('COOKIECUTTER_TEMPLATE_BRANCH', 'master'),
61 ]
62 }
63 return manualParams
64}
65
Denis Egorenkof4d3f522018-12-06 16:33:11 +040066def setGerritReviewComment(Boolean initComment = false) {
Denis Egorenko6c325112018-12-05 19:58:34 +040067 if (baseGerritConfig) {
Denis Egorenkof4d3f522018-12-06 16:33:11 +040068 while(commentLock) {
69 sleep 5
70 }
71 commentLock = true
Denis Egorenko6c325112018-12-05 19:58:34 +040072 LinkedHashMap config = baseGerritConfig.clone()
Denis Egorenkof4d3f522018-12-06 16:33:11 +040073 String jobResultComment = ''
74 jobResultComments.each { job, info ->
75 String skipped = ''
76 if (!initComment) {
77 skipped = voteMatrix.get(job, 'true') ? '' : '(skipped)'
78 }
79 jobResultComment += "- ${job} ${info.url}console : ${info.status} ${skipped}".trim() + '\n'
80 }
81 config['message'] = sh(script: "echo '${jobResultComment}'", returnStdout: true).trim()
Denis Egorenko6c325112018-12-05 19:58:34 +040082 gerrit.postGerritComment(config)
Denis Egorenkof4d3f522018-12-06 16:33:11 +040083 commentLock = false
Denis Egorenko6c325112018-12-05 19:58:34 +040084 }
Denis Egorenko76a0be82018-12-04 13:36:07 +040085}
86
87def runTests(String jobName, String extraVars) {
88 def propagateStatus = voteMatrix.get(jobName, true)
Denis Egorenko358a4fc2018-11-27 17:57:22 +040089 return {
Denis Egorenkof4d3f522018-12-06 16:33:11 +040090 def jobBuild = build job: jobName, propagate: false, parameters: [
Denis Egorenko76a0be82018-12-04 13:36:07 +040091 [$class: 'TextParameterValue', name: 'EXTRA_VARIABLES_YAML', value: extraVars ]
92 ]
Denis Egorenkof4d3f522018-12-06 16:33:11 +040093 jobResultComments[jobName] = [ 'url': jobBuild.absoluteUrl, 'status': jobBuild.result ]
94 setGerritReviewComment()
Denis Egorenko76a0be82018-12-04 13:36:07 +040095 if (propagateStatus && jobBuild.result == 'FAILURE') {
96 throw new Exception("Build ${jobName} is failed!")
Denis Egorenko358a4fc2018-11-27 17:57:22 +040097 }
98 }
99}
100
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400101def runTestSaltModelReclass(String jobName, String defaultGitUrl, String clusterGitUrl, String refSpec) {
102 def propagateStatus = voteMatrix.get(jobName, true)
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400103 return {
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400104 def jobBuild = build job: jobName, propagate: false, parameters: [
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400105 [$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: clusterGitUrl],
106 [$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: "HEAD"],
107 [$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: defaultGitUrl],
108 [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: refSpec ],
109 ]
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400110 jobResultComments[jobName] = [ 'url': jobBuild.absoluteUrl, 'status': jobBuild.result ]
111 setGerritReviewComment()
Denis Egorenko76a0be82018-12-04 13:36:07 +0400112 if (propagateStatus && jobBuild.result == 'FAILURE') {
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400113 throw new Exception("Build ${jobName} is failed!")
Denis Egorenko76a0be82018-12-04 13:36:07 +0400114 }
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400115 }
116}
117
Denis Egorenko6c325112018-12-05 19:58:34 +0400118def checkReclassSystemDocumentationCommit(gerritCredentials) {
119 gerrit.gerritPatchsetCheckout([
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400120 credentialsId: gerritCredentials
121 ])
122
123 sh("git diff-tree --no-commit-id --diff-filter=d --name-only -r HEAD | grep .yml | xargs -I {} python -c \"import yaml; yaml.load(open('{}', 'r'))\" \\;")
124
125 return sh(script: "git diff-tree --no-commit-id --name-only -r HEAD | grep -v .releasenotes", returnStatus: true) == 1
126
127}
128
129timeout(time: 12, unit: 'HOURS') {
130 node(slaveNode) {
131 def common = new com.mirantis.mk.Common()
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400132 def git = new com.mirantis.mk.Git()
133 def python = new com.mirantis.mk.Python()
134
135 // Var TEST_PARAMETERS_YAML contains any additional parameters for tests,
136 // like manually specified Gerrit Refs/URLs, additional parameters and so on
Denis Egorenko76a0be82018-12-04 13:36:07 +0400137 def buildTestParams = [:]
138 def buildTestParamsYaml = env.getProperty('TEST_PARAMETERS_YAML')
139 if (buildTestParamsYaml) {
140 common.mergeEnv(env, buildTestParamsYaml)
141 buildTestParams = readYaml text: buildTestParamsYaml
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400142 }
143
144 // init required job variables
145 LinkedHashMap job_env = env.getEnvironment().findAll { k, v -> v }
146
147 // Gerrit parameters
148 String gerritCredentials = job_env.get('CREDENTIALS_ID', 'gerrit')
149 String gerritRef = job_env.get('GERRIT_REFSPEC', '')
150 String gerritProject = ''
151 String gerritName = ''
152 String gerritScheme = ''
153 String gerritHost = ''
154 String gerritPort = ''
155 String gerritChangeNumber = ''
156
157 // Common and manual build parameters
158 LinkedHashMap projectsMap = [:]
Denis Egorenkoff770c62018-12-03 17:45:34 +0400159 String distribRevision = job_env.get('DISTRIB_REVISION', 'nightly')
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400160 ArrayList testModels = job_env.get('TEST_MODELS', 'mcp-virtual-lab,infra').split(',')
161
162 stage('Check build mode') {
163 def buildType = ''
164 if (gerritRef) {
165 // job is triggered by Gerrit, get all required Gerrit vars
166 gerritProject = job_env.get('GERRIT_PROJECT')
167 gerritName = job_env.get('GERRIT_NAME')
168 gerritScheme = job_env.get('GERRIT_SCHEME')
169 gerritHost = job_env.get('GERRIT_HOST')
170 gerritPort = job_env.get('GERRIT_PORT')
171 gerritChangeNumber = job_env.get('GERRIT_CHANGE_NUMBER')
Denis Egorenko6c325112018-12-05 19:58:34 +0400172 gerritPatchSetNumber = job_env.get('GERRIT_PATCHSET_NUMBER')
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400173 gerritBranch = job_env.get('GERRIT_BRANCH')
174
175 // check if change aren't already merged
176 def gerritChange = gerrit.getGerritChange(gerritName, gerritHost, gerritChangeNumber, gerritCredentials)
177 if (gerritChange.status == "MERGED") {
178 common.successMsg('Patch set is alredy merged, no need to test it')
179 currentBuild.result = 'SUCCESS'
180 return
181 }
182
183 projectsMap[gerritProject] = [
184 'url': "${gerritScheme}://${gerritName}@${gerritHost}:${gerritPort}/${gerritProject}",
185 'ref': gerritRef,
186 'branch': gerritBranch,
187 ]
188 buildType = 'Gerrit Trigger'
Denis Egorenko76a0be82018-12-04 13:36:07 +0400189 buildTestParams << job_env.findAll { k,v -> k ==~ /GERRIT_.+/ }
Denis Egorenko6c325112018-12-05 19:58:34 +0400190 baseGerritConfig = [
191 'gerritName': gerritName,
192 'gerritHost': gerritHost,
193 'gerritChangeNumber': gerritChangeNumber,
194 'credentialsId': gerritCredentials,
195 'gerritPatchSetNumber': gerritPatchSetNumber,
196 ]
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400197 } else {
198 projectsMap = getManualRefParams(job_env)
199 if (!projectsMap) {
200 error('Manual build detected and no valid Git refs provided!')
201 }
202 buildType = 'Manual build'
203 }
204 ArrayList descriptionMsgs = [ "<font color='red'>${buildType} detected!</font> Running with next parameters:" ]
205 for(String project in projectsMap.keySet()) {
Denis Egorenko7a806bb2018-12-03 14:37:05 +0400206 descriptionMsgs.add("Ref for ${project} => ${projectsMap[project]['ref']}")
207 descriptionMsgs.add("Branch for ${project} => ${projectsMap[project]['branch']}")
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400208 }
209 descriptionMsgs.add("Distrib revision => ${distribRevision}")
Denis Egorenko76a0be82018-12-04 13:36:07 +0400210 currentBuild.description = descriptionMsgs.join('<br/>')
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400211 }
212
213 stage("Run tests") {
214 def branches = [:]
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400215 String branchJobName = ''
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400216
217 if (projectsMap.containsKey(reclassSystemRepo)) {
Denis Egorenko6c325112018-12-05 19:58:34 +0400218 def documentationOnly = checkReclassSystemDocumentationCommit(gerritCredentials)
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400219 if (['master'].contains(gerritBranch) && !documentationOnly) {
220 for (int i = 0; i < testModels.size(); i++) {
221 def cluster = testModels[i]
Denis Egorenko76a0be82018-12-04 13:36:07 +0400222 def clusterGitUrl = projectsMap[reclassSystemRepo]['url'].substring(0, projectsMap[reclassSystemRepo]['url'].lastIndexOf("/") + 1) + cluster
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400223 branchJobName = "test-salt-model-${cluster}"
224 branches[branchJobName] = runTestSaltModelReclass(branchJobName, projectsMap[reclassSystemRepo]['url'], clusterGitUrl, projectsMap[reclassSystemRepo]['ref'])
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400225 }
226 } else {
227 common.warningMsg("Tests for ${testModels} skipped!")
228 }
229 }
230 if (projectsMap.containsKey(reclassSystemRepo) || projectsMap.containsKey(cookiecutterTemplatesRepo)) {
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400231 branchJobName = 'test-mk-cookiecutter-templates'
232 branches[branchJobName] = runTests(branchJobName, JsonOutput.toJson(buildTestParams))
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400233 }
234 if (projectsMap.containsKey(cookiecutterTemplatesRepo)) {
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400235 branchJobName = 'test-drivetrain'
236 branches[branchJobName] = runTests(branchJobName, JsonOutput.toJson(buildTestParams))
Denis Egorenko76a0be82018-12-04 13:36:07 +0400237 // TODO: enable oscore-test job once it's ready to consume EXTRA_VARIABLES_YAML
238 //branches['oscore-test-cookiecutter-models'] = runTests('oscore-test-cookiecutter-models', JsonOutput.toJson(buildTestParams))
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400239 }
240
Denis Egorenkof4d3f522018-12-06 16:33:11 +0400241 branches.keySet().each { key ->
242 if (branches[key] instanceof Closure) {
243 jobResultComments[key] = [ 'url': job_env.get('BUILD_URL'), 'status': 'WAITING' ]
244 }
245 }
246 setGerritReviewComment(true)
Denis Egorenko76a0be82018-12-04 13:36:07 +0400247 try {
248 parallel branches
249 } catch (Exception e) {
250 println e
251 println 'Job is in non-voting mode for now. Skipping fails.'
252 }
Denis Egorenko358a4fc2018-11-27 17:57:22 +0400253 }
254 }
255}