blob: 1a98f50db05c8ec462f979889c6adab4617dea02 [file] [log] [blame]
Richard Felkl26cae4d2017-12-19 00:19:16 +01001/**
2 *
3 * Update Salt environment pipeline
4 *
5 * Expected parameters:
azvyagintsev6dac3802018-11-09 15:11:30 +02006 * TARGET_MCP_VERSION Version of MCP to upgrade to
Denis Egorenkod13ad132018-12-17 16:56:02 +04007 * GIT_REFSPEC Git repo ref to be used
8 * DRIVE_TRAIN_PARAMS Yaml, DriveTrain releated params:
9 * SALT_MASTER_URL Salt API server location
10 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
Denis Egorenko70002bc2019-08-19 19:41:27 +040011 * BATCH_SIZE Use batch sizing during upgrade for large envs
Denis Egorenkod13ad132018-12-17 16:56:02 +040012 * UPGRADE_SALTSTACK Upgrade SaltStack packages to new version.
13 * UPDATE_CLUSTER_MODEL Update MCP version parameter in cluster model
14 * UPDATE_PIPELINES Update pipeline repositories on Gerrit
15 * UPDATE_LOCAL_REPOS Update local repositories
Richard Felkl26cae4d2017-12-19 00:19:16 +010016 */
17
Richard Felkl5f7fdaf2018-02-15 15:38:31 +010018salt = new com.mirantis.mk.Salt()
19common = new com.mirantis.mk.Common()
20python = new com.mirantis.mk.Python()
Richard Felkl970e0082018-06-12 18:00:51 +020021jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
azvyagintsevf3bf77a2018-11-15 19:30:04 +020022def pipelineTimeout = 12
Richard Felkl5f7fdaf2018-02-15 15:38:31 +010023venvPepper = "venvPepper"
Richard Felkl0e80d892018-06-20 13:44:54 +020024workspace = ""
Ivan Berezovskiy24b0be02019-12-11 15:31:10 +040025def saltMastURL = ''
26def saltMastCreds = ''
Denis Egorenkobf390102020-01-24 19:47:05 +040027def packageUpgradeMode = ''
Richard Felkl5f7fdaf2018-02-15 15:38:31 +010028
Denis Egorenkoada83882019-06-11 14:03:40 +040029def triggerMirrorJob(String jobName, String reclassSystemBranch) {
Richard Felkl970e0082018-06-12 18:00:51 +020030 params = jenkinsUtils.getJobParameters(jobName)
Denis Egorenkoada83882019-06-11 14:03:40 +040031 try {
32 build job: jobName, parameters: [
33 [$class: 'StringParameterValue', name: 'BRANCHES', value: params.get('BRANCHES')],
34 [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get('CREDENTIALS_ID')],
35 [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get('SOURCE_URL')],
36 [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get('TARGET_URL')]
37 ]
38 } catch (Exception updateErr) {
39 common.warningMsg(updateErr)
40 common.warningMsg('Attempt to update git repo in failsafe manner')
41 build job: jobName, parameters: [
42 [$class: 'StringParameterValue', name: 'BRANCHES', value: reclassSystemBranch.replace('origin/', '')],
43 [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get('CREDENTIALS_ID')],
44 [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get('SOURCE_URL')],
45 [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get('TARGET_URL')]
46 ]
47 }
Richard Felkl970e0082018-06-12 18:00:51 +020048}
49
azvyagintsev6dac3802018-11-09 15:11:30 +020050def updateSaltStack(target, pkgs) {
Denis Egorenko0d0c65f2019-02-26 16:05:03 +040051 salt.cmdRun(venvPepper, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
52 // can't use same function from pipeline lib, as at the moment of running upgrade pipeline Jenkins
53 // still using pipeline lib from current old mcp-version
azvyagintsev6dac3802018-11-09 15:11:30 +020054 common.retry(20, 60) {
Richard Felkl0e80d892018-06-20 13:44:54 +020055 salt.minionsReachable(venvPepper, 'I@salt:master', '*')
56 def running = salt.runSaltProcessStep(venvPepper, target, 'saltutil.running', [], null, true, 5)
azvyagintsev6dac3802018-11-09 15:11:30 +020057 for (value in running.get("return")[0].values()) {
58 if (value != []) {
Richard Felkl0e80d892018-06-20 13:44:54 +020059 throw new Exception("Not all salt-minions are ready for execution")
60 }
61 }
62 }
63
azvyagintsevf3bf77a2018-11-15 19:30:04 +020064 def saltVersion = salt.getPillar(venvPepper, 'I@salt:master', '_param:salt_version').get('return')[0].values()[0]
Richard Felklf23257e2018-08-06 09:24:23 +020065 def saltMinionVersions = salt.cmdRun(venvPepper, target, "apt-cache policy salt-common | awk '/Installed/ && /$saltVersion/'").get("return")
Richard Felkl0e80d892018-06-20 13:44:54 +020066 def saltMinionVersion = ""
67
azvyagintsev6dac3802018-11-09 15:11:30 +020068 for (minion in saltMinionVersions[0].keySet()) {
69 saltMinionVersion = saltMinionVersions[0].get(minion).replace("Salt command execution success", "").trim()
70 if (saltMinionVersion == "") {
Richard Felkl0e80d892018-06-20 13:44:54 +020071 error("Installed version of Salt on $minion doesn't match specified version in the model.")
72 }
73 }
74}
75
Denis Egorenko6f1c8732019-09-24 14:55:50 +040076def getWorkerThreads(saltId) {
77 if (env.getEnvironment().containsKey('SALT_MASTER_OPT_WORKER_THREADS')) {
78 return env['SALT_MASTER_OPT_WORKER_THREADS'].toString()
79 }
80 def threads = salt.cmdRun(saltId, "I@salt:master", "cat /etc/salt/master.d/master.conf | grep worker_threads | cut -f 2 -d ':'", true, null, true)
81 return threads['return'][0].values()[0].replaceAll('Salt command execution success','').trim()
82}
83
Ivan Berezovskiy304705c2019-12-03 18:49:09 +040084def wa29352(String cname) {
Denis Egorenkoada83882019-06-11 14:03:40 +040085 // WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/12/openssh/client/root.yml
86 // Default soft-param has been removed, what now makes not possible to render some old env's.
87 // Like fix, we found copy-paste already generated key from backups, to secrets.yml with correct key name
88 def wa29352ClassName = 'cluster.' + cname + '.infra.secrets_root_wa29352'
89 def wa29352File = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets_root_wa29352.yml"
90 def wa29352SecretsFile = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets.yml"
91 def _tempFile = '/tmp/wa29352_' + UUID.randomUUID().toString().take(8)
92 try {
93 salt.cmdRun(venvPepper, 'I@salt:master', "grep -qiv root_private_key ${wa29352SecretsFile}", true, null, false)
94 salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29352File}", true, null, false)
95 }
96 catch (Exception ex) {
Denis Egorenkoe0c88432019-10-07 12:46:14 +040097 common.infoMsg('Work-around for PROD-29352 already applied, nothing todo')
Denis Egorenkoada83882019-06-11 14:03:40 +040098 return
99 }
100 def rKeysDict = [
101 'parameters': [
102 '_param': [
103 'root_private_key': salt.getPillar(venvPepper, 'I@salt:master', '_param:root_private_key').get('return')[0].values()[0].trim(),
104 'root_public_key' : '',
105 ]
106 ]
107 ]
108 // save root key,and generate public one from it
109 writeFile file: _tempFile, text: rKeysDict['parameters']['_param']['root_private_key'].toString().trim()
110 sh('chmod 0600 ' + _tempFile)
111 rKeysDict['parameters']['_param']['root_public_key'] = sh(script: "ssh-keygen -q -y -f ${_tempFile}", returnStdout: true).trim()
112 sh('rm -fv ' + _tempFile)
113 writeYaml file: _tempFile, data: rKeysDict
114 def yamlData = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
115 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${yamlData}' | base64 -d > ${wa29352File}", false, null, false)
116 common.infoMsg("Add $wa29352ClassName class into secrets.yml")
117
118 // Add 'classes:' directive
119 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
120 "grep -q 'classes:' infra/secrets.yml || sed -i '1iclasses:' infra/secrets.yml")
121
122 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
123 "grep -q '${wa29352ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29352ClassName' infra/secrets.yml")
124 salt.fullRefresh(venvPepper, '*')
125 sh('rm -fv ' + _tempFile)
Denis Egorenkoe0c88432019-10-07 12:46:14 +0400126 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && git status && " +
127 "git add ${wa29352File} && git add -u && git commit --allow-empty -m 'Cluster model updated with WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/ at ${common.getDatetime()}' ")
128 common.infoMsg('Work-around for PROD-29352 successfully applied')
Denis Egorenkoada83882019-06-11 14:03:40 +0400129}
130
131def wa29155(ArrayList saltMinions, String cname) {
132 // WA for PROD-29155. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/
133 // CHeck for existence cmp nodes, and try to render it. Is failed, apply ssh-key wa
134 def ret = ''
135 def patched = false
136 def wa29155ClassName = 'cluster.' + cname + '.infra.secrets_nova_wa29155'
137 def wa29155File = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets_nova_wa29155.yml"
138
139 try {
140 salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29155File}", true, null, false)
141 }
142 catch (Exception ex) {
143 common.infoMsg('Work-around for PROD-29155 already apply, nothing todo')
144 return
145 }
Denis Egorenkoe983d452019-08-23 14:29:34 +0400146 salt.fullRefresh(venvPepper, 'I@salt:master')
Denis Egorenkoada83882019-06-11 14:03:40 +0400147 for (String minion in saltMinions) {
Denis Egorenkoada83882019-06-11 14:03:40 +0400148 // First attempt, second will be performed in next validateReclassModel() stages
149 try {
150 salt.cmdRun(venvPepper, 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
151 } catch (Exception e) {
152 common.errorMsg(e.toString())
153 if (patched) {
154 error("Node: ${minion} failed to render after reclass-system upgrade!WA29155 probably didn't help.")
155 }
156 // check, that failed exactly by our case, by key-length check.
157 def missed_key = salt.getPillar(venvPepper, minion, '_param:nova_compute_ssh_private').get("return")[0].values()[0]
158 if (missed_key != '') {
159 error("Node: ${minion} failed to render after reclass-system upgrade!")
160 }
161 common.warningMsg('Perform: Attempt to apply WA for PROD-29155\n' +
162 'See https://gerrit.mcp.mirantis.com/#/c/37932/ for more info')
163 common.warningMsg('WA-PROD-29155 Generating new ssh key at master node')
164 def _tempFile = "/tmp/nova_wa29155_" + UUID.randomUUID().toString().take(8)
165 common.infoMsg('Perform: generation NEW ssh-private key for nova-compute')
166 salt.cmdRun(venvPepper, 'I@salt:master', "ssh-keygen -f ${_tempFile} -N '' -q")
167 def _pub_k = salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'cmd.run', "cat ${_tempFile}.pub").get('return')[0].values()[0].trim()
168 def _priv_k = salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'cmd.run', "cat ${_tempFile}").get('return')[0].values()[0].trim()
169 salt.cmdRun(venvPepper, 'I@salt:master', "rm -fv ${_tempFile}", false, null, false)
170 def novaKeysDict = [
171 "parameters": [
172 "_param": [
173 "nova_compute_ssh_private": _priv_k,
174 "nova_compute_ssh_public" : _pub_k
175 ]
176 ]
177 ]
178 writeYaml file: _tempFile, data: novaKeysDict
179 def yamlData = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
180 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${yamlData}' | base64 -d > ${wa29155File}", false, null, false)
181 common.infoMsg("Add $wa29155ClassName class into secrets.yml")
182
183 // Add 'classes:' directive
184 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
185 "grep -q 'classes:' infra/secrets.yml || sed -i '1iclasses:' infra/secrets.yml")
186
187 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
188 "grep -q '${wa29155ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29155ClassName' infra/secrets.yml")
Ivan Berezovskiy304705c2019-12-03 18:49:09 +0400189 salt.fullRefresh(venvPepper, 'I@salt:master')
190 salt.fullRefresh(venvPepper, saltMinions)
Denis Egorenkoada83882019-06-11 14:03:40 +0400191 patched = true
192 }
193 }
194 if (patched) {
195 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && git status && " +
196 "git add ${wa29155File} && git add -u && git commit --allow-empty -m 'Cluster model updated with WA for PROD-29155. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/ at ${common.getDatetime()}' ")
197 common.infoMsg('Work-around for PROD-29155 successfully applied')
198 }
199
200}
201
Denis Egorenkobae9dc72019-07-29 17:44:46 +0400202def wa32284(String clusterName) {
203 def clientGluster = salt.getPillar(venvPepper, 'I@salt:master', "glusterfs:client:enabled").get("return")[0].values()[0]
204 def pkiGluster = salt.getPillar(venvPepper, 'I@salt:master', "glusterfs:client:volumes:salt_pki").get("return")[0].values()[0]
205 def nginxEnabledAtMaster = salt.getPillar(venvPepper, 'I@salt:master', 'nginx:server:enabled').get('return')[0].values()[0]
206 if (nginxEnabledAtMaster.toString().toLowerCase() == 'true' && clientGluster.toString().toLowerCase() == 'true' && pkiGluster) {
207 def nginxRequires = salt.getPillar(venvPepper, 'I@salt:master', 'nginx:server:wait_for_service').get('return')[0].values()[0]
208 if (nginxRequires.isEmpty()) {
209 def nginxRequiresClassName = "cluster.${clusterName}.infra.config.nginx_requires_wa32284"
210 def nginxRequiresClassFile = "/srv/salt/reclass/classes/cluster/${clusterName}/infra/config/nginx_requires_wa32284.yml"
211 def nginxRequiresBlock = ['parameters': ['nginx': ['server': ['wait_for_service': ['srv-salt-pki.mount'] ] ] ] ]
212 def _tempFile = '/tmp/wa32284_' + UUID.randomUUID().toString().take(8)
213 writeYaml file: _tempFile , data: nginxRequiresBlock
214 def nginxRequiresBlockString = sh(script: "cat ${_tempFile}", returnStdout: true).trim()
215 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${clusterName} && " +
216 "sed -i '/^parameters:/i - ${nginxRequiresClassName}' infra/config/init.yml")
217 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${nginxRequiresBlockString}' > ${nginxRequiresClassFile}", false, null, false)
218 }
219 }
220}
221
Denis Egorenkoda117892019-09-02 14:53:02 +0400222def wa32182(String cluster_name) {
Hanna Arhipova644aefd2019-09-06 17:55:40 +0300223 if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
Denis Egorenkoda117892019-09-02 14:53:02 +0400224 def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
225 def fixFile = "${clusterModelPath}/opencontrail/common_wa32182.yml"
226 def usualFile = "${clusterModelPath}/opencontrail/common.yml"
227 def fixFileContent = "classes:\n- system.opencontrail.common\n"
228 salt.cmdRun(venvPepper, 'I@salt:master', "test -f ${fixFile} -o -f ${usualFile} || echo '${fixFileContent}' > ${fixFile}")
229 def contrailFiles = ['opencontrail/analytics.yml', 'opencontrail/control.yml', 'openstack/compute/init.yml']
230 if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
231 contrailFiles.add('kubernetes/compute.yml')
232 }
233 for(String contrailFile in contrailFiles) {
234 contrailFile = "${clusterModelPath}/${contrailFile}"
235 def containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- cluster\\.${cluster_name}\\.opencontrail\\.common(_wa32182)?\$' ${contrailFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
236 if (containsFix) {
237 continue
238 } else {
239 salt.cmdRun(venvPepper, 'I@salt:master', "grep -q -E '^parameters:' ${contrailFile} && sed -i '/^parameters:/i - cluster.${cluster_name}.opencontrail.common_wa32182' ${contrailFile} || " +
240 "echo '- cluster.${cluster_name}.opencontrail.common_wa32182' >> ${contrailFile}")
241 }
242 }
243 }
244}
245
Denis Egorenko32461082019-10-25 13:13:39 +0400246def wa33867(String cluster_name) {
247 if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
248 def contrailControlFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/opencontrail/control.yml"
249 def line = salt.cmdRun(venvPepper, 'I@salt:master', "awk '/^- cluster.${cluster_name}.infra.backup.client_zookeeper/ {getline; print \$0}' ${contrailControlFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
250 if (line == "- cluster.${cluster_name}.infra") {
251 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^- cluster.${cluster_name}.infra\$/d' ${contrailControlFile}")
252 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^- cluster.${cluster_name}.infra.backup.client_zookeeper\$/i - cluster.${cluster_name}.infra' ${contrailControlFile}")
253 }
254 }
255}
256
Denis Egorenko7e934c22019-10-18 17:06:11 +0400257def wa33771(String cluster_name) {
258 def octaviaEnabled = salt.getMinions(venvPepper, 'I@octavia:api:enabled')
259 def octaviaWSGI = salt.getMinions(venvPepper, 'I@apache:server:site:octavia_api')
260 if (octaviaEnabled && ! octaviaWSGI) {
261 def openstackControl = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/control.yml"
262 def octaviaFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/octavia_wa33771.yml"
263 def octaviaContext = [
264 'classes': [ 'system.apache.server.site.octavia' ],
265 'parameters': [
266 '_param': [ 'apache_octavia_api_address' : '${_param:cluster_local_address}' ],
267 'apache': [ 'server': [ 'site': [ 'apache_proxy_openstack_api_octavia': [ 'enabled': false ] ] ] ]
268 ]
269 ]
270 def _tempFile = '/tmp/wa33771' + UUID.randomUUID().toString().take(8)
271 writeYaml file: _tempFile , data: octaviaContext
272 def octaviaFileContent = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
273 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^parameters:/i - cluster.${cluster_name}.openstack.octavia_wa33771' ${openstackControl}")
274 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${octaviaFileContent}' | base64 -d > ${octaviaFile}", false, null, false)
275 }
276}
277
Denis Egorenko4ac89a32019-11-14 19:56:17 +0400278def wa33930_33931(String cluster_name) {
279 def openstackControlFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/control.yml"
280 def fixName = 'clients_common_wa33930_33931'
281 def fixFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/${fixName}.yml"
282 def containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- cluster\\.${cluster_name}\\.openstack\\.${fixName}\$' ${openstackControlFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
283 if (! containsFix) {
284 def fixContext = [
285 'classes': [ 'service.nova.client', 'service.glance.client', 'service.neutron.client' ]
286 ]
287 if (salt.getMinions(venvPepper, 'I@manila:api:enabled')) {
288 fixContext['classes'] << 'service.manila.client'
289 }
290 if (salt.getMinions(venvPepper, 'I@ironic:api:enabled')) {
291 fixContext['classes'] << 'service.ironic.client'
292 }
293 if (salt.getMinions(venvPepper, 'I@gnocchi:server:enabled')) {
294 fixContext['classes'] << 'service.gnocchi.client'
295 }
296 if (salt.getMinions(venvPepper, 'I@barbican:server:enabled')) {
297 fixContext['classes'] << 'service.barbican.client.single'
298 }
299 def _tempFile = '/tmp/wa33930_33931' + UUID.randomUUID().toString().take(8)
300 writeYaml file: _tempFile , data: fixContext
301 def fixFileContent = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
302 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${fixFileContent}' | base64 -d > ${fixFile}", false, null, false)
303 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^parameters:/i - cluster.${cluster_name}.openstack.${fixName}' ${openstackControlFile}")
304 }
305}
306
azvyagintsev6dac3802018-11-09 15:11:30 +0200307def archiveReclassInventory(filename) {
Denis Egorenkoada83882019-06-11 14:03:40 +0400308 def _tmp_file = '/tmp/' + filename + UUID.randomUUID().toString().take(8)
309 // jenkins may fail at overheap. Compress data with gzip like WA
310 def ret = salt.cmdRun(venvPepper, 'I@salt:master', 'reclass -i 2>/dev/null | gzip -9 -c | base64', true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
311 def _tmp = sh(script: "echo '$ret' > ${_tmp_file}", returnStdout: false)
312 sh(script: "cat ${_tmp_file} | base64 -d | gzip -d > $filename", returnStdout: false)
313 archiveArtifacts artifacts: filename
314 sh(script: "rm -v ${_tmp_file}|| true")
Richard Felkl0e80d892018-06-20 13:44:54 +0200315}
316
Denis Egorenkod13ad132018-12-17 16:56:02 +0400317def validateReclassModel(ArrayList saltMinions, String suffix) {
318 try {
Denis Egorenkoada83882019-06-11 14:03:40 +0400319 for (String minion in saltMinions) {
320 common.infoMsg("Reclass model validation for minion ${minion}...")
321 def reclassInv = salt.cmdRun(venvPepper, 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
322 writeFile file: "inventory-${minion}-${suffix}.out", text: reclassInv.toString()
Denis Egorenkod13ad132018-12-17 16:56:02 +0400323 }
324 } catch (Exception e) {
325 common.errorMsg('Can not validate current Reclass model. Inspect failed minion manually.')
Denis Egorenkoada83882019-06-11 14:03:40 +0400326 error(e.toString())
Denis Egorenkod13ad132018-12-17 16:56:02 +0400327 }
328}
329
Denis Egorenkoada83882019-06-11 14:03:40 +0400330def archiveReclassModelChanges(ArrayList saltMinions, String oldSuffix = 'before', String newSuffix = 'after') {
331 for (String minion in saltMinions) {
332 def fileName = "reclass-model-${minion}-diff.out"
333 sh "diff -u inventory-${minion}-${oldSuffix}.out inventory-${minion}-${newSuffix}.out > ${fileName} || true"
334 archiveArtifacts artifacts: "${fileName}"
Denis Egorenkod13ad132018-12-17 16:56:02 +0400335 }
336}
337
Denis Egorenkobf390102020-01-24 19:47:05 +0400338def checkCICDDocker() {
339 common.infoMsg('Perform: Checking if Docker containers are up')
340 try {
341 common.retry(10, 30) {
342 salt.cmdRun(venvPepper, 'I@jenkins:client and I@docker:client', "! docker service ls | tail -n +2 | grep -v -E '\\s([0-9])/\\1\\s'")
343 }
344 }
345 catch (Exception ex) {
346 error("Docker containers for CI/CD services are having troubles with starting.")
347 }
348}
349
Denis Egorenko5a8a7e92018-12-27 14:13:48 +0400350if (common.validInputParam('PIPELINE_TIMEOUT')) {
351 try {
352 pipelineTimeout = env.PIPELINE_TIMEOUT.toInteger()
Denis Egorenkoada83882019-06-11 14:03:40 +0400353 } catch (Exception e) {
Denis Egorenko5a8a7e92018-12-27 14:13:48 +0400354 common.warningMsg("Provided PIPELINE_TIMEOUT parameter has invalid value: ${env.PIPELINE_TIMEOUT} - should be interger")
355 }
Martin Polreich965c9032018-08-09 16:33:57 +0200356}
357
358timeout(time: pipelineTimeout, unit: 'HOURS') {
Denis Egorenkoada83882019-06-11 14:03:40 +0400359 node("python") {
Jakub Josefa63f9862018-01-11 17:58:38 +0100360 try {
Denis Egorenkoada83882019-06-11 14:03:40 +0400361 def inventoryBeforeFilename = "reclass-inventory-before.out"
362 def inventoryAfterFilename = "reclass-inventory-after.out"
Richard Felkl0e80d892018-06-20 13:44:54 +0200363 workspace = common.getWorkspace()
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200364 targetMcpVersion = null
365 if (!common.validInputParam('TARGET_MCP_VERSION') && !common.validInputParam('MCP_VERSION')) {
366 error('You must specify MCP version in TARGET_MCP_VERSION|MCP_VERSION variable')
azvyagintsev6dac3802018-11-09 15:11:30 +0200367 }
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200368 // bw comp. for 2018.X => 2018.11 release
Denis Egorenkoada83882019-06-11 14:03:40 +0400369 if (common.validInputParam('MCP_VERSION')) {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200370 targetMcpVersion = env.MCP_VERSION
371 common.warningMsg("targetMcpVersion has been changed to:${targetMcpVersion}, which was taken from deprecated pipeline viriable:MCP_VERSION")
Denis Egorenkoada83882019-06-11 14:03:40 +0400372 } else {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200373 targetMcpVersion = env.TARGET_MCP_VERSION
374 }
375 // end bw comp. for 2018.X => 2018.11 release
Denis Egorenkod13ad132018-12-17 16:56:02 +0400376 def gitTargetMcpVersion = env.getProperty('GIT_REFSPEC')
Ivan Berezovskiyd62629c2019-01-16 19:26:18 +0400377 if (targetMcpVersion in ['nightly', 'testing']) {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200378 gitTargetMcpVersion = 'master'
Ivan Berezovskiyd62629c2019-01-16 19:26:18 +0400379 } else if (targetMcpVersion == 'proposed') {
380 gitTargetMcpVersion = 'proposed'
Denis Egorenko284690e2018-12-25 15:55:12 +0400381 } else if (!gitTargetMcpVersion) {
382 // backward compatibility for 2018.11.0
383 gitTargetMcpVersion = "release/${targetMcpVersion}"
azvyagintsev6dac3802018-11-09 15:11:30 +0200384 }
Ivan Berezovskiyd62629c2019-01-16 19:26:18 +0400385 common.warningMsg("gitTargetMcpVersion has been changed to:${gitTargetMcpVersion}")
Denis Egorenko284690e2018-12-25 15:55:12 +0400386 def upgradeSaltStack = ''
387 def updateClusterModel = ''
388 def updatePipelines = ''
389 def updateLocalRepos = ''
390 def reclassSystemBranch = ''
Denis Egorenkob559e542019-03-29 13:24:15 +0400391 def reclassSystemBranchDefault = gitTargetMcpVersion
Denis Egorenko70002bc2019-08-19 19:41:27 +0400392 def batchSize = ''
Ivan Berezovskiy304705c2019-12-03 18:49:09 +0400393 if (gitTargetMcpVersion ==~ /^\d\d\d\d\.\d\d?\.\d+$/) {
394 reclassSystemBranchDefault = "tags/${gitTargetMcpVersion}"
395 } else if (gitTargetMcpVersion != 'proposed') {
Denis Egorenkob559e542019-03-29 13:24:15 +0400396 reclassSystemBranchDefault = "origin/${gitTargetMcpVersion}"
397 }
Hanna Arhipova644aefd2019-09-06 17:55:40 +0300398 def driveTrainParamsYaml = env.getProperty('DRIVE_TRAIN_PARAMS')
399 if (driveTrainParamsYaml) {
400 def driveTrainParams = readYaml text: driveTrainParamsYaml
401 saltMastURL = driveTrainParams.get('SALT_MASTER_URL')
402 saltMastCreds = driveTrainParams.get('SALT_MASTER_CREDENTIALS')
403 upgradeSaltStack = driveTrainParams.get('UPGRADE_SALTSTACK', false).toBoolean()
404 updateClusterModel = driveTrainParams.get('UPDATE_CLUSTER_MODEL', false).toBoolean()
405 updatePipelines = driveTrainParams.get('UPDATE_PIPELINES', false).toBoolean()
406 updateLocalRepos = driveTrainParams.get('UPDATE_LOCAL_REPOS', false).toBoolean()
407 reclassSystemBranch = driveTrainParams.get('RECLASS_SYSTEM_BRANCH', reclassSystemBranchDefault)
Denis Egorenko70002bc2019-08-19 19:41:27 +0400408 batchSize = driveTrainParams.get('BATCH_SIZE', '')
Denis Egorenkobf390102020-01-24 19:47:05 +0400409 if (driveTrainParams.get('OS_DIST_UPGRADE', false).toBoolean() == true) {
410 packageUpgradeMode = 'dist-upgrade'
411 } else if (driveTrainParams.get('OS_UPGRADE', false).toBoolean() == true) {
412 packageUpgradeMode = 'upgrade'
413 }
Denis Egorenko284690e2018-12-25 15:55:12 +0400414 } else {
415 // backward compatibility for 2018.11.0
416 saltMastURL = env.getProperty('SALT_MASTER_URL')
417 saltMastCreds = env.getProperty('SALT_MASTER_CREDENTIALS')
Denis Egorenkoe5f39142018-12-28 12:30:48 +0400418 upgradeSaltStack = env.getProperty('UPGRADE_SALTSTACK').toBoolean()
419 updateClusterModel = env.getProperty('UPDATE_CLUSTER_MODEL').toBoolean()
420 updatePipelines = env.getProperty('UPDATE_PIPELINES').toBoolean()
421 updateLocalRepos = env.getProperty('UPDATE_LOCAL_REPOS').toBoolean()
Denis Egorenkob559e542019-03-29 13:24:15 +0400422 reclassSystemBranch = reclassSystemBranchDefault
Denis Egorenko284690e2018-12-25 15:55:12 +0400423 }
Denis Egorenkod13ad132018-12-17 16:56:02 +0400424 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
Denis Egorenkod13ad132018-12-17 16:56:02 +0400425 def minions = salt.getMinions(venvPepper, '*')
Denis Egorenkoc0538ce2019-04-19 15:05:18 +0400426 def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
427 if (cluster_name == '' || cluster_name == 'null' || cluster_name == null) {
428 error('Pillar data is broken for Salt master node! Please check it manually and re-run pipeline.')
429 }
Hanna Arhipova644aefd2019-09-06 17:55:40 +0300430 if (!batchSize) {
Denis Egorenko9ef25b92019-12-19 16:45:25 +0400431 // if no batch size provided get current worker threads and set batch size to 2/3 of it to avoid
432 // 'SaltReqTimeoutError: Message timed out' issue on Salt targets for large amount of nodes
433 // do not use toDouble/Double as it requires additional approved method
434 def workerThreads = getWorkerThreads(venvPepper).toInteger()
435 batch_size = (workerThreads * 2 / 3).toString().tokenize('.')[0]
Denis Egorenko70002bc2019-08-19 19:41:27 +0400436 }
Ivan Berezovskiy304705c2019-12-03 18:49:09 +0400437 def computeMinions = salt.getMinions(venvPepper, 'I@nova:compute')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400438
Denis Egorenkoada83882019-06-11 14:03:40 +0400439 stage('Update Reclass and Salt-Formulas') {
440 common.infoMsg('Perform: Full salt sync')
441 salt.fullRefresh(venvPepper, '*')
442 common.infoMsg('Perform: Validate reclass medata before processing')
443 validateReclassModel(minions, 'before')
444
445 common.infoMsg('Perform: archiveReclassInventory before upgrade')
Denis Egorenko89a3c572019-03-22 17:03:40 +0400446 archiveReclassInventory(inventoryBeforeFilename)
Denis Egorenkod13ad132018-12-17 16:56:02 +0400447
azvyagintsev6dac3802018-11-09 15:11:30 +0200448 try {
Denis Egorenkoada83882019-06-11 14:03:40 +0400449 salt.cmdRun(venvPepper, 'I@salt:master', 'cd /srv/salt/reclass/ && git status && git diff-index --quiet HEAD --')
Richard Felkl970e0082018-06-12 18:00:51 +0200450 }
azvyagintsev6dac3802018-11-09 15:11:30 +0200451 catch (Exception ex) {
Denis Egorenkoada83882019-06-11 14:03:40 +0400452 error('You have uncommitted changes in your Reclass cluster model repository. Please commit or reset them and rerun the pipeline.')
Richard Felkl970e0082018-06-12 18:00:51 +0200453 }
Denis Egorenkod13ad132018-12-17 16:56:02 +0400454 if (updateClusterModel) {
azvyagintsev6dac3802018-11-09 15:11:30 +0200455 common.infoMsg('Perform: UPDATE_CLUSTER_MODEL')
Sergey382581a2018-10-25 12:22:26 +0400456 def dateTime = common.getDatetime()
Ivan Berezovskiy1d780522020-01-17 13:55:18 +0400457 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git fetch")
Sergey382581a2018-10-25 12:22:26 +0400458 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenko928b3e82019-01-15 11:47:16 +0400459 "grep -r --exclude-dir=aptly -l 'mcp_version: .*' * | xargs --no-run-if-empty sed -i 's|mcp_version: .*|mcp_version: \"$targetMcpVersion\"|g'")
azvyagintsev6dac3802018-11-09 15:11:30 +0200460 // Do the same, for deprecated variable-duplicate
461 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenko928b3e82019-01-15 11:47:16 +0400462 "grep -r --exclude-dir=aptly -l 'apt_mk_version: .*' * | xargs --no-run-if-empty sed -i 's|apt_mk_version: .*|apt_mk_version: \"$targetMcpVersion\"|g'")
Denis Egorenko284690e2018-12-25 15:55:12 +0400463 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenko928b3e82019-01-15 11:47:16 +0400464 "grep -r --exclude-dir=aptly -l 'jenkins_pipelines_branch: .*' * | xargs --no-run-if-empty sed -i 's|jenkins_pipelines_branch: .*|jenkins_pipelines_branch: \"$gitTargetMcpVersion\"|g'")
Victor Ryzhenkin202c5062019-01-15 19:58:03 +0400465 // Set new k8s param
466 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
467 "grep -r --exclude-dir=aptly -l 'kubernetes_containerd_enabled: .*' * | xargs --no-run-if-empty sed -i 's|kubernetes_containerd_enabled: .*|kubernetes_containerd_enabled: True|g'")
Denis Egorenko2a39d2c2019-01-16 19:18:31 +0400468 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
469 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.salt' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.salt/system.linux.system.repo.mcp.apt_mirantis.salt-formulas/g'")
470 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenkoca871c22019-02-13 13:49:13 +0400471 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.contrail' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.contrail/system.linux.system.repo.mcp.apt_mirantis.contrail/g'")
Denis Egorenko2a39d2c2019-01-16 19:18:31 +0400472 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenkoca871c22019-02-13 13:49:13 +0400473 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.updates/system.linux.system.repo.mcp.apt_mirantis.update/g'")
Anton Samoylov72c2ccf2019-02-13 15:57:13 +0400474 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenkoada83882019-06-11 14:03:40 +0400475 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
Denis Egorenko6ca9f0d2019-07-10 19:47:39 +0400476
Denis Egorenko70f4f9a2019-09-16 16:41:26 +0400477 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/linux_system_repo_mcp_maas_url/d' maas.yml")
478 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/maas_region_main_archive/d' maas.yml")
479
Denis Egorenko6ca9f0d2019-07-10 19:47:39 +0400480 // Switch Jenkins/Gerrit to use LDAP SSL/TLS
481 def gerritldapURI = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
482 "grep -r --exclude-dir=aptly 'gerrit_ldap_server: .*' * | grep -Po 'gerrit_ldap_server: \\K.*' | tr -d '\"'", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
483 if (gerritldapURI.startsWith('ldap://')) {
484 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
485 "grep -r --exclude-dir=aptly -l 'gerrit_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|ldap://|ldaps://|g'")
486 } else if (! gerritldapURI.startsWith('ldaps://')) {
487 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
488 "grep -r --exclude-dir=aptly -l 'gerrit_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|gerrit_ldap_server: .*|gerrit_ldap_server: \"ldaps://${gerritldapURI}\"|g'")
489 }
490 def jenkinsldapURI = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
491 "grep -r --exclude-dir=aptly 'jenkins_security_ldap_server: .*' * | grep -Po 'jenkins_security_ldap_server: \\K.*' | tr -d '\"'", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
492 if (jenkinsldapURI.startsWith('ldap://')) {
493 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
494 "grep -r --exclude-dir=aptly -l 'jenkins_security_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|ldap://|ldaps://|g'")
495 } else if (! jenkinsldapURI.startsWith('ldaps://')) {
496 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
497 "grep -r --exclude-dir=aptly -l 'jenkins_security_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|jenkins_security_ldap_server: .*|jenkins_security_ldap_server: \"ldaps://${jenkinsldapURI}\"|g'")
498 }
499
Denis Egorenkobae9dc72019-07-29 17:44:46 +0400500 wa32284(cluster_name)
501
Denis Egorenkod13ad132018-12-17 16:56:02 +0400502 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout ${reclassSystemBranch}")
Sergey Kolekonov6ab15112019-01-23 16:47:43 +0400503 // Add kubernetes-extra repo
504 if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
Sergey Kolekonovac9554c2019-04-11 16:33:54 +0400505 // docker-engine conflicts with the recent containerd versions, so it's removed during upgrade. Thus update source engine
506 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
507 "grep -r -l 'engine: docker_hybrid' kubernetes | xargs --no-run-if-empty sed -i 's/engine: docker_hybrid/engine: archive/g'")
Sergey Kolekonov6ab15112019-01-23 16:47:43 +0400508 common.infoMsg("Add kubernetes-extra repo")
509 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
510 "grep -q system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra' kubernetes/common.yml")
511 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
512 "grep -q system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra' kubernetes/common.yml")
513 }
Ivan Berezovskiybe630ee2019-04-15 13:23:58 +0400514 // Add all update repositories
515 def repoIncludeBase = '- system.linux.system.repo.mcp.apt_mirantis.'
Denis Egorenko70f4f9a2019-09-16 16:41:26 +0400516 def updateRepoList = ['cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'maas', 'percona', 'salt-formulas', 'saltstack', 'ubuntu']
Ivan Berezovskiybe630ee2019-04-15 13:23:58 +0400517 updateRepoList.each { repo ->
518 def repoNameUpdateInclude = "${repoIncludeBase}update.${repo}"
519 def filesWithInclude = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -Plr '\\${repoIncludeBase}${repo}\$' . || true", false).get('return')[0].values()[0].trim().tokenize('\n')
520 filesWithInclude.each { file ->
521 def updateRepoIncludeExist = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -P '\\${repoNameUpdateInclude}\$' ${file} || echo not_found", false, null, true).get('return')[0].values()[0].trim()
522 if (updateRepoIncludeExist == 'not_found') {
523 // Include needs to be added
524 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenkoada83882019-06-11 14:03:40 +0400525 "sed -i 's/\\( *\\)${repoIncludeBase}${repo}\$/&\\n\\1${repoNameUpdateInclude}/g' ${file}")
Ivan Berezovskiybe630ee2019-04-15 13:23:58 +0400526 common.infoMsg("Update repo for ${repo} is added to ${file}")
527 }
528 }
529 }
Denis Egorenkoda117892019-09-02 14:53:02 +0400530 wa32182(cluster_name)
Denis Egorenko7e934c22019-10-18 17:06:11 +0400531 wa33771(cluster_name)
Denis Egorenko32461082019-10-25 13:13:39 +0400532 wa33867(cluster_name)
Denis Egorenko4ac89a32019-11-14 19:56:17 +0400533 wa33930_33931(cluster_name)
Sergey382581a2018-10-25 12:22:26 +0400534 // Add new defaults
535 common.infoMsg("Add new defaults")
Denis Egorenkod13ad132018-12-17 16:56:02 +0400536 salt.cmdRun(venvPepper, 'I@salt:master', "grep '^ mcp_version: ' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +
Denis Egorenko928b3e82019-01-15 11:47:16 +0400537 "sed -i 's|^ _param:| _param:\\n mcp_version: \"$targetMcpVersion\"|' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml")
Sergey382581a2018-10-25 12:22:26 +0400538 salt.cmdRun(venvPepper, 'I@salt:master', "grep '^- system.defaults\$' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +
Denis Egorenko928b3e82019-01-15 11:47:16 +0400539 "sed -i 's|^classes:|classes:\\n- system.defaults|' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml")
Ivan Berezovskiyadb7d832019-04-23 14:32:02 +0400540 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
541 "grep -r -l 'docker_image_jenkins: .*' cicd | xargs --no-run-if-empty sed -i 's|\\s*docker_image_jenkins: .*||g'")
542 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
543 "grep -r -l 'docker_image_jenkins_slave: .*' cicd | xargs --no-run-if-empty sed -i 's|\\s*docker_image_jenkins_slave: .*||g'")
Sergey382581a2018-10-25 12:22:26 +0400544 common.infoMsg("The following changes were made to the cluster model and will be commited. " +
azvyagintsev6dac3802018-11-09 15:11:30 +0200545 "Please consider if you want to push them to the remote repository or not. You have to do this manually when the run is finished.")
Sergey382581a2018-10-25 12:22:26 +0400546 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && git diff")
547 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && git status && " +
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200548 "git add -u && git commit --allow-empty -m 'Cluster model update to the release $targetMcpVersion on $dateTime'")
Sergey382581a2018-10-25 12:22:26 +0400549 }
Denis Egorenkod13ad132018-12-17 16:56:02 +0400550 try {
551 common.infoMsg('Perform: UPDATE Salt Formulas')
Denis Egorenkoada83882019-06-11 14:03:40 +0400552 salt.fullRefresh(venvPepper, '*')
553 salt.enforceState(venvPepper, 'I@salt:master', 'linux.system.repo')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400554 def saltEnv = salt.getPillar(venvPepper, 'I@salt:master', "_param:salt_master_base_environment").get("return")[0].values()[0]
Denis Egorenkoada83882019-06-11 14:03:40 +0400555 salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'state.sls_id', ["salt_master_${saltEnv}_pkg_formulas", 'salt.master.env'])
556 salt.fullRefresh(venvPepper, '*')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400557 } catch (Exception updateErr) {
558 common.warningMsg(updateErr)
559 common.warningMsg('Failed to update Salt Formulas repos/packages. Check current available documentation on https://docs.mirantis.com/mcp/latest/, how to update packages.')
560 input message: 'Continue anyway?'
561 }
562
Ivan Berezovskiy304705c2019-12-03 18:49:09 +0400563 wa29352(cluster_name)
Denis Egorenkoe983d452019-08-23 14:29:34 +0400564 wa29155(computeMinions, cluster_name)
Denis Egorenkoada83882019-06-11 14:03:40 +0400565
Denis Egorenkod13ad132018-12-17 16:56:02 +0400566 try {
567 common.infoMsg('Perform: UPDATE Reclass package')
568 salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'pkg.install', ["reclass"])
569 } catch (Exception updateErr) {
570 common.warningMsg(updateErr)
571 common.warningMsg('Failed to update Reclass package. Check current available documentation on https://docs.mirantis.com/mcp/latest/, how to update packages.')
572 input message: 'Continue anyway?'
573 }
574
575 salt.fullRefresh(venvPepper, 'I@salt:master')
Sergey1b81c062018-10-16 17:10:21 +0400576 salt.enforceState(venvPepper, 'I@salt:master', 'reclass.storage', true)
Ivan Berezovskiy304705c2019-12-03 18:49:09 +0400577 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && git status && " +
578 "git add -u && git commit --allow-empty -m 'Reclass nodes update to the release ${targetMcpVersion} on ${common.getDatetime()}'")
Denis Egorenkod13ad132018-12-17 16:56:02 +0400579 try {
580 salt.enforceState(venvPepper, "I@salt:master", 'reclass', true)
581 }
582 catch (Exception ex) {
583 error("Reclass fails rendering. Pay attention to your cluster model.")
584 }
585
586 salt.fullRefresh(venvPepper, '*')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400587 try {
588 salt.cmdRun(venvPepper, 'I@salt:master', "reclass-salt --top")
589 }
590 catch (Exception ex) {
Denis Egorenkoada83882019-06-11 14:03:40 +0400591
592 error('Reclass fails rendering. Pay attention to your cluster model.' +
593 'ErrorMessage:' + ex.toString())
Denis Egorenkod13ad132018-12-17 16:56:02 +0400594 }
595
Denis Egorenkoada83882019-06-11 14:03:40 +0400596 common.infoMsg('Perform: archiveReclassInventory AFTER upgrade')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400597 archiveReclassInventory(inventoryAfterFilename)
598
599 sh "diff -u $inventoryBeforeFilename $inventoryAfterFilename > reclass-inventory-diff.out || true"
600 archiveArtifacts artifacts: "reclass-inventory-diff.out"
601
Denis Egorenkoada83882019-06-11 14:03:40 +0400602 validateReclassModel(minions, 'after')
603 archiveReclassModelChanges(minions)
Jakub Josefa63f9862018-01-11 17:58:38 +0100604 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100605
Denis Egorenkod13ad132018-12-17 16:56:02 +0400606 if (updateLocalRepos) {
azvyagintsev6dac3802018-11-09 15:11:30 +0200607 stage("Update local repos") {
Jakub Josefa63f9862018-01-11 17:58:38 +0100608 common.infoMsg("Updating local repositories")
Sam Stoelingaaab79702018-04-09 18:49:39 -0700609
Sergey1b81c062018-10-16 17:10:21 +0400610 def engine = salt.getPillar(venvPepper, 'I@aptly:publisher', "aptly:publisher:source:engine")
Sam Stoelingaaab79702018-04-09 18:49:39 -0700611 runningOnDocker = engine.get("return")[0].containsValue("docker")
612
613 if (runningOnDocker) {
614 common.infoMsg("Aptly is running as Docker container")
azvyagintsev6dac3802018-11-09 15:11:30 +0200615 } else {
Sam Stoelingaaab79702018-04-09 18:49:39 -0700616 common.infoMsg("Aptly isn't running as Docker container. Going to use aptly user for executing aptly commands")
617 }
618
azvyagintsev6dac3802018-11-09 15:11:30 +0200619 if (runningOnDocker) {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200620 salt.cmdRun(venvPepper, 'I@aptly:publisher', "aptly mirror list --raw | grep -E '*' | xargs --no-run-if-empty -n 1 aptly mirror drop -force", true, null, true)
azvyagintsev6dac3802018-11-09 15:11:30 +0200621 } else {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200622 salt.cmdRun(venvPepper, 'I@aptly:publisher', "aptly mirror list --raw | grep -E '*' | xargs --no-run-if-empty -n 1 aptly mirror drop -force", true, null, true, ['runas=aptly'])
Richard Felkl5f7fdaf2018-02-15 15:38:31 +0100623 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100624
Sergey1b81c062018-10-16 17:10:21 +0400625 salt.enforceState(venvPepper, 'I@aptly:publisher', 'aptly', true)
Richard Felkl26cae4d2017-12-19 00:19:16 +0100626
azvyagintsev6dac3802018-11-09 15:11:30 +0200627 if (runningOnDocker) {
Sergey1b81c062018-10-16 17:10:21 +0400628 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv"], null, true)
629 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-frv -u http://10.99.0.1:8080"], null, true)
azvyagintsev6dac3802018-11-09 15:11:30 +0200630 } else {
Sergey1b81c062018-10-16 17:10:21 +0400631 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv", 'runas=aptly'], null, true)
632 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-afrv", 'runas=aptly'], null, true)
Richard Felkl5f7fdaf2018-02-15 15:38:31 +0100633 }
Richard Felkl79d7df12018-01-05 16:40:11 +0100634
Sergey1b81c062018-10-16 17:10:21 +0400635 salt.enforceState(venvPepper, 'I@aptly:publisher', 'docker.client.registry', true)
Richard Felkl26cae4d2017-12-19 00:19:16 +0100636
Sergey1b81c062018-10-16 17:10:21 +0400637 salt.enforceState(venvPepper, 'I@aptly:publisher', 'debmirror', true)
Richard Felkl26cae4d2017-12-19 00:19:16 +0100638
Sergey1b81c062018-10-16 17:10:21 +0400639 salt.enforceState(venvPepper, 'I@aptly:publisher', 'git.server', true)
Richard Felkl5f7fdaf2018-02-15 15:38:31 +0100640
Sergey1b81c062018-10-16 17:10:21 +0400641 salt.enforceState(venvPepper, 'I@aptly:publisher', 'linux.system.file', true)
Jakub Josefa63f9862018-01-11 17:58:38 +0100642 }
643 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100644
Denis Egorenkoada83882019-06-11 14:03:40 +0400645 stage('Update Drivetrain') {
Denis Egorenkod13ad132018-12-17 16:56:02 +0400646 if (upgradeSaltStack) {
647 updateSaltStack("I@salt:master", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
Richard Felkl0e80d892018-06-20 13:44:54 +0200648
Denis Egorenko70002bc2019-08-19 19:41:27 +0400649 salt.enforceState(venvPepper, "I@linux:system", 'linux.system.repo', true, true, batchSize)
Richard Felkl0e80d892018-06-20 13:44:54 +0200650 updateSaltStack("I@salt:minion and not I@salt:master", '["salt-minion"]')
651 }
652
Denis Egorenkod13ad132018-12-17 16:56:02 +0400653 if (updatePipelines) {
Denis Egorenkoada83882019-06-11 14:03:40 +0400654 common.infoMsg('Perform: UPDATE git repos')
655 triggerMirrorJob('git-mirror-downstream-mk-pipelines', reclassSystemBranch)
656 triggerMirrorJob('git-mirror-downstream-pipeline-library', reclassSystemBranch)
Richard Felkl970e0082018-06-12 18:00:51 +0200657 }
658
Denis Egorenkod00c17e2019-04-25 20:17:27 +0400659 // update minions certs
Denis Egorenko70002bc2019-08-19 19:41:27 +0400660 salt.enforceState(venvPepper, "I@salt:minion", 'salt.minion.cert', true, true, batchSize)
Denis Egorenkoada83882019-06-11 14:03:40 +0400661
662 // Retry needed only for rare race-condition in user appearance
663 common.infoMsg('Perform: updating users and keys')
Denis Egorenko70002bc2019-08-19 19:41:27 +0400664 salt.enforceState(venvPepper, "I@linux:system", 'linux.system.user', true, true, batchSize)
Denis Egorenkoada83882019-06-11 14:03:40 +0400665 common.infoMsg('Perform: updating openssh')
Denis Egorenko70002bc2019-08-19 19:41:27 +0400666 salt.enforceState(venvPepper, "I@linux:system", 'openssh', true, true, batchSize)
Denis Egorenkoaeaa1de2019-01-15 11:58:12 +0400667
Denis Egorenkoe69920e2019-10-30 13:26:59 +0400668 // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
669 def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
670 def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
Ivan Berezovskiy304705c2019-12-03 18:49:09 +0400671 salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
Denis Egorenkoe69920e2019-10-30 13:26:59 +0400672
Denis Egorenkoada83882019-06-11 14:03:40 +0400673 salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
Ivan Berezovskiy24b0be02019-12-11 15:31:10 +0400674 }
675 }
676 catch (Throwable e) {
677 // If there was an error or exception thrown, the build failed
678 currentBuild.result = "FAILURE"
679 throw e
680 }
681 }
682 // docker.client state may trigger change of jenkins master or jenkins slave services,
683 // so we need wait for slave to reconnect and continue pipeline
684 sleep(180)
Denis Egorenkobf390102020-01-24 19:47:05 +0400685 def cidNodes = []
Ivan Berezovskiy24b0be02019-12-11 15:31:10 +0400686 node('python') {
687 try {
688 stage('Update Drivetrain: Phase 2') {
689 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
Denis Egorenkobf390102020-01-24 19:47:05 +0400690 checkCICDDocker()
Denis Egorenko6ca9f0d2019-07-10 19:47:39 +0400691
Ivan Berezovskiy304705c2019-12-03 18:49:09 +0400692 // Apply changes for HaProxy on CI/CD nodes
693 salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
694
695 salt.enforceState(venvPepper, 'I@jenkins:client and not I@salt:master', 'jenkins.client', true, true, null, false, 60, 2)
Denis Egorenko84a0bc02019-07-23 18:01:21 +0400696
697 // update Nginx proxy settings for Jenkins/Gerrit if needed
698 if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
699 salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
700 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100701 }
Denis Egorenkobf390102020-01-24 19:47:05 +0400702 if (packageUpgradeMode) {
703 cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
704 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100705 }
Jakub Josefa63f9862018-01-11 17:58:38 +0100706 catch (Throwable e) {
707 // If there was an error or exception thrown, the build failed
708 currentBuild.result = "FAILURE"
709 throw e
Richard Felkl26cae4d2017-12-19 00:19:16 +0100710 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100711 }
Denis Egorenkobf390102020-01-24 19:47:05 +0400712
713 stage('Upgrade OS') {
714 if (packageUpgradeMode) {
715 def debian = new com.mirantis.mk.Debian()
716 def statusFile = '/tmp/rebooted_during_upgrade'
717 for(cidNode in cidNodes) {
718 node('python') {
719 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
720 // cmd.run async to prevent connection close in case of slave shutdown, give 5 seconds to handle request response
721 salt.cmdRun(venvPepper, "I@salt:master", "salt -C '${cidNode}' cmd.run 'sleep 5; touch ${statusFile}; salt-call service.stop docker' --async")
722 }
723 sleep(30)
724 node('python') {
725 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
726 debian.osUpgradeNode(venvPepper, cidNode, packageUpgradeMode, false, 60)
727 salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': cidNode, wait: 60, timeout: 10])
728 if (salt.runSaltProcessStep(venvPepper, cidNode, 'file.file_exists', [statusFile], null, true, 5)['return'][0].values()[0].toBoolean()) {
729 salt.cmdRun(venvPepper, "I@salt:master", "salt -C '${cidNode}' cmd.run 'rm ${statusFile} && salt-call service.start docker'") // in case if node was not rebooted
730 sleep(10)
731 }
732 checkCICDDocker()
733 }
734 }
735 } else {
736 common.infoMsg('Upgrade OS skipped...')
737 }
738 }
Hanna Arhipova644aefd2019-09-06 17:55:40 +0300739}