blob: a6e5358ddf347f584690a5adbeedfcea3820b3ca [file] [log] [blame]
Richard Felkl26cae4d2017-12-19 00:19:16 +01001/**
2 *
3 * Update Salt environment pipeline
4 *
5 * Expected parameters:
azvyagintsev6dac3802018-11-09 15:11:30 +02006 * TARGET_MCP_VERSION Version of MCP to upgrade to
Denis Egorenkod13ad132018-12-17 16:56:02 +04007 * GIT_REFSPEC Git repo ref to be used
8 * DRIVE_TRAIN_PARAMS Yaml, DriveTrain releated params:
9 * SALT_MASTER_URL Salt API server location
10 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
Denis Egorenkoca6aeca2019-08-19 19:41:27 +040011 * BATCH_SIZE Use batch sizing during upgrade for large envs
Denis Egorenkod13ad132018-12-17 16:56:02 +040012 * UPGRADE_SALTSTACK Upgrade SaltStack packages to new version.
13 * UPDATE_CLUSTER_MODEL Update MCP version parameter in cluster model
14 * UPDATE_PIPELINES Update pipeline repositories on Gerrit
15 * UPDATE_LOCAL_REPOS Update local repositories
Ivan Berezovskiy22390f62020-02-17 17:08:47 +040016 * OS_UPGRADE Run apt-get upgrade on Drivetrain nodes
17 * OS_DIST_UPGRADE Run apt-get dist-upgrade on Drivetrain nodes and reboot to apply changes
18 * APPLY_MODEL_WORKAROUNDS Whether to apply cluster model workarounds from the pipeline
Richard Felkl26cae4d2017-12-19 00:19:16 +010019 */
20
Richard Felkl5f7fdaf2018-02-15 15:38:31 +010021salt = new com.mirantis.mk.Salt()
22common = new com.mirantis.mk.Common()
23python = new com.mirantis.mk.Python()
Richard Felkl970e0082018-06-12 18:00:51 +020024jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
azvyagintsevf3bf77a2018-11-15 19:30:04 +020025def pipelineTimeout = 12
Richard Felkl5f7fdaf2018-02-15 15:38:31 +010026venvPepper = "venvPepper"
Richard Felkl0e80d892018-06-20 13:44:54 +020027workspace = ""
Ivan Berezovskiya766d432019-12-11 15:31:10 +040028def saltMastURL = ''
29def saltMastCreds = ''
Denis Egorenko96cc57d2020-01-24 19:47:05 +040030def packageUpgradeMode = ''
Ivan Berezovskiy22390f62020-02-17 17:08:47 +040031batchSize = ''
Richard Felkl5f7fdaf2018-02-15 15:38:31 +010032
Ivan Berezovskiyc4dc5012020-03-19 14:03:43 +040033def fullRefreshOneByOne(venvPepper, minions) {
34 for (minion in minions) {
35 salt.runSaltProcessStep(venvPepper, minion, 'saltutil.refresh_pillar', [], null, true, 60)
36 salt.runSaltProcessStep(venvPepper, minion, 'saltutil.refresh_grains', [], null, true, 60)
37 salt.runSaltProcessStep(venvPepper, minion, 'saltutil.sync_all', [], null, true, 180)
38 }
39}
40
azvyagintsevbc359d42019-04-10 19:53:22 +030041def triggerMirrorJob(String jobName, String reclassSystemBranch) {
Richard Felkl970e0082018-06-12 18:00:51 +020042 params = jenkinsUtils.getJobParameters(jobName)
azvyagintsevbc359d42019-04-10 19:53:22 +030043 try {
44 build job: jobName, parameters: [
45 [$class: 'StringParameterValue', name: 'BRANCHES', value: params.get('BRANCHES')],
46 [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get('CREDENTIALS_ID')],
47 [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get('SOURCE_URL')],
48 [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get('TARGET_URL')]
49 ]
50 } catch (Exception updateErr) {
51 common.warningMsg(updateErr)
52 common.warningMsg('Attempt to update git repo in failsafe manner')
53 build job: jobName, parameters: [
54 [$class: 'StringParameterValue', name: 'BRANCHES', value: reclassSystemBranch.replace('origin/', '')],
55 [$class: 'StringParameterValue', name: 'CREDENTIALS_ID', value: params.get('CREDENTIALS_ID')],
56 [$class: 'StringParameterValue', name: 'SOURCE_URL', value: params.get('SOURCE_URL')],
57 [$class: 'StringParameterValue', name: 'TARGET_URL', value: params.get('TARGET_URL')]
58 ]
59 }
Richard Felkl970e0082018-06-12 18:00:51 +020060}
61
azvyagintsev6dac3802018-11-09 15:11:30 +020062def updateSaltStack(target, pkgs) {
azvyagintsev97fd5ae2019-06-03 19:09:43 +030063 salt.cmdRun(venvPepper, 'I@salt:master', "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
Denis Egorenko4b54e7f2019-02-26 16:05:03 +040064 // can't use same function from pipeline lib, as at the moment of running upgrade pipeline Jenkins
65 // still using pipeline lib from current old mcp-version
Denis V. Meltsaykinfd7586b2020-06-02 11:23:18 +020066
67 // sleep to make sure package update started, otherwise checks will pass on still running old instance
68 sleep(120)
69
azvyagintsev6dac3802018-11-09 15:11:30 +020070 common.retry(20, 60) {
Richard Felkl0e80d892018-06-20 13:44:54 +020071 salt.minionsReachable(venvPepper, 'I@salt:master', '*')
72 def running = salt.runSaltProcessStep(venvPepper, target, 'saltutil.running', [], null, true, 5)
azvyagintsev6dac3802018-11-09 15:11:30 +020073 for (value in running.get("return")[0].values()) {
74 if (value != []) {
Richard Felkl0e80d892018-06-20 13:44:54 +020075 throw new Exception("Not all salt-minions are ready for execution")
76 }
77 }
78 }
79
azvyagintsevf3bf77a2018-11-15 19:30:04 +020080 def saltVersion = salt.getPillar(venvPepper, 'I@salt:master', '_param:salt_version').get('return')[0].values()[0]
Richard Felklf23257e2018-08-06 09:24:23 +020081 def saltMinionVersions = salt.cmdRun(venvPepper, target, "apt-cache policy salt-common | awk '/Installed/ && /$saltVersion/'").get("return")
Richard Felkl0e80d892018-06-20 13:44:54 +020082 def saltMinionVersion = ""
83
azvyagintsev6dac3802018-11-09 15:11:30 +020084 for (minion in saltMinionVersions[0].keySet()) {
85 saltMinionVersion = saltMinionVersions[0].get(minion).replace("Salt command execution success", "").trim()
86 if (saltMinionVersion == "") {
Richard Felkl0e80d892018-06-20 13:44:54 +020087 error("Installed version of Salt on $minion doesn't match specified version in the model.")
88 }
89 }
90}
91
Denis Egorenko20a43462019-09-24 14:55:50 +040092def getWorkerThreads(saltId) {
93 if (env.getEnvironment().containsKey('SALT_MASTER_OPT_WORKER_THREADS')) {
94 return env['SALT_MASTER_OPT_WORKER_THREADS'].toString()
95 }
96 def threads = salt.cmdRun(saltId, "I@salt:master", "cat /etc/salt/master.d/master.conf | grep worker_threads | cut -f 2 -d ':'", true, null, true)
97 return threads['return'][0].values()[0].replaceAll('Salt command execution success','').trim()
98}
99
Ivan Berezovskiyf5c911b2019-12-03 18:49:09 +0400100def wa29352(String cname) {
azvyagintsev745a6e52019-04-18 13:19:39 +0300101 // WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/12/openssh/client/root.yml
102 // Default soft-param has been removed, what now makes not possible to render some old env's.
103 // Like fix, we found copy-paste already generated key from backups, to secrets.yml with correct key name
104 def wa29352ClassName = 'cluster.' + cname + '.infra.secrets_root_wa29352'
105 def wa29352File = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets_root_wa29352.yml"
106 def wa29352SecretsFile = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets.yml"
107 def _tempFile = '/tmp/wa29352_' + UUID.randomUUID().toString().take(8)
108 try {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400109 salt.cmdRun(venvPepper, 'I@salt:master', "! grep -qi root_private_key: ${wa29352SecretsFile}", true, null, false)
azvyagintsev745a6e52019-04-18 13:19:39 +0300110 salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29352File}", true, null, false)
111 }
112 catch (Exception ex) {
Denis Egorenkodfa35422019-10-07 12:46:14 +0400113 common.infoMsg('Work-around for PROD-29352 already applied, nothing todo')
azvyagintsev745a6e52019-04-18 13:19:39 +0300114 return
115 }
116 def rKeysDict = [
117 'parameters': [
118 '_param': [
119 'root_private_key': salt.getPillar(venvPepper, 'I@salt:master', '_param:root_private_key').get('return')[0].values()[0].trim(),
120 'root_public_key' : '',
121 ]
122 ]
123 ]
124 // save root key,and generate public one from it
125 writeFile file: _tempFile, text: rKeysDict['parameters']['_param']['root_private_key'].toString().trim()
126 sh('chmod 0600 ' + _tempFile)
127 rKeysDict['parameters']['_param']['root_public_key'] = sh(script: "ssh-keygen -q -y -f ${_tempFile}", returnStdout: true).trim()
128 sh('rm -fv ' + _tempFile)
129 writeYaml file: _tempFile, data: rKeysDict
130 def yamlData = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
131 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${yamlData}' | base64 -d > ${wa29352File}", false, null, false)
132 common.infoMsg("Add $wa29352ClassName class into secrets.yml")
133
134 // Add 'classes:' directive
135 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
136 "grep -q 'classes:' infra/secrets.yml || sed -i '1iclasses:' infra/secrets.yml")
137
138 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
139 "grep -q '${wa29352ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29352ClassName' infra/secrets.yml")
azvyagintsev745a6e52019-04-18 13:19:39 +0300140 sh('rm -fv ' + _tempFile)
Denis Egorenkodfa35422019-10-07 12:46:14 +0400141 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && git status && " +
142 "git add ${wa29352File} && git add -u && git commit --allow-empty -m 'Cluster model updated with WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/ at ${common.getDatetime()}' ")
143 common.infoMsg('Work-around for PROD-29352 successfully applied')
azvyagintsev745a6e52019-04-18 13:19:39 +0300144}
145
azvyagintsevbc359d42019-04-10 19:53:22 +0300146def wa29155(ArrayList saltMinions, String cname) {
147 // WA for PROD-29155. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/
148 // CHeck for existence cmp nodes, and try to render it. Is failed, apply ssh-key wa
azvyagintsevbc359d42019-04-10 19:53:22 +0300149 def patched = false
150 def wa29155ClassName = 'cluster.' + cname + '.infra.secrets_nova_wa29155'
151 def wa29155File = "/srv/salt/reclass/classes/cluster/${cname}/infra/secrets_nova_wa29155.yml"
152
153 try {
154 salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29155File}", true, null, false)
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400155 def patch_required = false
156 for (String minion in saltMinions) {
157 def nova_key = salt.getPillar(venvPepper, minion, '_param:nova_compute_ssh_private').get("return")[0].values()[0]
158 if (nova_key == '' || nova_key == 'null' || nova_key == null) {
159 patch_required = true
160 break // no exception, proceeding to apply the patch
161 }
162 }
163 if (!patch_required) {
164 error('No need to apply work-around for PROD-29155')
165 }
azvyagintsevbc359d42019-04-10 19:53:22 +0300166 }
167 catch (Exception ex) {
azvyagintsevbc359d42019-04-10 19:53:22 +0300168 return
169 }
Denis Egorenko14fa6e12019-08-23 14:29:34 +0400170 salt.fullRefresh(venvPepper, 'I@salt:master')
azvyagintsevbc359d42019-04-10 19:53:22 +0300171 for (String minion in saltMinions) {
azvyagintsevbc359d42019-04-10 19:53:22 +0300172 // First attempt, second will be performed in next validateReclassModel() stages
173 try {
azvyagintsev745a6e52019-04-18 13:19:39 +0300174 salt.cmdRun(venvPepper, 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
azvyagintsevbc359d42019-04-10 19:53:22 +0300175 } catch (Exception e) {
176 common.errorMsg(e.toString())
177 if (patched) {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400178 error("Node: ${minion} failed to render after reclass-system upgrade! WA29155 probably didn't help.")
azvyagintsevbc359d42019-04-10 19:53:22 +0300179 }
azvyagintsev745a6e52019-04-18 13:19:39 +0300180 common.warningMsg('Perform: Attempt to apply WA for PROD-29155\n' +
181 'See https://gerrit.mcp.mirantis.com/#/c/37932/ for more info')
azvyagintsevbc359d42019-04-10 19:53:22 +0300182 common.warningMsg('WA-PROD-29155 Generating new ssh key at master node')
183 def _tempFile = "/tmp/nova_wa29155_" + UUID.randomUUID().toString().take(8)
184 common.infoMsg('Perform: generation NEW ssh-private key for nova-compute')
185 salt.cmdRun(venvPepper, 'I@salt:master', "ssh-keygen -f ${_tempFile} -N '' -q")
186 def _pub_k = salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'cmd.run', "cat ${_tempFile}.pub").get('return')[0].values()[0].trim()
187 def _priv_k = salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'cmd.run', "cat ${_tempFile}").get('return')[0].values()[0].trim()
188 salt.cmdRun(venvPepper, 'I@salt:master', "rm -fv ${_tempFile}", false, null, false)
189 def novaKeysDict = [
190 "parameters": [
191 "_param": [
192 "nova_compute_ssh_private": _priv_k,
193 "nova_compute_ssh_public" : _pub_k
194 ]
195 ]
196 ]
azvyagintsev745a6e52019-04-18 13:19:39 +0300197 writeYaml file: _tempFile, data: novaKeysDict
azvyagintsevbc359d42019-04-10 19:53:22 +0300198 def yamlData = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
199 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${yamlData}' | base64 -d > ${wa29155File}", false, null, false)
200 common.infoMsg("Add $wa29155ClassName class into secrets.yml")
201
202 // Add 'classes:' directive
203 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
204 "grep -q 'classes:' infra/secrets.yml || sed -i '1iclasses:' infra/secrets.yml")
205
206 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && " +
207 "grep -q '${wa29155ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29155ClassName' infra/secrets.yml")
Ivan Berezovskiyf5c911b2019-12-03 18:49:09 +0400208 salt.fullRefresh(venvPepper, 'I@salt:master')
Ivan Berezovskiyc4dc5012020-03-19 14:03:43 +0400209 salt.runSaltProcessStep(venvPepper, saltMinions, 'saltutil.refresh_pillar', [], null, true, 60)
azvyagintsevbc359d42019-04-10 19:53:22 +0300210 patched = true
211 }
212 }
213 if (patched) {
214 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && git status && " +
215 "git add ${wa29155File} && git add -u && git commit --allow-empty -m 'Cluster model updated with WA for PROD-29155. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/ at ${common.getDatetime()}' ")
216 common.infoMsg('Work-around for PROD-29155 successfully applied')
217 }
218
219}
220
Ivan Berezovskiy5b62e9c2020-01-24 17:52:26 +0400221def wa32284(String cluster_name) {
Denis Egorenko1b11d232019-07-29 17:44:46 +0400222 def clientGluster = salt.getPillar(venvPepper, 'I@salt:master', "glusterfs:client:enabled").get("return")[0].values()[0]
223 def pkiGluster = salt.getPillar(venvPepper, 'I@salt:master', "glusterfs:client:volumes:salt_pki").get("return")[0].values()[0]
224 def nginxEnabledAtMaster = salt.getPillar(venvPepper, 'I@salt:master', 'nginx:server:enabled').get('return')[0].values()[0]
225 if (nginxEnabledAtMaster.toString().toLowerCase() == 'true' && clientGluster.toString().toLowerCase() == 'true' && pkiGluster) {
226 def nginxRequires = salt.getPillar(venvPepper, 'I@salt:master', 'nginx:server:wait_for_service').get('return')[0].values()[0]
227 if (nginxRequires.isEmpty()) {
Ivan Berezovskiy5b62e9c2020-01-24 17:52:26 +0400228 def nginxRequiresClassName = "cluster.${cluster_name}.infra.config.nginx_requires_wa32284"
229 def nginxRequiresClassFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/infra/config/nginx_requires_wa32284.yml"
Denis Egorenko1b11d232019-07-29 17:44:46 +0400230 def nginxRequiresBlock = ['parameters': ['nginx': ['server': ['wait_for_service': ['srv-salt-pki.mount'] ] ] ] ]
231 def _tempFile = '/tmp/wa32284_' + UUID.randomUUID().toString().take(8)
232 writeYaml file: _tempFile , data: nginxRequiresBlock
233 def nginxRequiresBlockString = sh(script: "cat ${_tempFile}", returnStdout: true).trim()
Ivan Berezovskiy5b62e9c2020-01-24 17:52:26 +0400234 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && " +
Denis Egorenko1b11d232019-07-29 17:44:46 +0400235 "sed -i '/^parameters:/i - ${nginxRequiresClassName}' infra/config/init.yml")
236 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${nginxRequiresBlockString}' > ${nginxRequiresClassFile}", false, null, false)
Ivan Berezovskiy5b62e9c2020-01-24 17:52:26 +0400237 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && git status && git add ${nginxRequiresClassFile}")
Denis Egorenko1b11d232019-07-29 17:44:46 +0400238 }
239 }
240}
241
Oleksii Molchanov5b7fceb2020-03-13 13:28:10 +0200242def check_34406(String cluster_name) {
243 def sphinxpasswordPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:sphinx_proxy_password_generated').get("return")[0].values()[0]
244 if (sphinxpasswordPillar == '' || sphinxpasswordPillar == 'null' || sphinxpasswordPillar == null) {
245 error('Sphinx password is not defined.\n' +
246 'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-9/mu-9-addressed/mu-9-dtrain/mu-9-dt-manual.html#i-34406 for more info')
247 }
248}
249
Oleksii Molchanov05c49832020-09-28 19:28:06 +0300250def check_35705(String cluster_name) {
251 def galeracheckpasswordPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:galera_clustercheck_password').get("return")[0].values()[0]
252 if (galeracheckpasswordPillar == '' || galeracheckpasswordPillar == 'null' || galeracheckpasswordPillar == null) {
253 error('Galera clustercheck password is not defined.\n' +
254 'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#improper-operation-of-galera-ha for more info')
255 }
256}
257
Ksawery Dziekoński09fa3ce2020-12-02 13:32:40 +0100258def check_35884(String cluster_name) {
Vladimir Khlyunevff208812020-12-08 19:34:36 +0400259 if (salt.getMinions(venvPepper, 'I@prometheus:alerta or I@prometheus:alertmanager')) {
260 def alertaApiKeyGenPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:alerta_admin_api_key_generated').get("return")[0].values()[0]
261 def alertaApiKeyPillar = salt.getPillar(venvPepper, 'I@prometheus:alerta or I@prometheus:alertmanager', '_param:alerta_admin_key').get("return")[0].values()[0]
Ksawery Dziekoński09fa3ce2020-12-02 13:32:40 +0100262
Vladimir Khlyunevff208812020-12-08 19:34:36 +0400263 if (alertaApiKeyGenPillar == '' || alertaApiKeyGenPillar == 'null' || alertaApiKeyGenPillar == null || alertaApiKeyPillar == '' || alertaApiKeyPillar == 'null' || alertaApiKeyPillar == null) {
264 error('Alerta admin API key not defined.\n' +
265 'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#i-35884 for more info')
266 }
Ksawery Dziekoński09fa3ce2020-12-02 13:32:40 +0100267 }
268}
269
Vladimir Khlyunev8b7008f2021-07-15 15:18:49 +0400270// ceph cluster class ordering for radosgw
271def check_36461(String cluster_name){
272 if (!salt.testTarget(venvPepper, 'I@ceph:radosgw')) {
273 return
274 }
275 def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
276 def checkFile = "${clusterModelPath}/ceph/rgw.yml"
277 def saltTarget = "I@salt:master"
278 try {
279 salt.cmdRun(venvPepper, saltTarget, "test -f ${checkFile}")
280 }
281 catch (Exception e) {
282 common.warningMsg("Unable to check ordering of RadosGW imports, file ${checkFile} not found, skipping")
283 return
284 }
285 def fileContent = salt.cmdRun(venvPepper, saltTarget, "cat ${checkFile}").get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
286 def yamlData = readYaml text: fileContent
287 def infraClassImport = "cluster.${cluster_name}.infra"
288 def cephClassImport = "cluster.${cluster_name}.ceph"
289 def cephCommonClassImport = "cluster.${cluster_name}.ceph.common"
290 def infraClassFound = false
291 def importErrorDetected = false
292 def importErrorMessage = """Ceph classes in '${checkFile}' are used in wrong order! Please reorder it:
293'${infraClassImport}' should be placed before '${cephClassImport}' and '${cephCommonClassImport}'.
Vladimir Khlyunev318fc892021-07-20 14:33:53 +0400294For additional information please see https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-15/mu-15-addressed/mu-15-dtrain/mu-15-dtrain-manual.html"""
Vladimir Khlyunev8b7008f2021-07-15 15:18:49 +0400295 for (yamlClass in yamlData.classes) {
296 switch(yamlClass){
297 case infraClassImport:
298 infraClassFound = true;
299 break;
300 case cephClassImport:
301 if (!infraClassFound) {
302 importErrorDetected = true
303 };
304 break;
305 case cephCommonClassImport:
306 if (!infraClassFound) {
307 importErrorDetected = true
308 };
309 break;
310 }
311 }
312 if (importErrorDetected) {
313 common.errorMsg(importErrorMessage)
314 error(importErrorMessage)
315 }
316}
317
Denis Egorenkoaf18b252019-09-02 14:53:02 +0400318def wa32182(String cluster_name) {
Hanna Arhipovad5eec4c2019-09-06 17:55:40 +0300319 if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
Denis Egorenkoaf18b252019-09-02 14:53:02 +0400320 def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
321 def fixFile = "${clusterModelPath}/opencontrail/common_wa32182.yml"
322 def usualFile = "${clusterModelPath}/opencontrail/common.yml"
323 def fixFileContent = "classes:\n- system.opencontrail.common\n"
324 salt.cmdRun(venvPepper, 'I@salt:master', "test -f ${fixFile} -o -f ${usualFile} || echo '${fixFileContent}' > ${fixFile}")
325 def contrailFiles = ['opencontrail/analytics.yml', 'opencontrail/control.yml', 'openstack/compute/init.yml']
326 if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
327 contrailFiles.add('kubernetes/compute.yml')
328 }
Ivan Berezovskiy9f6330b2020-02-05 21:18:17 +0400329 for (String contrailFile in contrailFiles) {
Denis Egorenkoaf18b252019-09-02 14:53:02 +0400330 contrailFile = "${clusterModelPath}/${contrailFile}"
331 def containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- cluster\\.${cluster_name}\\.opencontrail\\.common(_wa32182)?\$' ${contrailFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
332 if (containsFix) {
333 continue
334 } else {
335 salt.cmdRun(venvPepper, 'I@salt:master', "grep -q -E '^parameters:' ${contrailFile} && sed -i '/^parameters:/i - cluster.${cluster_name}.opencontrail.common_wa32182' ${contrailFile} || " +
Ivan Berezovskiy9f6330b2020-02-05 21:18:17 +0400336 "echo '- cluster.${cluster_name}.opencontrail.common_wa32182' >> ${contrailFile}")
Denis Egorenkoaf18b252019-09-02 14:53:02 +0400337 }
338 }
Ivan Berezovskiy9f6330b2020-02-05 21:18:17 +0400339 salt.cmdRun(venvPepper, 'I@salt:master', "test -f ${fixFile} && cd ${clusterModelPath} && git status && git add ${fixFile} || true")
Denis Egorenkoaf18b252019-09-02 14:53:02 +0400340 }
341}
342
Denis Egorenkoba631332019-10-25 13:13:39 +0400343def wa33867(String cluster_name) {
344 if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
345 def contrailControlFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/opencontrail/control.yml"
346 def line = salt.cmdRun(venvPepper, 'I@salt:master', "awk '/^- cluster.${cluster_name}.infra.backup.client_zookeeper/ {getline; print \$0}' ${contrailControlFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
347 if (line == "- cluster.${cluster_name}.infra") {
348 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^- cluster.${cluster_name}.infra\$/d' ${contrailControlFile}")
349 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^- cluster.${cluster_name}.infra.backup.client_zookeeper\$/i - cluster.${cluster_name}.infra' ${contrailControlFile}")
350 }
351 }
352}
353
Denis Egorenkoe312cc92019-10-18 17:06:11 +0400354def wa33771(String cluster_name) {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400355 if (salt.getMinions(venvPepper, 'I@_param:openstack_node_role and I@apache:server')) {
356 def octaviaEnabled = salt.getMinions(venvPepper, 'I@octavia:api:enabled')
357 def octaviaWSGI = salt.getMinions(venvPepper, 'I@apache:server:site:octavia_api')
358 if (octaviaEnabled && !octaviaWSGI) {
359 def openstackControl = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/control.yml"
360 def octaviaFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/octavia_wa33771.yml"
361 def octaviaContext = [
362 'classes' : ['system.apache.server.site.octavia'],
363 'parameters': [
364 '_param': ['apache_octavia_api_address': '${_param:cluster_local_address}'],
365 ]
Denis Egorenkoe312cc92019-10-18 17:06:11 +0400366 ]
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400367 def openstackHTTPSEnabled = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_internal_protocol").get("return")[0].values()[0]
368 if (openstackHTTPSEnabled == 'https') {
369 octaviaContext['parameters'] << ['apache': ['server': ['site': ['apache_proxy_openstack_api_octavia': ['enabled': false]]]]]
370 }
371 def _tempFile = '/tmp/wa33771' + UUID.randomUUID().toString().take(8)
372 writeYaml file: _tempFile, data: octaviaContext
373 def octaviaFileContent = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
374 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^parameters:/i - cluster.${cluster_name}.openstack.octavia_wa33771' ${openstackControl}")
375 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${octaviaFileContent}' | base64 -d > ${octaviaFile}", false, null, false)
376 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && git status && git add ${octaviaFile}")
Denis Egorenko59f82a52020-01-30 17:17:25 +0400377 }
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400378 } else {
379 common.warningMsg("Apache server is not defined on controller nodes. Skipping Octavia WSGI workaround");
Denis Egorenkoe312cc92019-10-18 17:06:11 +0400380 }
381}
382
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400383def wa33930_33931(String cluster_name) {
384 def openstackControlFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/control.yml"
385 def fixName = 'clients_common_wa33930_33931'
386 def fixFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/${fixName}.yml"
387 def containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- cluster\\.${cluster_name}\\.openstack\\.${fixName}\$' ${openstackControlFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
388 if (! containsFix) {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400389 def fixContext = [ 'classes': [ ] ]
390 def novaControllerNodes = salt.getMinions(venvPepper, 'I@nova:controller')
391 for (novaController in novaControllerNodes) {
392 def novaClientPillar = salt.getPillar(venvPepper, novaController, "nova:client").get("return")[0].values()[0]
393 if (novaClientPillar == '' || novaClientPillar == 'null' || novaClientPillar == null) {
394 fixContext['classes'] << 'service.nova.client'
395 break
396 }
397 }
398 def glanceServerNodes = salt.getMinions(venvPepper, 'I@glance:server')
399 for (glanceServer in glanceServerNodes) {
400 def glanceClientPillar = salt.getPillar(venvPepper, glanceServer, "glance:client").get("return")[0].values()[0]
401 if (glanceClientPillar == '' || glanceClientPillar == 'null' || glanceClientPillar == null) {
402 fixContext['classes'] << 'service.glance.client'
403 break
404 }
405 }
406 def neutronServerNodes = salt.getMinions(venvPepper, 'I@neutron:server')
407 for (neutronServer in neutronServerNodes) {
408 def neutronServerPillar = salt.getPillar(venvPepper, neutronServer, "neutron:client").get("return")[0].values()[0]
409 if (neutronServerPillar == '' || neutronServerPillar == 'null' || neutronServerPillar == null) {
410 fixContext['classes'] << 'service.neutron.client'
411 break
412 }
413 }
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400414 if (salt.getMinions(venvPepper, 'I@manila:api:enabled')) {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400415 def manilaApiNodes = salt.getMinions(venvPepper, 'I@manila:api')
416 for (manilaNode in manilaApiNodes) {
417 def manilaNodePillar = salt.getPillar(venvPepper, manilaNode, "manila:client").get("return")[0].values()[0]
418 if (manilaNodePillar == '' || manilaNodePillar == 'null' || manilaNodePillar == null) {
419 fixContext['classes'] << 'service.manila.client'
420 break
421 }
422 }
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400423 }
424 if (salt.getMinions(venvPepper, 'I@ironic:api:enabled')) {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400425 def ironicApiNodes = salt.getMinions(venvPepper, 'I@ironic:api')
426 for (ironicNode in ironicApiNodes) {
427 def ironicNodePillar = salt.getPillar(venvPepper, ironicNode, "ironic:client").get("return")[0].values()[0]
428 if (ironicNodePillar == '' || ironicNodePillar == 'null' || ironicNodePillar == null) {
429 fixContext['classes'] << 'service.ironic.client'
430 break
431 }
432 }
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400433 }
434 if (salt.getMinions(venvPepper, 'I@gnocchi:server:enabled')) {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400435 def gnocchiServerNodes = salt.getMinions(venvPepper, 'I@gnocchi:server')
436 for (gnocchiNode in gnocchiServerNodes) {
437 def gnocchiNodePillar = salt.getPillar(venvPepper, gnocchiNode, "gnocchi:client").get("return")[0].values()[0]
438 if (gnocchiNodePillar == '' || gnocchiNodePillar == 'null' || gnocchiNodePillar == null) {
439 fixContext['classes'] << 'service.gnocchi.client'
440 break
441 }
442 }
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400443 }
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400444
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400445 if (salt.getMinions(venvPepper, 'I@barbican:server:enabled')) {
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400446 def barbicanServerNodes = salt.getMinions(venvPepper, 'I@barbican:server')
447 for (barbicanNode in barbicanServerNodes) {
448 def barbicanNodePillar = salt.getPillar(venvPepper, barbicanNode, "barbican:client").get("return")[0].values()[0]
449 if (barbicanNodePillar == '' || barbicanNodePillar == 'null' || barbicanNodePillar == null) {
450 fixContext['classes'] << 'service.barbican.client.single'
451 break
452 }
453 }
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400454 }
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400455 if (fixContext['classes'] != []) {
456 def _tempFile = '/tmp/wa33930_33931' + UUID.randomUUID().toString().take(8)
457 writeYaml file: _tempFile, data: fixContext
458 def fixFileContent = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
459 salt.cmdRun(venvPepper, 'I@salt:master', "echo '${fixFileContent}' | base64 -d > ${fixFile}", false, null, false)
460 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^parameters:/i - cluster.${cluster_name}.openstack.${fixName}' ${openstackControlFile}")
461 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && git status && git add ${fixFile}")
462 }
Denis Egorenkoe9fc0592019-11-14 19:56:17 +0400463 }
464}
465
Denis Egorenko215acda2020-01-28 16:41:44 +0400466def wa34245(cluster_name) {
467 def infraInitFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/infra/init.yml"
468 def fixName = 'hosts_wa34245'
469 def fixFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/infra/${fixName}.yml"
470 if (salt.testTarget(venvPepper, 'I@keystone:server')) {
471 def fixApplied = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- cluster.${cluster_name}.infra.${fixName}\$' ${infraInitFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
472 if (!fixApplied) {
473 def fixFileContent = []
474 def containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- system\\.linux\\.network\\.hosts\\.openstack\$' ${infraInitFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
475 if (!containsFix) {
476 fixFileContent << '- system.linux.network.hosts.openstack'
477 }
478 if (salt.testTarget(venvPepper, 'I@gnocchi:server')) {
479 containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- system\\.linux\\.network\\.hosts\\.openstack\\.telemetry\$' ${infraInitFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
480 if (!containsFix) {
481 fixFileContent << '- system.linux.network.hosts.openstack.telemetry'
482 }
483 }
484 if (salt.testTarget(venvPepper, 'I@manila:api')) {
485 containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- system\\.linux\\.network\\.hosts\\.openstack\\.share\$' ${infraInitFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
486 if (!containsFix) {
487 fixFileContent << '- system.linux.network.hosts.openstack.share'
488 }
489 }
490 if (salt.testTarget(venvPepper, 'I@barbican:server')) {
491 containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- system\\.linux\\.network\\.hosts\\.openstack\\.kmn\$' ${infraInitFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
492 if (!containsFix) {
493 fixFileContent << '- system.linux.network.hosts.openstack.kmn'
494 }
495 }
496 if (fixFileContent) {
497 salt.cmdRun(venvPepper, 'I@salt:master', "echo 'classes:\n${fixFileContent.join('\n')}' > ${fixFile}")
498 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^parameters:/i - cluster.${cluster_name}.infra.${fixName}' ${infraInitFile}")
Ivan Berezovskiy5b62e9c2020-01-24 17:52:26 +0400499 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && git status && git add ${fixFile}")
Denis Egorenko215acda2020-01-28 16:41:44 +0400500 }
501 }
502 }
503}
504
Ivan Berezovskiy5b62e9c2020-01-24 17:52:26 +0400505def wa34528(String cluster_name) {
506 // Mysql users have to be defined on each Galera node
507 if(salt.getMinions(venvPepper, 'I@galera:master').isEmpty()) {
508 common.errorMsg('No Galera master found in cluster. Skipping')
509 return
510 }
511 def mysqlUsersMasterPillar = salt.getPillar(venvPepper, 'I@galera:master', 'mysql:server:database').get("return")[0].values()[0]
512 if (mysqlUsersMasterPillar == '' || mysqlUsersMasterPillar == 'null' || mysqlUsersMasterPillar == null) {
513 common.errorMsg('Pillar data is broken for Galera master node!')
514 input message: 'Do you want to ignore and continue without Galera pillar patch?'
515 return
516 }
517 def fileToPatch = salt.cmdRun(venvPepper, 'I@salt:master', "ls /srv/salt/reclass/classes/cluster/${cluster_name}/openstack/database/init.yml || " +
518 "ls /srv/salt/reclass/classes/cluster/${cluster_name}/openstack/database/slave.yml || echo 'File not found'", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
519 if (fileToPatch == 'File not found') {
520 common.errorMsg('Cluster model is old and cannot be patched for PROD-34528. Patching is possible for 2019.2.x cluster models only')
521 return
522 }
523 def patchRequired = false
524 def mysqlUsersSlavePillar = ''
525 def galeraSlaveNodes = salt.getMinions(venvPepper, 'I@galera:slave')
526 if (!galeraSlaveNodes.isEmpty()) {
527 for (galeraSlave in galeraSlaveNodes) {
528 mysqlUsersSlavePillar = salt.getPillar(venvPepper, galeraSlave, 'mysql:server:database').get("return")[0].values()[0]
529 if (mysqlUsersSlavePillar == '' || mysqlUsersSlavePillar == 'null' || mysqlUsersSlavePillar == null) {
530 common.errorMsg('Mysql users data is not defined for Galera slave nodes. Fixing...')
531 patchRequired = true
532 break
533 }
534 }
535 if (patchRequired) {
536 def fixFileContent = []
537 def fixName = 'db_wa34528'
538 def fixFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/database/${fixName}.yml"
539 for (dbName in mysqlUsersMasterPillar.keySet()) {
540 def classIncluded = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- system\\.galera\\.server\\.database\\.${dbName}\$'" +
541 " /srv/salt/reclass/classes/cluster/${cluster_name}/openstack/database/master.yml", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
542 if(classIncluded) {
543 fixFileContent << "- system.galera.server.database.${dbName}"
544 }
545 def sslClassIncluded = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- system\\.galera\\.server\\.database\\.x509\\.${dbName}\$'" +
546 " /srv/salt/reclass/classes/cluster/${cluster_name}/openstack/database/master.yml", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
547 if(sslClassIncluded) {
548 fixFileContent << "- system.galera.server.database.x509.${dbName}"
549 }
550 }
551 if (fixFileContent) {
552 salt.cmdRun(venvPepper, 'I@salt:master', "echo 'classes:\n${fixFileContent.join('\n')}' > ${fixFile}")
553 salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^parameters:/i - cluster.${cluster_name}.openstack.database.${fixName}' ${fileToPatch}")
554 }
555 salt.fullRefresh(venvPepper, 'I@galera:slave')
556 // Verify
557 for (galeraSlave in galeraSlaveNodes) {
558 mysqlUsersSlavePillar = salt.getPillar(venvPepper, galeraSlave, 'mysql:server:database').get("return")[0].values()[0]
559 if (mysqlUsersSlavePillar == '' || mysqlUsersSlavePillar == 'null' || mysqlUsersSlavePillar == null || mysqlUsersSlavePillar.keySet() != mysqlUsersMasterPillar.keySet()) {
560 common.errorMsg("Mysql user data is different on master and slave node ${galeraSlave}.")
561 input message: 'Do you want to ignore and continue?'
562 }
563 }
564 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && git status && git add ${fixFile}")
565 common.infoMsg('Galera slaves patching is done')
566 } else {
567 common.infoMsg('Galera slaves patching is not required')
568 }
569 }
570}
571
azvyagintsev6dac3802018-11-09 15:11:30 +0200572def archiveReclassInventory(filename) {
azvyagintsevbc359d42019-04-10 19:53:22 +0300573 def _tmp_file = '/tmp/' + filename + UUID.randomUUID().toString().take(8)
574 // jenkins may fail at overheap. Compress data with gzip like WA
azvyagintsev745a6e52019-04-18 13:19:39 +0300575 def ret = salt.cmdRun(venvPepper, 'I@salt:master', 'reclass -i 2>/dev/null | gzip -9 -c | base64', true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
azvyagintsevbc359d42019-04-10 19:53:22 +0300576 def _tmp = sh(script: "echo '$ret' > ${_tmp_file}", returnStdout: false)
577 sh(script: "cat ${_tmp_file} | base64 -d | gzip -d > $filename", returnStdout: false)
578 archiveArtifacts artifacts: filename
579 sh(script: "rm -v ${_tmp_file}|| true")
Richard Felkl0e80d892018-06-20 13:44:54 +0200580}
581
Denis Egorenkod13ad132018-12-17 16:56:02 +0400582def validateReclassModel(ArrayList saltMinions, String suffix) {
583 try {
azvyagintsevbc359d42019-04-10 19:53:22 +0300584 for (String minion in saltMinions) {
Denis Egorenkod13ad132018-12-17 16:56:02 +0400585 common.infoMsg("Reclass model validation for minion ${minion}...")
azvyagintsev745a6e52019-04-18 13:19:39 +0300586 def reclassInv = salt.cmdRun(venvPepper, 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
Denis Egorenkod13ad132018-12-17 16:56:02 +0400587 writeFile file: "inventory-${minion}-${suffix}.out", text: reclassInv.toString()
588 }
589 } catch (Exception e) {
590 common.errorMsg('Can not validate current Reclass model. Inspect failed minion manually.')
azvyagintsevbc359d42019-04-10 19:53:22 +0300591 error(e.toString())
Denis Egorenkod13ad132018-12-17 16:56:02 +0400592 }
593}
594
azvyagintsevbc359d42019-04-10 19:53:22 +0300595def archiveReclassModelChanges(ArrayList saltMinions, String oldSuffix = 'before', String newSuffix = 'after') {
596 for (String minion in saltMinions) {
Denis Egorenkod13ad132018-12-17 16:56:02 +0400597 def fileName = "reclass-model-${minion}-diff.out"
598 sh "diff -u inventory-${minion}-${oldSuffix}.out inventory-${minion}-${newSuffix}.out > ${fileName} || true"
599 archiveArtifacts artifacts: "${fileName}"
600 }
601}
602
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300603def checkDebsums() {
604 // check for salt-formulas consistency
605 try {
606 try {
607 salt.cmdRun(venvPepper, 'I@salt:master', "salt -C 'I@salt:master' pkg.install force_yes=True pkgs=[debsums]")
608 }
609 catch (Exception ex) {
610 common.warningMsg('Unable to install package "debsums" at cfg01. Salt-formulas integrity check skipped')
611 }
612 salt.cmdRun(venvPepper, 'I@salt:master', '> /root/debdsums_report; for i in $(dpkg-query -W -f=\'${Package}\\n\' | sed "s/ //g" |grep \'salt-formula-\'); do debsums -s ${i} 2>> /root/debdsums_report; done')
613 salt.cmdRun(venvPepper, 'I@salt:master', 'if [ -s "/root/debdsums_report" ]; then exit 1 ; fi')
614 }
615 catch (Exception ex) {
616 common.errorMsg(salt.cmdRun(venvPepper, 'I@salt:master', 'cat /root/debdsums_report ', true, null, false).get('return')[0].values()[0].trim())
617 common.errorMsg(ex.toString())
618 error('You have unexpected changes in formulas. All of them will be overwrited by update. Unable to continue in automatic way')
619 }
620}
621
Denis Egorenko96cc57d2020-01-24 19:47:05 +0400622def checkCICDDocker() {
623 common.infoMsg('Perform: Checking if Docker containers are up')
624 try {
625 common.retry(10, 30) {
626 salt.cmdRun(venvPepper, 'I@jenkins:client and I@docker:client', "! docker service ls | tail -n +2 | grep -v -E '\\s([0-9])/\\1\\s'")
627 }
628 }
629 catch (Exception ex) {
630 error("Docker containers for CI/CD services are having troubles with starting.")
631 }
632}
633
Denis Egorenko5a8a7e92018-12-27 14:13:48 +0400634if (common.validInputParam('PIPELINE_TIMEOUT')) {
635 try {
636 pipelineTimeout = env.PIPELINE_TIMEOUT.toInteger()
azvyagintsevbc359d42019-04-10 19:53:22 +0300637 } catch (Exception e) {
Denis Egorenko5a8a7e92018-12-27 14:13:48 +0400638 common.warningMsg("Provided PIPELINE_TIMEOUT parameter has invalid value: ${env.PIPELINE_TIMEOUT} - should be interger")
639 }
Martin Polreich965c9032018-08-09 16:33:57 +0200640}
641
642timeout(time: pipelineTimeout, unit: 'HOURS') {
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300643 node('python') {
Jakub Josefa63f9862018-01-11 17:58:38 +0100644 try {
azvyagintsevbc359d42019-04-10 19:53:22 +0300645 def inventoryBeforeFilename = "reclass-inventory-before.out"
646 def inventoryAfterFilename = "reclass-inventory-after.out"
Richard Felkl0e80d892018-06-20 13:44:54 +0200647 workspace = common.getWorkspace()
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200648 targetMcpVersion = null
649 if (!common.validInputParam('TARGET_MCP_VERSION') && !common.validInputParam('MCP_VERSION')) {
650 error('You must specify MCP version in TARGET_MCP_VERSION|MCP_VERSION variable')
azvyagintsev6dac3802018-11-09 15:11:30 +0200651 }
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200652 // bw comp. for 2018.X => 2018.11 release
azvyagintsevbc359d42019-04-10 19:53:22 +0300653 if (common.validInputParam('MCP_VERSION')) {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200654 targetMcpVersion = env.MCP_VERSION
655 common.warningMsg("targetMcpVersion has been changed to:${targetMcpVersion}, which was taken from deprecated pipeline viriable:MCP_VERSION")
azvyagintsevbc359d42019-04-10 19:53:22 +0300656 } else {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200657 targetMcpVersion = env.TARGET_MCP_VERSION
658 }
659 // end bw comp. for 2018.X => 2018.11 release
Denis Egorenkod13ad132018-12-17 16:56:02 +0400660 def gitTargetMcpVersion = env.getProperty('GIT_REFSPEC')
Ivan Berezovskiydb325092019-01-16 19:26:18 +0400661 if (targetMcpVersion in ['nightly', 'testing']) {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200662 gitTargetMcpVersion = 'master'
Ivan Berezovskiydb325092019-01-16 19:26:18 +0400663 } else if (targetMcpVersion == 'proposed') {
664 gitTargetMcpVersion = 'proposed'
Denis Egorenko284690e2018-12-25 15:55:12 +0400665 } else if (!gitTargetMcpVersion) {
666 // backward compatibility for 2018.11.0
667 gitTargetMcpVersion = "release/${targetMcpVersion}"
azvyagintsev6dac3802018-11-09 15:11:30 +0200668 }
Ivan Berezovskiydb325092019-01-16 19:26:18 +0400669 common.warningMsg("gitTargetMcpVersion has been changed to:${gitTargetMcpVersion}")
Denis Egorenko284690e2018-12-25 15:55:12 +0400670 def upgradeSaltStack = ''
671 def updateClusterModel = ''
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400672 def applyWorkarounds = true
Denis Egorenko284690e2018-12-25 15:55:12 +0400673 def updatePipelines = ''
674 def updateLocalRepos = ''
675 def reclassSystemBranch = ''
Denis Egorenko517e06c2019-03-29 13:24:15 +0400676 def reclassSystemBranchDefault = gitTargetMcpVersion
Ivan Berezovskiyf5c911b2019-12-03 18:49:09 +0400677 if (gitTargetMcpVersion ==~ /^\d\d\d\d\.\d\d?\.\d+$/) {
678 reclassSystemBranchDefault = "tags/${gitTargetMcpVersion}"
679 } else if (gitTargetMcpVersion != 'proposed') {
Denis Egorenko517e06c2019-03-29 13:24:15 +0400680 reclassSystemBranchDefault = "origin/${gitTargetMcpVersion}"
681 }
Hanna Arhipovad5eec4c2019-09-06 17:55:40 +0300682 def driveTrainParamsYaml = env.getProperty('DRIVE_TRAIN_PARAMS')
683 if (driveTrainParamsYaml) {
684 def driveTrainParams = readYaml text: driveTrainParamsYaml
685 saltMastURL = driveTrainParams.get('SALT_MASTER_URL')
686 saltMastCreds = driveTrainParams.get('SALT_MASTER_CREDENTIALS')
687 upgradeSaltStack = driveTrainParams.get('UPGRADE_SALTSTACK', false).toBoolean()
688 updateClusterModel = driveTrainParams.get('UPDATE_CLUSTER_MODEL', false).toBoolean()
689 updatePipelines = driveTrainParams.get('UPDATE_PIPELINES', false).toBoolean()
690 updateLocalRepos = driveTrainParams.get('UPDATE_LOCAL_REPOS', false).toBoolean()
691 reclassSystemBranch = driveTrainParams.get('RECLASS_SYSTEM_BRANCH', reclassSystemBranchDefault)
Denis Egorenkoca6aeca2019-08-19 19:41:27 +0400692 batchSize = driveTrainParams.get('BATCH_SIZE', '')
Denis Egorenko96cc57d2020-01-24 19:47:05 +0400693 if (driveTrainParams.get('OS_DIST_UPGRADE', false).toBoolean() == true) {
694 packageUpgradeMode = 'dist-upgrade'
695 } else if (driveTrainParams.get('OS_UPGRADE', false).toBoolean() == true) {
696 packageUpgradeMode = 'upgrade'
697 }
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400698 applyWorkarounds = driveTrainParams.get('APPLY_MODEL_WORKAROUNDS', true).toBoolean()
Denis Egorenko284690e2018-12-25 15:55:12 +0400699 } else {
700 // backward compatibility for 2018.11.0
701 saltMastURL = env.getProperty('SALT_MASTER_URL')
702 saltMastCreds = env.getProperty('SALT_MASTER_CREDENTIALS')
Denis Egorenkoe5f39142018-12-28 12:30:48 +0400703 upgradeSaltStack = env.getProperty('UPGRADE_SALTSTACK').toBoolean()
704 updateClusterModel = env.getProperty('UPDATE_CLUSTER_MODEL').toBoolean()
705 updatePipelines = env.getProperty('UPDATE_PIPELINES').toBoolean()
706 updateLocalRepos = env.getProperty('UPDATE_LOCAL_REPOS').toBoolean()
Denis Egorenko517e06c2019-03-29 13:24:15 +0400707 reclassSystemBranch = reclassSystemBranchDefault
Denis Egorenko284690e2018-12-25 15:55:12 +0400708 }
Denis Egorenkod13ad132018-12-17 16:56:02 +0400709 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
Denis Egorenkod13ad132018-12-17 16:56:02 +0400710 def minions = salt.getMinions(venvPepper, '*')
Denis Egorenkofbcee412019-04-19 15:05:18 +0400711 def cluster_name = salt.getPillar(venvPepper, 'I@salt:master', "_param:cluster_name").get("return")[0].values()[0]
712 if (cluster_name == '' || cluster_name == 'null' || cluster_name == null) {
713 error('Pillar data is broken for Salt master node! Please check it manually and re-run pipeline.')
714 }
Hanna Arhipovad5eec4c2019-09-06 17:55:40 +0300715 if (!batchSize) {
Denis Egorenko7b098c32019-12-19 16:45:25 +0400716 // if no batch size provided get current worker threads and set batch size to 2/3 of it to avoid
717 // 'SaltReqTimeoutError: Message timed out' issue on Salt targets for large amount of nodes
718 // do not use toDouble/Double as it requires additional approved method
719 def workerThreads = getWorkerThreads(venvPepper).toInteger()
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400720 batchSize = (workerThreads * 2 / 3).toString().tokenize('.')[0]
Denis Egorenkoca6aeca2019-08-19 19:41:27 +0400721 }
Ivan Berezovskiyf5c911b2019-12-03 18:49:09 +0400722 def computeMinions = salt.getMinions(venvPepper, 'I@nova:compute')
Ivan Berezovskiyc4dc5012020-03-19 14:03:43 +0400723 def allMinions = salt.getMinions(venvPepper, '*')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400724
azvyagintsevbc359d42019-04-10 19:53:22 +0300725 stage('Update Reclass and Salt-Formulas') {
726 common.infoMsg('Perform: Full salt sync')
Ivan Berezovskiyc4dc5012020-03-19 14:03:43 +0400727 fullRefreshOneByOne(venvPepper, allMinions)
Oleksii Molchanov5b7fceb2020-03-13 13:28:10 +0200728
729 check_34406(cluster_name)
Oleksii Molchanov05c49832020-09-28 19:28:06 +0300730 check_35705(cluster_name)
Ksawery Dziekoński09fa3ce2020-12-02 13:32:40 +0100731 check_35884(cluster_name)
Vladimir Khlyunev8b7008f2021-07-15 15:18:49 +0400732 check_36461(cluster_name)
Oleksii Molchanov5b7fceb2020-03-13 13:28:10 +0200733
azvyagintsevbc359d42019-04-10 19:53:22 +0300734 common.infoMsg('Perform: Validate reclass medata before processing')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400735 validateReclassModel(minions, 'before')
736
azvyagintsev745a6e52019-04-18 13:19:39 +0300737 common.infoMsg('Perform: archiveReclassInventory before upgrade')
738 archiveReclassInventory(inventoryBeforeFilename)
739
azvyagintsev6dac3802018-11-09 15:11:30 +0200740 try {
azvyagintsevbc359d42019-04-10 19:53:22 +0300741 salt.cmdRun(venvPepper, 'I@salt:master', 'cd /srv/salt/reclass/ && git status && git diff-index --quiet HEAD --')
Richard Felkl970e0082018-06-12 18:00:51 +0200742 }
azvyagintsev6dac3802018-11-09 15:11:30 +0200743 catch (Exception ex) {
azvyagintsevbc359d42019-04-10 19:53:22 +0300744 error('You have uncommitted changes in your Reclass cluster model repository. Please commit or reset them and rerun the pipeline.')
Richard Felkl970e0082018-06-12 18:00:51 +0200745 }
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300746 checkDebsums()
Denis Egorenkod13ad132018-12-17 16:56:02 +0400747 if (updateClusterModel) {
azvyagintsev6dac3802018-11-09 15:11:30 +0200748 common.infoMsg('Perform: UPDATE_CLUSTER_MODEL')
Sergey382581a2018-10-25 12:22:26 +0400749 def dateTime = common.getDatetime()
Ivan Berezovskiycfa1b2f2020-01-17 13:55:18 +0400750 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git fetch")
Sergey382581a2018-10-25 12:22:26 +0400751 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenkofcbc6b82019-01-15 11:47:16 +0400752 "grep -r --exclude-dir=aptly -l 'mcp_version: .*' * | xargs --no-run-if-empty sed -i 's|mcp_version: .*|mcp_version: \"$targetMcpVersion\"|g'")
azvyagintsev6dac3802018-11-09 15:11:30 +0200753 // Do the same, for deprecated variable-duplicate
754 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenkofcbc6b82019-01-15 11:47:16 +0400755 "grep -r --exclude-dir=aptly -l 'apt_mk_version: .*' * | xargs --no-run-if-empty sed -i 's|apt_mk_version: .*|apt_mk_version: \"$targetMcpVersion\"|g'")
Denis Egorenko284690e2018-12-25 15:55:12 +0400756 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenkofcbc6b82019-01-15 11:47:16 +0400757 "grep -r --exclude-dir=aptly -l 'jenkins_pipelines_branch: .*' * | xargs --no-run-if-empty sed -i 's|jenkins_pipelines_branch: .*|jenkins_pipelines_branch: \"$gitTargetMcpVersion\"|g'")
Victor Ryzhenkinabbefe62019-01-15 19:58:03 +0400758 // Set new k8s param
759 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
760 "grep -r --exclude-dir=aptly -l 'kubernetes_containerd_enabled: .*' * | xargs --no-run-if-empty sed -i 's|kubernetes_containerd_enabled: .*|kubernetes_containerd_enabled: True|g'")
Denis Egorenkoe5b49512019-01-16 19:18:31 +0400761 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
762 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.salt' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.salt/system.linux.system.repo.mcp.apt_mirantis.salt-formulas/g'")
763 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Denis Egorenko301cc8a2019-02-13 13:49:13 +0400764 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.contrail' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.contrail/system.linux.system.repo.mcp.apt_mirantis.contrail/g'")
Denis Egorenkoe5b49512019-01-16 19:18:31 +0400765 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
Vladimir Khlyunev8efa75a2021-07-13 14:44:01 +0400766 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.updates\$' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.updates\$/system.linux.system.repo.mcp.apt_mirantis.update/g'")
Anton Samoylov308c7ca2019-02-13 15:57:13 +0400767 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
azvyagintsevbc359d42019-04-10 19:53:22 +0300768 "grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
Denis Egorenko21c78be2019-07-10 19:47:39 +0400769
Denis Egorenko6f07b592019-09-16 16:41:26 +0400770 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/linux_system_repo_mcp_maas_url/d' maas.yml")
771 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/maas_region_main_archive/d' maas.yml")
772
Denis Egorenko21c78be2019-07-10 19:47:39 +0400773 // Switch Jenkins/Gerrit to use LDAP SSL/TLS
774 def gerritldapURI = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
775 "grep -r --exclude-dir=aptly 'gerrit_ldap_server: .*' * | grep -Po 'gerrit_ldap_server: \\K.*' | tr -d '\"'", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
776 if (gerritldapURI.startsWith('ldap://')) {
777 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
778 "grep -r --exclude-dir=aptly -l 'gerrit_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|ldap://|ldaps://|g'")
779 } else if (! gerritldapURI.startsWith('ldaps://')) {
780 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
781 "grep -r --exclude-dir=aptly -l 'gerrit_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|gerrit_ldap_server: .*|gerrit_ldap_server: \"ldaps://${gerritldapURI}\"|g'")
782 }
783 def jenkinsldapURI = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
784 "grep -r --exclude-dir=aptly 'jenkins_security_ldap_server: .*' * | grep -Po 'jenkins_security_ldap_server: \\K.*' | tr -d '\"'", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
785 if (jenkinsldapURI.startsWith('ldap://')) {
786 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
787 "grep -r --exclude-dir=aptly -l 'jenkins_security_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|ldap://|ldaps://|g'")
788 } else if (! jenkinsldapURI.startsWith('ldaps://')) {
789 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
790 "grep -r --exclude-dir=aptly -l 'jenkins_security_ldap_server: .*' * | xargs --no-run-if-empty sed -i 's|jenkins_security_ldap_server: .*|jenkins_security_ldap_server: \"ldaps://${jenkinsldapURI}\"|g'")
791 }
792
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400793 if (applyWorkarounds) {
794 wa32284(cluster_name)
795 wa34245(cluster_name)
796 }
Denis Egorenko1b11d232019-07-29 17:44:46 +0400797
Denis Egorenkod13ad132018-12-17 16:56:02 +0400798 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/system && git checkout ${reclassSystemBranch}")
Sergey Kolekonov5dfbf7a2019-01-23 16:47:43 +0400799 // Add kubernetes-extra repo
800 if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
Sergey Kolekonov3fde8b12019-04-11 16:33:54 +0400801 // docker-engine conflicts with the recent containerd versions, so it's removed during upgrade. Thus update source engine
802 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
803 "grep -r -l 'engine: docker_hybrid' kubernetes | xargs --no-run-if-empty sed -i 's/engine: docker_hybrid/engine: archive/g'")
Sergey Kolekonov5dfbf7a2019-01-23 16:47:43 +0400804 common.infoMsg("Add kubernetes-extra repo")
805 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
806 "grep -q system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.update.kubernetes_extra' kubernetes/common.yml")
807 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
808 "grep -q system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra kubernetes/common.yml || sed -i '/classes:/ a - system.linux.system.repo.mcp.apt_mirantis.kubernetes_extra' kubernetes/common.yml")
809 }
Ivan Berezovskiy41c76792019-04-15 13:23:58 +0400810 // Add all update repositories
811 def repoIncludeBase = '- system.linux.system.repo.mcp.apt_mirantis.'
Denis Egorenko6f07b592019-09-16 16:41:26 +0400812 def updateRepoList = ['cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'maas', 'percona', 'salt-formulas', 'saltstack', 'ubuntu']
Ivan Berezovskiy41c76792019-04-15 13:23:58 +0400813 updateRepoList.each { repo ->
814 def repoNameUpdateInclude = "${repoIncludeBase}update.${repo}"
815 def filesWithInclude = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -Plr '\\${repoIncludeBase}${repo}\$' . || true", false).get('return')[0].values()[0].trim().tokenize('\n')
816 filesWithInclude.each { file ->
817 def updateRepoIncludeExist = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -P '\\${repoNameUpdateInclude}\$' ${file} || echo not_found", false, null, true).get('return')[0].values()[0].trim()
818 if (updateRepoIncludeExist == 'not_found') {
819 // Include needs to be added
820 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
azvyagintsevbc359d42019-04-10 19:53:22 +0300821 "sed -i 's/\\( *\\)${repoIncludeBase}${repo}\$/&\\n\\1${repoNameUpdateInclude}/g' ${file}")
Ivan Berezovskiy41c76792019-04-15 13:23:58 +0400822 common.infoMsg("Update repo for ${repo} is added to ${file}")
823 }
824 }
825 }
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400826 if (applyWorkarounds) {
827 wa32182(cluster_name)
828 wa33771(cluster_name)
829 wa33867(cluster_name)
830 wa33930_33931(cluster_name)
831 wa34528(cluster_name)
832 }
Sergey382581a2018-10-25 12:22:26 +0400833 // Add new defaults
834 common.infoMsg("Add new defaults")
Denis Egorenkod13ad132018-12-17 16:56:02 +0400835 salt.cmdRun(venvPepper, 'I@salt:master', "grep '^ mcp_version: ' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +
Denis Egorenkofcbc6b82019-01-15 11:47:16 +0400836 "sed -i 's|^ _param:| _param:\\n mcp_version: \"$targetMcpVersion\"|' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml")
Sergey382581a2018-10-25 12:22:26 +0400837 salt.cmdRun(venvPepper, 'I@salt:master', "grep '^- system.defaults\$' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +
Denis Egorenkofcbc6b82019-01-15 11:47:16 +0400838 "sed -i 's|^classes:|classes:\\n- system.defaults|' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml")
Ivan Berezovskiy5113c7c2019-04-23 14:32:02 +0400839 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300840 "grep -r -l 'docker_image_jenkins: .*' cicd | xargs --no-run-if-empty sed -i 's|\\s*docker_image_jenkins: .*||g'")
Ivan Berezovskiy5113c7c2019-04-23 14:32:02 +0400841 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300842 "grep -r -l 'docker_image_jenkins_slave: .*' cicd | xargs --no-run-if-empty sed -i 's|\\s*docker_image_jenkins_slave: .*||g'")
Sergey382581a2018-10-25 12:22:26 +0400843 common.infoMsg("The following changes were made to the cluster model and will be commited. " +
azvyagintsev6dac3802018-11-09 15:11:30 +0200844 "Please consider if you want to push them to the remote repository or not. You have to do this manually when the run is finished.")
Sergey382581a2018-10-25 12:22:26 +0400845 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && git diff")
846 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && git status && " +
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200847 "git add -u && git commit --allow-empty -m 'Cluster model update to the release $targetMcpVersion on $dateTime'")
Sergey382581a2018-10-25 12:22:26 +0400848 }
Denis Egorenkod13ad132018-12-17 16:56:02 +0400849 try {
850 common.infoMsg('Perform: UPDATE Salt Formulas')
Ivan Berezovskiyc4dc5012020-03-19 14:03:43 +0400851 fullRefreshOneByOne(venvPepper, allMinions)
Denis Egorenkoea0ba7d2019-07-04 14:16:40 +0400852 salt.enforceState(venvPepper, 'I@salt:master', 'linux.system.repo', true, true, null, false, 60, 2)
Denis Egorenkod13ad132018-12-17 16:56:02 +0400853 def saltEnv = salt.getPillar(venvPepper, 'I@salt:master', "_param:salt_master_base_environment").get("return")[0].values()[0]
azvyagintsevbc359d42019-04-10 19:53:22 +0300854 salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'state.sls_id', ["salt_master_${saltEnv}_pkg_formulas", 'salt.master.env'])
Ivan Berezovskiyc4dc5012020-03-19 14:03:43 +0400855 fullRefreshOneByOne(venvPepper, allMinions)
Denis Egorenkod13ad132018-12-17 16:56:02 +0400856 } catch (Exception updateErr) {
857 common.warningMsg(updateErr)
858 common.warningMsg('Failed to update Salt Formulas repos/packages. Check current available documentation on https://docs.mirantis.com/mcp/latest/, how to update packages.')
859 input message: 'Continue anyway?'
860 }
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400861 if (applyWorkarounds) {
862 wa29352(cluster_name)
863 wa29155(computeMinions, cluster_name)
864 }
Denis Egorenkod13ad132018-12-17 16:56:02 +0400865
866 try {
867 common.infoMsg('Perform: UPDATE Reclass package')
868 salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'pkg.install', ["reclass"])
869 } catch (Exception updateErr) {
870 common.warningMsg(updateErr)
871 common.warningMsg('Failed to update Reclass package. Check current available documentation on https://docs.mirantis.com/mcp/latest/, how to update packages.')
872 input message: 'Continue anyway?'
873 }
874
875 salt.fullRefresh(venvPepper, 'I@salt:master')
Denis Egorenkoea0ba7d2019-07-04 14:16:40 +0400876 salt.enforceState(venvPepper, 'I@salt:master', 'reclass.storage', true, true, null, false, 60, 2)
Ivan Berezovskiyf5c911b2019-12-03 18:49:09 +0400877 salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/${cluster_name} && git status && " +
878 "git add -u && git commit --allow-empty -m 'Reclass nodes update to the release ${targetMcpVersion} on ${common.getDatetime()}'")
Denis Egorenkod13ad132018-12-17 16:56:02 +0400879 try {
Denis Egorenkoea0ba7d2019-07-04 14:16:40 +0400880 salt.enforceState(venvPepper, 'I@salt:master', 'reclass', true, true, null, false, 60, 2)
Denis Egorenkod13ad132018-12-17 16:56:02 +0400881 }
882 catch (Exception ex) {
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300883 common.errorMsg(ex.toString())
884 error('Reclass fails rendering. Pay attention to your cluster model.')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400885 }
886
Ivan Berezovskiyc4dc5012020-03-19 14:03:43 +0400887 fullRefreshOneByOne(venvPepper, allMinions)
Denis Egorenkod13ad132018-12-17 16:56:02 +0400888 try {
889 salt.cmdRun(venvPepper, 'I@salt:master', "reclass-salt --top")
890 }
891 catch (Exception ex) {
azvyagintsevbc359d42019-04-10 19:53:22 +0300892
893 error('Reclass fails rendering. Pay attention to your cluster model.' +
894 'ErrorMessage:' + ex.toString())
Denis Egorenkod13ad132018-12-17 16:56:02 +0400895 }
896
azvyagintsevbc359d42019-04-10 19:53:22 +0300897 common.infoMsg('Perform: archiveReclassInventory AFTER upgrade')
Denis Egorenkod13ad132018-12-17 16:56:02 +0400898 archiveReclassInventory(inventoryAfterFilename)
899
900 sh "diff -u $inventoryBeforeFilename $inventoryAfterFilename > reclass-inventory-diff.out || true"
901 archiveArtifacts artifacts: "reclass-inventory-diff.out"
902
903 validateReclassModel(minions, 'after')
904 archiveReclassModelChanges(minions)
Jakub Josefa63f9862018-01-11 17:58:38 +0100905 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100906
Denis Egorenkod13ad132018-12-17 16:56:02 +0400907 if (updateLocalRepos) {
azvyagintsev6dac3802018-11-09 15:11:30 +0200908 stage("Update local repos") {
Jakub Josefa63f9862018-01-11 17:58:38 +0100909 common.infoMsg("Updating local repositories")
Sam Stoelingaaab79702018-04-09 18:49:39 -0700910
Sergey1b81c062018-10-16 17:10:21 +0400911 def engine = salt.getPillar(venvPepper, 'I@aptly:publisher', "aptly:publisher:source:engine")
Sam Stoelingaaab79702018-04-09 18:49:39 -0700912 runningOnDocker = engine.get("return")[0].containsValue("docker")
913
914 if (runningOnDocker) {
915 common.infoMsg("Aptly is running as Docker container")
azvyagintsev6dac3802018-11-09 15:11:30 +0200916 } else {
Sam Stoelingaaab79702018-04-09 18:49:39 -0700917 common.infoMsg("Aptly isn't running as Docker container. Going to use aptly user for executing aptly commands")
918 }
919
azvyagintsev6dac3802018-11-09 15:11:30 +0200920 if (runningOnDocker) {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200921 salt.cmdRun(venvPepper, 'I@aptly:publisher', "aptly mirror list --raw | grep -E '*' | xargs --no-run-if-empty -n 1 aptly mirror drop -force", true, null, true)
azvyagintsev6dac3802018-11-09 15:11:30 +0200922 } else {
azvyagintsevf3bf77a2018-11-15 19:30:04 +0200923 salt.cmdRun(venvPepper, 'I@aptly:publisher', "aptly mirror list --raw | grep -E '*' | xargs --no-run-if-empty -n 1 aptly mirror drop -force", true, null, true, ['runas=aptly'])
Richard Felkl5f7fdaf2018-02-15 15:38:31 +0100924 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100925
Sergey1b81c062018-10-16 17:10:21 +0400926 salt.enforceState(venvPepper, 'I@aptly:publisher', 'aptly', true)
Richard Felkl26cae4d2017-12-19 00:19:16 +0100927
azvyagintsev6dac3802018-11-09 15:11:30 +0200928 if (runningOnDocker) {
Sergey1b81c062018-10-16 17:10:21 +0400929 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv"], null, true)
930 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-frv -u http://10.99.0.1:8080"], null, true)
azvyagintsev6dac3802018-11-09 15:11:30 +0200931 } else {
Sergey1b81c062018-10-16 17:10:21 +0400932 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv", 'runas=aptly'], null, true)
933 salt.runSaltProcessStep(venvPepper, 'I@aptly:publisher', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-afrv", 'runas=aptly'], null, true)
Richard Felkl5f7fdaf2018-02-15 15:38:31 +0100934 }
Richard Felkl79d7df12018-01-05 16:40:11 +0100935
Sergey1b81c062018-10-16 17:10:21 +0400936 salt.enforceState(venvPepper, 'I@aptly:publisher', 'docker.client.registry', true)
Richard Felkl26cae4d2017-12-19 00:19:16 +0100937
Sergey1b81c062018-10-16 17:10:21 +0400938 salt.enforceState(venvPepper, 'I@aptly:publisher', 'debmirror', true)
Richard Felkl26cae4d2017-12-19 00:19:16 +0100939
Sergey1b81c062018-10-16 17:10:21 +0400940 salt.enforceState(venvPepper, 'I@aptly:publisher', 'git.server', true)
Richard Felkl5f7fdaf2018-02-15 15:38:31 +0100941
Sergey1b81c062018-10-16 17:10:21 +0400942 salt.enforceState(venvPepper, 'I@aptly:publisher', 'linux.system.file', true)
Jakub Josefa63f9862018-01-11 17:58:38 +0100943 }
944 }
Richard Felkl26cae4d2017-12-19 00:19:16 +0100945
Vladimir Khlyunev20d63672021-06-18 13:43:08 +0400946 stage('Update Drivetrain: Part 1') {
Vladimir Khlyunev2dd5bfd2021-03-19 18:45:54 +0400947 salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.repo', true, true, batchSize, false, 60, 2)
Oleksii Molchanov68eb72c2020-05-15 13:33:45 +0300948 salt.enforceState(venvPepper, '*', 'linux.system.package', true, true, batchSize, false, 60, 2)
949
Denis Egorenkod13ad132018-12-17 16:56:02 +0400950 if (upgradeSaltStack) {
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300951 updateSaltStack('I@salt:master', '["salt-master", "salt-common", "salt-api", "salt-minion"]')
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300952 updateSaltStack('I@salt:minion and not I@salt:master', '["salt-minion"]')
Richard Felkl0e80d892018-06-20 13:44:54 +0200953 }
954
Denis Egorenkod13ad132018-12-17 16:56:02 +0400955 if (updatePipelines) {
azvyagintsevbc359d42019-04-10 19:53:22 +0300956 common.infoMsg('Perform: UPDATE git repos')
957 triggerMirrorJob('git-mirror-downstream-mk-pipelines', reclassSystemBranch)
958 triggerMirrorJob('git-mirror-downstream-pipeline-library', reclassSystemBranch)
Richard Felkl970e0082018-06-12 18:00:51 +0200959 }
960
Denis Egorenko7a4cf032019-04-25 20:48:26 +0400961 // update minions certs
Denis Egorenkoaf3a2202019-04-26 15:13:16 +0400962 // call for `salt.minion.ca` state on related nodes to make sure
963 // mine was updated with required data after salt-minion/salt-master restart salt:minion:ca
Denis Egorenkoca6aeca2019-08-19 19:41:27 +0400964 salt.enforceState(venvPepper, 'I@salt:minion:ca', 'salt.minion.ca', true, true, batchSize, false, 60, 2)
965 salt.enforceState(venvPepper, 'I@salt:minion', 'salt.minion.cert', true, true, batchSize, false, 60, 2)
Denis Egorenko7a4cf032019-04-25 20:48:26 +0400966
Ann Kamyshnikovae8910112019-05-06 12:56:38 +0400967 // run `salt.minion` to refresh all minion configs (for example _keystone.conf)
Ivan Berezovskiy22390f62020-02-17 17:08:47 +0400968 salt.enforceState(venvPepper, 'I@salt:minion', 'salt.minion', true, true, batchSize, false, 60, 2)
azvyagintsevbc359d42019-04-10 19:53:22 +0300969 // Retry needed only for rare race-condition in user appearance
970 common.infoMsg('Perform: updating users and keys')
Denis Egorenkoca6aeca2019-08-19 19:41:27 +0400971 salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.user', true, true, batchSize, false, 60, 2)
azvyagintsevbc359d42019-04-10 19:53:22 +0300972 common.infoMsg('Perform: updating openssh')
Denis Egorenkoca6aeca2019-08-19 19:41:27 +0400973 salt.enforceState(venvPepper, 'I@linux:system', 'openssh', true, true, batchSize, false, 60, 2)
Denis Egorenko65cf2a42019-01-15 11:58:12 +0400974
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300975 // apply salt API TLS if needed
976 def nginxAtMaster = salt.getPillar(venvPepper, 'I@salt:master', 'nginx:server:enabled').get('return')[0].values()[0]
977 if (nginxAtMaster.toString().toLowerCase() == 'true') {
Denis Egorenkoea0ba7d2019-07-04 14:16:40 +0400978 salt.enforceState(venvPepper, 'I@salt:master', 'nginx', true, true, null, false, 60, 2)
azvyagintsev97fd5ae2019-06-03 19:09:43 +0300979 }
Ivan Berezovskiya766d432019-12-11 15:31:10 +0400980 }
981 }
982 catch (Throwable e) {
983 // If there was an error or exception thrown, the build failed
984 currentBuild.result = "FAILURE"
985 throw e
986 }
987 }
Denis Egorenko96cc57d2020-01-24 19:47:05 +0400988 stage('Upgrade OS') {
989 if (packageUpgradeMode) {
Vladimir Khlyunev20d63672021-06-18 13:43:08 +0400990 def cidNodes = []
991 node('python') {
992 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
993 cidNodes = salt.getMinions(venvPepper, 'I@_param:drivetrain_role:cicd')
994 }
Denis Egorenko96cc57d2020-01-24 19:47:05 +0400995 def debian = new com.mirantis.mk.Debian()
996 def statusFile = '/tmp/rebooted_during_upgrade'
997 for(cidNode in cidNodes) {
998 node('python') {
999 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
1000 // cmd.run async to prevent connection close in case of slave shutdown, give 5 seconds to handle request response
1001 salt.cmdRun(venvPepper, "I@salt:master", "salt -C '${cidNode}' cmd.run 'sleep 5; touch ${statusFile}; salt-call service.stop docker' --async")
1002 }
1003 sleep(30)
1004 node('python') {
1005 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
1006 debian.osUpgradeNode(venvPepper, cidNode, packageUpgradeMode, false, 60)
1007 salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': cidNode, wait: 60, timeout: 10])
1008 if (salt.runSaltProcessStep(venvPepper, cidNode, 'file.file_exists', [statusFile], null, true, 5)['return'][0].values()[0].toBoolean()) {
1009 salt.cmdRun(venvPepper, "I@salt:master", "salt -C '${cidNode}' cmd.run 'rm ${statusFile} && salt-call service.start docker'") // in case if node was not rebooted
1010 sleep(10)
1011 }
1012 checkCICDDocker()
1013 }
1014 }
1015 } else {
1016 common.infoMsg('Upgrade OS skipped...')
1017 }
1018 }
Vladimir Khlyunev20d63672021-06-18 13:43:08 +04001019
1020 node('python') {
1021 stage('Update Drivetrain: Part 2') {
1022 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
1023 // Gerrit 2019.2.0 (2.13.6) version has wrong file name for download-commands plugin and was not loaded, let's remove if still there before upgrade
1024 def gerritGlusterPath = salt.getPillar(venvPepper, 'I@gerrit:client', 'glusterfs:client:volumes:gerrit:path').get('return')[0].values()[0]
1025 def wrongPluginJarName = "${gerritGlusterPath}/plugins/project-download-commands.jar"
1026 salt.cmdRun(venvPepper, 'I@gerrit:client', "test -f ${wrongPluginJarName} && rm ${wrongPluginJarName} || true")
1027
1028 salt.enforceStateWithTest(venvPepper, 'I@jenkins:client:security and not I@salt:master', 'jenkins.client.security', "", true, true, null, true, 60, 2)
1029 salt.enforceStateWithTest(venvPepper, 'I@jenkins:client and I@docker:client:images and not I@salt:master', 'docker.client.images', "", true, true, null, true, 60, 2)
1030 salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and I@docker:client and not I@salt:master' state.sls docker.client --async")
1031 }
1032 }
1033 // docker.client state may trigger change of jenkins master or jenkins slave services,
1034 // so we need wait for slave to reconnect and continue pipeline
1035 sleep(180)
1036 node('python') {
1037 stage('Update Drivetrain: Part 3') {
1038 python.setupPepperVirtualenv(venvPepper, saltMastURL, saltMastCreds)
1039 checkCICDDocker()
1040
1041 // update Nginx proxy settings for Jenkins/Gerrit if needed
1042 if (salt.testTarget(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit')) {
1043 salt.enforceState(venvPepper, 'I@nginx:server:site:nginx_proxy_jenkins and I@nginx:server:site:nginx_proxy_gerrit', 'nginx.server', true, true, null, false, 60, 2)
1044 }
1045 // Apply changes for HaProxy on CI/CD nodes
1046 salt.enforceState(venvPepper, 'I@keepalived:cluster:instance:cicd_control_vip and I@haproxy:proxy', 'haproxy.proxy', true)
1047 salt.upgradePackageAndRestartSaltMinion(venvPepper, 'I@jenkins:client and not I@salt:master', 'python-jenkins')
1048 salt.cmdRun(venvPepper, "I@salt:master", "salt -C 'I@jenkins:client and not I@salt:master' state.sls jenkins.client --async")
1049
1050 common.warningMsg("Jenkins update started in background in order to handle plugin post-install issues.")
1051 common.warningMsg("Please wait until it finished. Jenkins could be restarted during this procedure.")
1052 common.warningMsg("You can monitor job progress by running 'salt-run jobs.active' on master node")
1053 common.warningMsg("For ensuring that upgrade is done and there are no errors you can run the following command on salt master node")
1054 common.warningMsg("salt-run jobs.lookup_jid %salt_job_id%")
1055 common.warningMsg("Salt job ID could be found in the log above.")
1056 }
1057 }
Jakub Josef2c21c6c2018-02-08 18:51:42 +01001058}