Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mk / mk-pipelines / 603627340864d915860106b5978daaae73ce8573 / . / openstack-control-upgrade.groovy
blob: 60fa1603628892cb30e1f80b0de1470240e7166f [file] [log] [blame]
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]1/**
2 * Update packages on given nodes
3 *
4 * Expected parameters:
5 * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
6 * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]7 * STAGE_TEST_UPGRADE Run test upgrade stage (bool)
8 * STAGE_REAL_UPGRADE Run real upgrade stage (bool)
9 * STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]10 *
11**/
12
13def common = new com.mirantis.mk.Common()
14def salt = new com.mirantis.mk.Salt()
15
16
17def saltMaster
18
19timestamps {
20 node() {
21
22 stage('Connect to Salt API') {
23 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
24 }
25
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]26 if (STAGE_TEST_UPGRADE.toBoolean() == true) {
27 stage('Test upgrade') {
28
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]29
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]30 try {
31 salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
32 } catch (Exception e) {
33 common.warningMsg(" Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
34 }
35
36 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
37 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]38
39
40 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
41 def domain = _pillar['return'][0].values()[0].values()[0]
42 print(_pillar)
43 print(domain)
44
45 // read backupninja variable
46 _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
47 def backupninja_backup_host = _pillar['return'][0].values()[0]
48 print(_pillar)
49 print(backupninja_backup_host)
50
51 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
52 def kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]53 print(_pillar)
54 print(kvm01)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]55
56 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
57 def upgNodeProvider = _pillar['return'][0].values()[0]
58 print(_pillar)
59 print(upgNodeProvider)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]60
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]61
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]62 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
63 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]64
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]65
66 try {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]67 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]68 } catch (Exception e) {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]69 common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]70 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]71
72
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]73 // salt 'kvm02*' state.sls salt.control
74 salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]75
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]76 sleep(60)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]77
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]78 // salt '*' saltutil.refresh_pillar
79 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
80 // salt '*' saltutil.sync_all
81 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]82
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]83 // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
84 try {
85 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
86 } catch (Exception e) {
87 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
88 }
89 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]90
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]91 // salt "upg*" state.sls rabbitmq
92 salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
93 try {
94 salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
95 } catch (Exception e) {
96 common.warningMsg('salt-minion was restarted. We should continue to run')
97 }
98 try {
99 salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
100 } catch (Exception e) {
101 common.warningMsg('salt-minion was restarted. We should continue to run')
102 }
103 // salt '*' state.apply salt.minion.grains
104 //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
105 // salt -C 'I@backupninja:server' state.sls backupninja
106 salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
107 // salt -C 'I@backupninja:client' state.sls backupninja
108 salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
109 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
110 salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
111 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
112 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
113 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]114
115
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]116 def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
117 if(databases && databases != ""){
118 def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
119 for( i = 0; i < databasesList.size(); i++){
120 if(databasesList[i].toLowerCase().contains('upgrade')){
121 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
122 common.warningMsg("removing database ${databasesList[i]}")
123 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
124 }
125 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]126 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]127 }else{
128 common.errorMsg("No _upgrade databases were returned")
129 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]130
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]131 try {
132 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
133 } catch (Exception e) {
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]134 common.warningMsg('Restarting Apache2')
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]135 salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]136 }
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]137 try {
138 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
139 } catch (Exception e) {
140 common.warningMsg('running keystone.client state again')
141 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
142 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]143 try {
144 salt.enforceState(saltMaster, 'upg*', 'glance')
145 } catch (Exception e) {
146 common.warningMsg('running glance state again')
147 salt.enforceState(saltMaster, 'upg*', 'glance')
148 }
149 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
150 try {
151 salt.enforceState(saltMaster, 'upg*', 'nova')
152 } catch (Exception e) {
153 common.warningMsg('running nova state again')
154 salt.enforceState(saltMaster, 'upg*', 'nova')
155 }
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]156 // run nova state again as sometimes nova does not enforce itself for some reason
157 try {
158 salt.enforceState(saltMaster, 'upg*', 'nova')
159 } catch (Exception e) {
160 common.warningMsg('running nova state again')
161 salt.enforceState(saltMaster, 'upg*', 'nova')
162 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]163 try {
164 salt.enforceState(saltMaster, 'upg*', 'cinder')
165 } catch (Exception e) {
166 common.warningMsg('running cinder state again')
167 salt.enforceState(saltMaster, 'upg*', 'cinder')
168 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]169 try {
170 salt.enforceState(saltMaster, 'upg*', 'neutron')
171 } catch (Exception e) {
172 common.warningMsg('running neutron state again')
173 salt.enforceState(saltMaster, 'upg*', 'neutron')
174 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]175 try {
176 salt.enforceState(saltMaster, 'upg*', 'heat')
177 } catch (Exception e) {
178 common.warningMsg('running heat state again')
179 salt.enforceState(saltMaster, 'upg*', 'heat')
180 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]181 salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]182
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]183 if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
184 stage('Ask for manual confirmation') {
185 input message: "Do you want to continue with upgrade?"
186 }
187 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]188 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]189 }
190
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]191 if (STAGE_REAL_UPGRADE.toBoolean() == true) {
192 stage('Real upgrade') {
193 // # actual upgrade
194
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]195 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
196 domain = _pillar['return'][0].values()[0].values()[0]
197 print(_pillar)
198 print(domain)
199
200 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
201 kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]202 print(_pillar)
203 print(kvm01)
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]204
205 def errorOccured = false
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]206
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]207 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
208 def ctl01NodeProvider = _pillar['return'][0].values()[0]
209
210 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
211 def ctl02NodeProvider = _pillar['return'][0].values()[0]
212
213 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
214 def ctl03NodeProvider = _pillar['return'][0].values()[0]
215
216 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
217 def prx01NodeProvider = _pillar['return'][0].values()[0]
218
219 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
220 def prx02NodeProvider = _pillar['return'][0].values()[0]
221
222
223 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
224 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
225 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
226 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
227 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
228
229
230 try {
231 salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
232 } catch (Exception e) {
233 common.warningMsg('File already exists')
234 }
235 try {
236 salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
237 } catch (Exception e) {
238 common.warningMsg('File already exists')
239 }
240 try {
241 salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
242 } catch (Exception e) {
243 common.warningMsg('File already exists')
244 }
245 try {
246 salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
247 } catch (Exception e) {
248 common.warningMsg('File already exists')
249 }
250 try {
251 salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
252 } catch (Exception e) {
253 common.warningMsg('File already exists')
254 }
255
256
257 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
258 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
259 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
260 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
261 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
262
263
264 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
265 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]266
267 try {
268 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
269 } catch (Exception e) {
270 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
271 }
272
273 // salt 'kvm*' state.sls salt.control
274 salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
275
276 sleep(60)
277
278 // salt '*' saltutil.refresh_pillar
279 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
280 // salt '*' saltutil.sync_all
281 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
282
283 try {
284 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
285 } catch (Exception e) {
286 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
287 }
288 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
289
290 // salt 'ctl*' state.sls keepalived
291 // salt 'ctl*' state.sls haproxy
292 salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
293 // salt 'ctl*' service.restart rsyslog
294 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
295 // salt "ctl*" state.sls memcached
296 // salt "ctl*" state.sls keystone.server
297 try {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]298 try {
299 salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
300 } catch (Exception e) {
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]301 common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]302 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]303 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
304 }
305 // salt 'ctl01*' state.sls keystone.client
306 try {
307 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
308 } catch (Exception e) {
309 common.warningMsg('running keystone.client state again')
310 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
311 }
312 try {
313 salt.enforceState(saltMaster, 'ctl*', 'glance')
314 } catch (Exception e) {
315 common.warningMsg('running glance state again')
316 salt.enforceState(saltMaster, 'ctl*', 'glance')
317 } // salt 'ctl*' state.sls glusterfs.client
318 salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
319 // salt 'ctl*' state.sls keystone.server
320 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
321 // salt 'ctl*' state.sls nova
322 try {
323 salt.enforceState(saltMaster, 'ctl*', 'nova')
324 } catch (Exception e) {
325 common.warningMsg('running nova state again')
326 salt.enforceState(saltMaster, 'ctl*', 'nova')
327 }
328 // salt 'ctl*' state.sls cinder
329 try {
330 salt.enforceState(saltMaster, 'ctl*', 'cinder')
331 } catch (Exception e) {
332 common.warningMsg('running cinder state again')
333 salt.enforceState(saltMaster, 'ctl*', 'cinder')
334 }
335 try {
336 salt.enforceState(saltMaster, 'ctl*', 'neutron')
337 } catch (Exception e) {
338 common.warningMsg('running neutron state again')
339 salt.enforceState(saltMaster, 'ctl*', 'neutron')
340 }
341 // salt 'ctl*' state.sls heat
342 try {
343 salt.enforceState(saltMaster, 'ctl*', 'heat')
344 } catch (Exception e) {
345 common.warningMsg('running heat state again')
346 salt.enforceState(saltMaster, 'ctl*', 'heat')
347 }
348
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]349 } catch (Exception e) {
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]350 errorOccured = true
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]351 common.warningMsg('Some states that require syncdb failed. Restoring production databases')
352 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
353 if(databases && databases != ""){
354 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
355 for( i = 0; i < databasesList.size(); i++){
356 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
357 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
358 common.warningMsg("removing database ${databasesList[i]}")
359 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
360 }
361 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]362 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]363 }else{
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]364 common.errorMsg("No none _upgrade databases were returned. You have to restore production databases before running the real control upgrade again. This is because database schema for some services already happened. To do that delete the production databases, remove none upgrade database files from /root/mysql/flags/ and run salt 'I@mysql:client' state.sls mysql.client on the salt-master node")
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]365 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]366 common.errorMsg("Stage Real control upgrade failed")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]367 }
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]368 if(!errorOccured){
369 // salt 'cmp*' cmd.run 'service nova-compute restart'
370 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]371
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]372 // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
373 // salt 'ctl*' state.sls keepalived
374 // salt 'prx*' state.sls keepalived
375 salt.enforceState(saltMaster, 'prx*', 'keepalived')
376 // salt 'prx*' state.sls horizon
377 salt.enforceState(saltMaster, 'prx*', 'horizon')
378 // salt 'prx*' state.sls nginx
379 salt.enforceState(saltMaster, 'prx*', 'nginx')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]380
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]381 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
382 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]383 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]384
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]385 if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
386 stage('Ask for manual confirmation') {
387 input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
388 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]389 }
390 }
391
392 if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
393 stage('Rollback upgrade') {
394
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]395 stage('Ask for manual confirmation') {
396 input message: "Do you really want to continue with the rollback?"
397 }
398
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]399 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
400 domain = _pillar['return'][0].values()[0].values()[0]
401 print(_pillar)
402 print(domain)
403
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]404 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
405 kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]406 print(_pillar)
407 print(kvm01)
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]408
409 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
410 def ctl01NodeProvider = _pillar['return'][0].values()[0]
411
412 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
413 def ctl02NodeProvider = _pillar['return'][0].values()[0]
414
415 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
416 def ctl03NodeProvider = _pillar['return'][0].values()[0]
417
418 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
419 def prx01NodeProvider = _pillar['return'][0].values()[0]
420
421 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
422 def prx02NodeProvider = _pillar['return'][0].values()[0]
423
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]424 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
425 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
426 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
427 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
428 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
429
430 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
431 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
432 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
433 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
434 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
435
436 try {
437 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
438 } catch (Exception e) {
439 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
440 }
441
442 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
443 if(databases && databases != ""){
444 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
445 for( i = 0; i < databasesList.size(); i++){
446 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
447 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
448 common.warningMsg("removing database ${databasesList[i]}")
449 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
450 }
451 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]452 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]453 }else{
454 common.errorMsg("No none _upgrade databases were returned")
455 }
456
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]457 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
458 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
459 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
460 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
461 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
462
463 // salt 'cmp*' cmd.run 'service nova-compute restart'
464 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
465
466 sleep(60)
467
468 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
469 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]470 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]471 }
472}