Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mk / mk-pipelines / 69de13ae43619eed5a8e725b790507feaa486ad1 / . / openstack-control-upgrade.groovy
blob: fe39b1b74427a3ccd2ff529b5c5a797783bf12c4 [file] [log] [blame]
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]1/**
2 * Update packages on given nodes
3 *
4 * Expected parameters:
5 * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
6 * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]7 * STAGE_TEST_UPGRADE Run test upgrade stage (bool)
8 * STAGE_REAL_UPGRADE Run real upgrade stage (bool)
9 * STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]10 *
11**/
12
13def common = new com.mirantis.mk.Common()
14def salt = new com.mirantis.mk.Salt()
15
16
17def saltMaster
18
19timestamps {
20 node() {
21
22 stage('Connect to Salt API') {
23 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
24 }
25
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]26 if (STAGE_TEST_UPGRADE.toBoolean() == true) {
27 stage('Test upgrade') {
28
29 //salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
30
31 // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
32 // salt '*' saltutil.sync_all
33 // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
34
35
36 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
37 def domain = _pillar['return'][0].values()[0].values()[0]
38 print(_pillar)
39 print(domain)
40
41 // read backupninja variable
42 _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
43 def backupninja_backup_host = _pillar['return'][0].values()[0]
44 print(_pillar)
45 print(backupninja_backup_host)
46
47 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
48 def kvm01 = _pillar['return'][0].values()[0].values()[0]
49 def kvm03 = _pillar['return'][0].values()[2].values()[0]
50 def kvm02 = _pillar['return'][0].values()[1].values()[0]
51 print(_pillar)
52 print(kvm01)
53 print(kvm02)
54 print(kvm03)
55
56 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
57 def upgNodeProvider = _pillar['return'][0].values()[0]
58 print(_pillar)
59 print(upgNodeProvider)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]60
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]61
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]62 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
63 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]64
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]65
66 try {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]67 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]68 } catch (Exception e) {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]69 common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]70 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]71
72
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]73 // salt 'kvm02*' state.sls salt.control
74 salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]75
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]76 sleep(60)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]77
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]78 // salt '*' saltutil.refresh_pillar
79 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
80 // salt '*' saltutil.sync_all
81 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]82
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]83 // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
84 try {
85 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
86 } catch (Exception e) {
87 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
88 }
89 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]90
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]91 // salt "upg*" state.sls rabbitmq
92 salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
93 try {
94 salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
95 } catch (Exception e) {
96 common.warningMsg('salt-minion was restarted. We should continue to run')
97 }
98 try {
99 salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
100 } catch (Exception e) {
101 common.warningMsg('salt-minion was restarted. We should continue to run')
102 }
103 // salt '*' state.apply salt.minion.grains
104 //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
105 // salt -C 'I@backupninja:server' state.sls backupninja
106 salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
107 // salt -C 'I@backupninja:client' state.sls backupninja
108 salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
109 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
110 salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
111 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
112 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
113 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]114
115
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]116 def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
117 if(databases && databases != ""){
118 def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
119 for( i = 0; i < databasesList.size(); i++){
120 if(databasesList[i].toLowerCase().contains('upgrade')){
121 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
122 common.warningMsg("removing database ${databasesList[i]}")
123 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
124 }
125 }
126 }else{
127 common.errorMsg("No _upgrade databases were returned")
128 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]129
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]130 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]131
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]132 try {
133 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
134 } catch (Exception e) {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame^]135 common.warningMsg('Enforcing keystone.server state again')
136 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]137 }
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame^]138 try {
139 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
140 } catch (Exception e) {
141 common.warningMsg('running keystone.client state again')
142 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
143 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]144 try {
145 salt.enforceState(saltMaster, 'upg*', 'glance')
146 } catch (Exception e) {
147 common.warningMsg('running glance state again')
148 salt.enforceState(saltMaster, 'upg*', 'glance')
149 }
150 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
151 try {
152 salt.enforceState(saltMaster, 'upg*', 'nova')
153 } catch (Exception e) {
154 common.warningMsg('running nova state again')
155 salt.enforceState(saltMaster, 'upg*', 'nova')
156 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]157 try {
158 salt.enforceState(saltMaster, 'upg*', 'cinder')
159 } catch (Exception e) {
160 common.warningMsg('running cinder state again')
161 salt.enforceState(saltMaster, 'upg*', 'cinder')
162 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]163 try {
164 salt.enforceState(saltMaster, 'upg*', 'neutron')
165 } catch (Exception e) {
166 common.warningMsg('running neutron state again')
167 salt.enforceState(saltMaster, 'upg*', 'neutron')
168 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]169 try {
170 salt.enforceState(saltMaster, 'upg*', 'heat')
171 } catch (Exception e) {
172 common.warningMsg('running heat state again')
173 salt.enforceState(saltMaster, 'upg*', 'heat')
174 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]175 salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
176 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]177 }
178
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]179 if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
180 stage('Ask for manual confirmation') {
181 input message: "Do you want to continue with upgrade?"
182 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]183 }
184
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]185 if (STAGE_REAL_UPGRADE.toBoolean() == true) {
186 stage('Real upgrade') {
187 // # actual upgrade
188
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]189 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
190 domain = _pillar['return'][0].values()[0].values()[0]
191 print(_pillar)
192 print(domain)
193
194 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
195 kvm01 = _pillar['return'][0].values()[0].values()[0]
196 kvm03 = _pillar['return'][0].values()[2].values()[0]
197 kvm02 = _pillar['return'][0].values()[1].values()[0]
198 print(_pillar)
199 print(kvm01)
200 print(kvm02)
201 print(kvm03)
202
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]203 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
204 def ctl01NodeProvider = _pillar['return'][0].values()[0]
205
206 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
207 def ctl02NodeProvider = _pillar['return'][0].values()[0]
208
209 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
210 def ctl03NodeProvider = _pillar['return'][0].values()[0]
211
212 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
213 def prx01NodeProvider = _pillar['return'][0].values()[0]
214
215 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
216 def prx02NodeProvider = _pillar['return'][0].values()[0]
217
218
219 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
220 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
221 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
222 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
223 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
224
225
226 try {
227 salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
228 } catch (Exception e) {
229 common.warningMsg('File already exists')
230 }
231 try {
232 salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
233 } catch (Exception e) {
234 common.warningMsg('File already exists')
235 }
236 try {
237 salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
238 } catch (Exception e) {
239 common.warningMsg('File already exists')
240 }
241 try {
242 salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
243 } catch (Exception e) {
244 common.warningMsg('File already exists')
245 }
246 try {
247 salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
248 } catch (Exception e) {
249 common.warningMsg('File already exists')
250 }
251
252
253 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
254 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
255 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
256 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
257 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
258
259
260 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
261 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]262
263 try {
264 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
265 } catch (Exception e) {
266 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
267 }
268
269 // salt 'kvm*' state.sls salt.control
270 salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
271
272 sleep(60)
273
274 // salt '*' saltutil.refresh_pillar
275 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
276 // salt '*' saltutil.sync_all
277 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
278
279 try {
280 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
281 } catch (Exception e) {
282 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
283 }
284 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
285
286 // salt 'ctl*' state.sls keepalived
287 // salt 'ctl*' state.sls haproxy
288 salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
289 // salt 'ctl*' service.restart rsyslog
290 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
291 // salt "ctl*" state.sls memcached
292 // salt "ctl*" state.sls keystone.server
293 try {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame^]294 try {
295 salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
296 } catch (Exception e) {
297 common.warningMsg('Enforcing keystone.server state again')
298 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
299 }
300 // salt 'ctl01*' state.sls keystone.client
301 try {
302 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
303 } catch (Exception e) {
304 common.warningMsg('running keystone.client state again')
305 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
306 }
307 try {
308 salt.enforceState(saltMaster, 'ctl*', 'glance')
309 } catch (Exception e) {
310 common.warningMsg('running glance state again')
311 salt.enforceState(saltMaster, 'ctl*', 'glance')
312 } // salt 'ctl*' state.sls glusterfs.client
313 salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
314 // salt 'ctl*' state.sls keystone.server
315 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
316 // salt 'ctl*' state.sls nova
317 try {
318 salt.enforceState(saltMaster, 'ctl*', 'nova')
319 } catch (Exception e) {
320 common.warningMsg('running nova state again')
321 salt.enforceState(saltMaster, 'ctl*', 'nova')
322 }
323 // salt 'ctl*' state.sls cinder
324 try {
325 salt.enforceState(saltMaster, 'ctl*', 'cinder')
326 } catch (Exception e) {
327 common.warningMsg('running cinder state again')
328 salt.enforceState(saltMaster, 'ctl*', 'cinder')
329 }
330 try {
331 salt.enforceState(saltMaster, 'ctl*', 'neutron')
332 } catch (Exception e) {
333 common.warningMsg('running neutron state again')
334 salt.enforceState(saltMaster, 'ctl*', 'neutron')
335 }
336 // salt 'ctl*' state.sls heat
337 try {
338 salt.enforceState(saltMaster, 'ctl*', 'heat')
339 } catch (Exception e) {
340 common.warningMsg('running heat state again')
341 salt.enforceState(saltMaster, 'ctl*', 'heat')
342 }
343
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]344 } catch (Exception e) {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame^]345 common.warningMsg('Some states that require syncdb failed. Restoring production databases')
346 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
347 if(databases && databases != ""){
348 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
349 for( i = 0; i < databasesList.size(); i++){
350 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
351 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
352 common.warningMsg("removing database ${databasesList[i]}")
353 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
354 }
355 }
356 }else{
357 common.errorMsg("No none _upgrade databases were returned")
358 }
359
360 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]361 }
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame^]362
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]363 // salt 'cmp*' cmd.run 'service nova-compute restart'
364 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
365
366 // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
367 // salt 'ctl*' state.sls keepalived
368 // salt 'prx*' state.sls keepalived
369 salt.enforceState(saltMaster, 'prx*', 'keepalived')
370 // salt 'prx*' state.sls horizon
371 salt.enforceState(saltMaster, 'prx*', 'horizon')
372 // salt 'prx*' state.sls nginx
373 salt.enforceState(saltMaster, 'prx*', 'nginx')
374
375 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
376 }
377
378 }
379
380
381 if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
382 stage('Ask for manual confirmation') {
383 input message: "Please verify that control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
384 }
385 stage('Ask for manual confirmation') {
386 input message: "Do you really want to continue with the rollback?"
387 }
388 }
389
390 if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
391 stage('Rollback upgrade') {
392
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]393 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
394 domain = _pillar['return'][0].values()[0].values()[0]
395 print(_pillar)
396 print(domain)
397
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]398 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
399 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
400 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
401 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
402 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
403
404 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
405 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
406 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
407 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
408 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
409
410 try {
411 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
412 } catch (Exception e) {
413 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
414 }
415
416 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
417 if(databases && databases != ""){
418 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
419 for( i = 0; i < databasesList.size(); i++){
420 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
421 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
422 common.warningMsg("removing database ${databasesList[i]}")
423 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
424 }
425 }
426 }else{
427 common.errorMsg("No none _upgrade databases were returned")
428 }
429
430 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
431
432 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
433 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
434 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
435 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
436 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
437
438 // salt 'cmp*' cmd.run 'service nova-compute restart'
439 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
440
441 sleep(60)
442
443 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
444 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]445 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]446 }
447}