Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mk / mk-pipelines / 19d4c104a2f3a3def8109d03ef0580f4cdb61af5 / . / openstack-control-upgrade.groovy
blob: 9e6fd0169c4cf38930a3dab7ea75e036b94e6981 [file] [log] [blame]
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]1/**
2 * Update packages on given nodes
3 *
4 * Expected parameters:
5 * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
6 * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]7 * STAGE_TEST_UPGRADE Run test upgrade stage (bool)
8 * STAGE_REAL_UPGRADE Run real upgrade stage (bool)
9 * STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]10 *
11**/
12
13def common = new com.mirantis.mk.Common()
14def salt = new com.mirantis.mk.Salt()
15
16
17def saltMaster
18
19timestamps {
20 node() {
21
22 stage('Connect to Salt API') {
23 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
24 }
25
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]26 if (STAGE_TEST_UPGRADE.toBoolean() == true) {
27 stage('Test upgrade') {
28
29 //salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
30
31 // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
32 // salt '*' saltutil.sync_all
33 // salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
34
35
36 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
37 def domain = _pillar['return'][0].values()[0].values()[0]
38 print(_pillar)
39 print(domain)
40
41 // read backupninja variable
42 _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
43 def backupninja_backup_host = _pillar['return'][0].values()[0]
44 print(_pillar)
45 print(backupninja_backup_host)
46
47 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
48 def kvm01 = _pillar['return'][0].values()[0].values()[0]
49 def kvm03 = _pillar['return'][0].values()[2].values()[0]
50 def kvm02 = _pillar['return'][0].values()[1].values()[0]
51 print(_pillar)
52 print(kvm01)
53 print(kvm02)
54 print(kvm03)
55
56 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
57 def upgNodeProvider = _pillar['return'][0].values()[0]
58 print(_pillar)
59 print(upgNodeProvider)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]60
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]61
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]62 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
63 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]64
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]65
66 try {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]67 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]68 } catch (Exception e) {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]69 common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]70 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]71
72
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]73 // salt 'kvm02*' state.sls salt.control
74 salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]75
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]76 sleep(60)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]77
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]78 // salt '*' saltutil.refresh_pillar
79 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
80 // salt '*' saltutil.sync_all
81 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]82
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]83 // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
84 try {
85 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
86 } catch (Exception e) {
87 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
88 }
89 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]90
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]91 // salt "upg*" state.sls rabbitmq
92 salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
93 try {
94 salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
95 } catch (Exception e) {
96 common.warningMsg('salt-minion was restarted. We should continue to run')
97 }
98 try {
99 salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
100 } catch (Exception e) {
101 common.warningMsg('salt-minion was restarted. We should continue to run')
102 }
103 // salt '*' state.apply salt.minion.grains
104 //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
105 // salt -C 'I@backupninja:server' state.sls backupninja
106 salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
107 // salt -C 'I@backupninja:client' state.sls backupninja
108 salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
109 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
110 salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
111 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
112 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
113 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]114
115
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]116 def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
117 if(databases && databases != ""){
118 def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
119 for( i = 0; i < databasesList.size(); i++){
120 if(databasesList[i].toLowerCase().contains('upgrade')){
121 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
122 common.warningMsg("removing database ${databasesList[i]}")
123 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
124 }
125 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]126 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]127 }else{
128 common.errorMsg("No _upgrade databases were returned")
129 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]130
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]131 try {
132 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
133 } catch (Exception e) {
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]134 common.warningMsg('Reloading Apache2 and enforcing keystone.server state again')
135 salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]136 }
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]137 try {
138 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
139 } catch (Exception e) {
140 common.warningMsg('running keystone.client state again')
141 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
142 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]143 try {
144 salt.enforceState(saltMaster, 'upg*', 'glance')
145 } catch (Exception e) {
146 common.warningMsg('running glance state again')
147 salt.enforceState(saltMaster, 'upg*', 'glance')
148 }
149 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
150 try {
151 salt.enforceState(saltMaster, 'upg*', 'nova')
152 } catch (Exception e) {
153 common.warningMsg('running nova state again')
154 salt.enforceState(saltMaster, 'upg*', 'nova')
155 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]156 try {
157 salt.enforceState(saltMaster, 'upg*', 'cinder')
158 } catch (Exception e) {
159 common.warningMsg('running cinder state again')
160 salt.enforceState(saltMaster, 'upg*', 'cinder')
161 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]162 try {
163 salt.enforceState(saltMaster, 'upg*', 'neutron')
164 } catch (Exception e) {
165 common.warningMsg('running neutron state again')
166 salt.enforceState(saltMaster, 'upg*', 'neutron')
167 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]168 try {
169 salt.enforceState(saltMaster, 'upg*', 'heat')
170 } catch (Exception e) {
171 common.warningMsg('running heat state again')
172 salt.enforceState(saltMaster, 'upg*', 'heat')
173 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]174 salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
175 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]176 }
177
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]178 if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
179 stage('Ask for manual confirmation') {
180 input message: "Do you want to continue with upgrade?"
181 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]182 }
183
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]184 if (STAGE_REAL_UPGRADE.toBoolean() == true) {
185 stage('Real upgrade') {
186 // # actual upgrade
187
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]188 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
189 domain = _pillar['return'][0].values()[0].values()[0]
190 print(_pillar)
191 print(domain)
192
193 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
194 kvm01 = _pillar['return'][0].values()[0].values()[0]
195 kvm03 = _pillar['return'][0].values()[2].values()[0]
196 kvm02 = _pillar['return'][0].values()[1].values()[0]
197 print(_pillar)
198 print(kvm01)
199 print(kvm02)
200 print(kvm03)
201
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]202 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
203 def ctl01NodeProvider = _pillar['return'][0].values()[0]
204
205 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
206 def ctl02NodeProvider = _pillar['return'][0].values()[0]
207
208 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
209 def ctl03NodeProvider = _pillar['return'][0].values()[0]
210
211 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
212 def prx01NodeProvider = _pillar['return'][0].values()[0]
213
214 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
215 def prx02NodeProvider = _pillar['return'][0].values()[0]
216
217
218 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
219 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
220 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
221 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
222 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
223
224
225 try {
226 salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
227 } catch (Exception e) {
228 common.warningMsg('File already exists')
229 }
230 try {
231 salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
232 } catch (Exception e) {
233 common.warningMsg('File already exists')
234 }
235 try {
236 salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
237 } catch (Exception e) {
238 common.warningMsg('File already exists')
239 }
240 try {
241 salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
242 } catch (Exception e) {
243 common.warningMsg('File already exists')
244 }
245 try {
246 salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
247 } catch (Exception e) {
248 common.warningMsg('File already exists')
249 }
250
251
252 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
253 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
254 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
255 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
256 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
257
258
259 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
260 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]261
262 try {
263 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
264 } catch (Exception e) {
265 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
266 }
267
268 // salt 'kvm*' state.sls salt.control
269 salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
270
271 sleep(60)
272
273 // salt '*' saltutil.refresh_pillar
274 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
275 // salt '*' saltutil.sync_all
276 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
277
278 try {
279 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
280 } catch (Exception e) {
281 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
282 }
283 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
284
285 // salt 'ctl*' state.sls keepalived
286 // salt 'ctl*' state.sls haproxy
287 salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
288 // salt 'ctl*' service.restart rsyslog
289 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
290 // salt "ctl*" state.sls memcached
291 // salt "ctl*" state.sls keystone.server
292 try {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]293 try {
294 salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
295 } catch (Exception e) {
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]296 common.warningMsg('Reloading Apache2 and enforcing keystone.server state again')
297 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]298 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
299 }
300 // salt 'ctl01*' state.sls keystone.client
301 try {
302 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
303 } catch (Exception e) {
304 common.warningMsg('running keystone.client state again')
305 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
306 }
307 try {
308 salt.enforceState(saltMaster, 'ctl*', 'glance')
309 } catch (Exception e) {
310 common.warningMsg('running glance state again')
311 salt.enforceState(saltMaster, 'ctl*', 'glance')
312 } // salt 'ctl*' state.sls glusterfs.client
313 salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
314 // salt 'ctl*' state.sls keystone.server
315 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
316 // salt 'ctl*' state.sls nova
317 try {
318 salt.enforceState(saltMaster, 'ctl*', 'nova')
319 } catch (Exception e) {
320 common.warningMsg('running nova state again')
321 salt.enforceState(saltMaster, 'ctl*', 'nova')
322 }
323 // salt 'ctl*' state.sls cinder
324 try {
325 salt.enforceState(saltMaster, 'ctl*', 'cinder')
326 } catch (Exception e) {
327 common.warningMsg('running cinder state again')
328 salt.enforceState(saltMaster, 'ctl*', 'cinder')
329 }
330 try {
331 salt.enforceState(saltMaster, 'ctl*', 'neutron')
332 } catch (Exception e) {
333 common.warningMsg('running neutron state again')
334 salt.enforceState(saltMaster, 'ctl*', 'neutron')
335 }
336 // salt 'ctl*' state.sls heat
337 try {
338 salt.enforceState(saltMaster, 'ctl*', 'heat')
339 } catch (Exception e) {
340 common.warningMsg('running heat state again')
341 salt.enforceState(saltMaster, 'ctl*', 'heat')
342 }
343
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]344 } catch (Exception e) {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]345 common.warningMsg('Some states that require syncdb failed. Restoring production databases')
346 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
347 if(databases && databases != ""){
348 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
349 for( i = 0; i < databasesList.size(); i++){
350 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
351 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
352 common.warningMsg("removing database ${databasesList[i]}")
353 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
354 }
355 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]356 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]357 }else{
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]358 common.errorMsg("No none _upgrade databases were returned. You have to restore production databases before running the real control upgrade again. This is because database schema for some services already happened. To do that delete the production databases and run salt 'I@mysql:client' state.sls mysql.client on the salt-master node")
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]359 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]360 common.errorMsg("Stage Real control upgrade failed")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]361 }
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]362
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]363 // salt 'cmp*' cmd.run 'service nova-compute restart'
364 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
365
366 // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
367 // salt 'ctl*' state.sls keepalived
368 // salt 'prx*' state.sls keepalived
369 salt.enforceState(saltMaster, 'prx*', 'keepalived')
370 // salt 'prx*' state.sls horizon
371 salt.enforceState(saltMaster, 'prx*', 'horizon')
372 // salt 'prx*' state.sls nginx
373 salt.enforceState(saltMaster, 'prx*', 'nginx')
374
375 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
376 }
377
378 }
379
380
381 if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
382 stage('Ask for manual confirmation') {
383 input message: "Please verify that control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
384 }
385 stage('Ask for manual confirmation') {
386 input message: "Do you really want to continue with the rollback?"
387 }
388 }
389
390 if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
391 stage('Rollback upgrade') {
392
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]393 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
394 domain = _pillar['return'][0].values()[0].values()[0]
395 print(_pillar)
396 print(domain)
397
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]398 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
399 kvm01 = _pillar['return'][0].values()[0].values()[0]
400 kvm03 = _pillar['return'][0].values()[2].values()[0]
401 kvm02 = _pillar['return'][0].values()[1].values()[0]
402 print(_pillar)
403 print(kvm01)
404 print(kvm02)
405 print(kvm03)
406
407 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
408 def ctl01NodeProvider = _pillar['return'][0].values()[0]
409
410 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
411 def ctl02NodeProvider = _pillar['return'][0].values()[0]
412
413 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
414 def ctl03NodeProvider = _pillar['return'][0].values()[0]
415
416 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
417 def prx01NodeProvider = _pillar['return'][0].values()[0]
418
419 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
420 def prx02NodeProvider = _pillar['return'][0].values()[0]
421
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]422 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
423 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
424 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
425 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
426 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
427
428 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
429 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
430 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
431 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
432 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
433
434 try {
435 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
436 } catch (Exception e) {
437 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
438 }
439
440 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
441 if(databases && databases != ""){
442 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
443 for( i = 0; i < databasesList.size(); i++){
444 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
445 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
446 common.warningMsg("removing database ${databasesList[i]}")
447 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
448 }
449 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]450 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]451 }else{
452 common.errorMsg("No none _upgrade databases were returned")
453 }
454
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]455 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
456 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
457 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
458 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
459 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
460
461 // salt 'cmp*' cmd.run 'service nova-compute restart'
462 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
463
464 sleep(60)
465
466 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
467 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]468 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]469 }
470}