Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mk / mk-pipelines / 2eb6005a5dc7cca63fc5acfe19d9ae15cf267c1a / . / openstack-control-upgrade.groovy
blob: 7dd2ea1ae6bb8800b84a4c384a84cff12fb7422e [file] [log] [blame]
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]1/**
2 * Update packages on given nodes
3 *
4 * Expected parameters:
5 * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
6 * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]7 * STAGE_TEST_UPGRADE Run test upgrade stage (bool)
8 * STAGE_REAL_UPGRADE Run real upgrade stage (bool)
9 * STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]10 *
11**/
12
13def common = new com.mirantis.mk.Common()
14def salt = new com.mirantis.mk.Salt()
15
16
17def saltMaster
18
19timestamps {
20 node() {
21
22 stage('Connect to Salt API') {
23 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
24 }
25
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]26 if (STAGE_TEST_UPGRADE.toBoolean() == true) {
27 stage('Test upgrade') {
28
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]29
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]30 try {
31 salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
32 } catch (Exception e) {
Jiri Broulik34674322017-05-25 19:10:57 +0200[diff] [blame]33 common.warningMsg("Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]34 }
35
Jiri Broulik34674322017-05-25 19:10:57 +0200[diff] [blame]36 try {
37 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
38 } catch (Exception e) {
39 common.warningMsg("No response from some minions. We should continue to run")
40 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]41
Jiri Broulik34674322017-05-25 19:10:57 +0200[diff] [blame]42 try {
43 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
44 } catch (Exception e) {
45 common.warningMsg("No response from some minions. We should continue to run")
46 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]47
48 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
49 def domain = _pillar['return'][0].values()[0].values()[0]
50 print(_pillar)
51 print(domain)
52
53 // read backupninja variable
54 _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
55 def backupninja_backup_host = _pillar['return'][0].values()[0]
56 print(_pillar)
57 print(backupninja_backup_host)
58
59 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
60 def kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]61 print(_pillar)
62 print(kvm01)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]63
64 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
65 def upgNodeProvider = _pillar['return'][0].values()[0]
66 print(_pillar)
67 print(upgNodeProvider)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]68
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]69
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]70 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
71 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]72
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]73
74 try {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]75 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]76 } catch (Exception e) {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]77 common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]78 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]79
80
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]81 // salt 'kvm02*' state.sls salt.control
82 salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]83
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]84 sleep(60)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]85
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]86 // salt '*' saltutil.refresh_pillar
87 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
88 // salt '*' saltutil.sync_all
89 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]90
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]91 // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
92 try {
93 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
94 } catch (Exception e) {
95 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
96 }
97 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]98
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]99 // salt "upg*" state.sls rabbitmq
100 salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
101 try {
102 salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
103 } catch (Exception e) {
104 common.warningMsg('salt-minion was restarted. We should continue to run')
105 }
106 try {
107 salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
108 } catch (Exception e) {
109 common.warningMsg('salt-minion was restarted. We should continue to run')
110 }
111 // salt '*' state.apply salt.minion.grains
112 //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
113 // salt -C 'I@backupninja:server' state.sls backupninja
114 salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
115 // salt -C 'I@backupninja:client' state.sls backupninja
116 salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
117 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
118 salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
119 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
120 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
Jiri Broulik2be7f672017-05-25 11:07:06 +0200[diff] [blame]121 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]122
123
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]124 def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
125 if(databases && databases != ""){
126 def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
127 for( i = 0; i < databasesList.size(); i++){
128 if(databasesList[i].toLowerCase().contains('upgrade')){
129 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
130 common.warningMsg("removing database ${databasesList[i]}")
131 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
132 }
133 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]134 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]135 }else{
136 common.errorMsg("No _upgrade databases were returned")
137 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]138
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]139 try {
140 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
141 } catch (Exception e) {
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]142 common.warningMsg('Restarting Apache2')
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]143 salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]144 }
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]145 try {
146 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
147 } catch (Exception e) {
148 common.warningMsg('running keystone.client state again')
149 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
150 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]151 try {
152 salt.enforceState(saltMaster, 'upg*', 'glance')
153 } catch (Exception e) {
154 common.warningMsg('running glance state again')
155 salt.enforceState(saltMaster, 'upg*', 'glance')
156 }
157 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
158 try {
159 salt.enforceState(saltMaster, 'upg*', 'nova')
160 } catch (Exception e) {
161 common.warningMsg('running nova state again')
162 salt.enforceState(saltMaster, 'upg*', 'nova')
163 }
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]164 // run nova state again as sometimes nova does not enforce itself for some reason
165 try {
166 salt.enforceState(saltMaster, 'upg*', 'nova')
167 } catch (Exception e) {
168 common.warningMsg('running nova state again')
169 salt.enforceState(saltMaster, 'upg*', 'nova')
170 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]171 try {
172 salt.enforceState(saltMaster, 'upg*', 'cinder')
173 } catch (Exception e) {
174 common.warningMsg('running cinder state again')
175 salt.enforceState(saltMaster, 'upg*', 'cinder')
176 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]177 try {
178 salt.enforceState(saltMaster, 'upg*', 'neutron')
179 } catch (Exception e) {
180 common.warningMsg('running neutron state again')
181 salt.enforceState(saltMaster, 'upg*', 'neutron')
182 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]183 try {
184 salt.enforceState(saltMaster, 'upg*', 'heat')
185 } catch (Exception e) {
186 common.warningMsg('running heat state again')
187 salt.enforceState(saltMaster, 'upg*', 'heat')
188 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]189 salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]190
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]191 if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
192 stage('Ask for manual confirmation') {
193 input message: "Do you want to continue with upgrade?"
194 }
195 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]196 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]197 }
198
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]199 if (STAGE_REAL_UPGRADE.toBoolean() == true) {
200 stage('Real upgrade') {
201 // # actual upgrade
202
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]203 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
204 domain = _pillar['return'][0].values()[0].values()[0]
205 print(_pillar)
206 print(domain)
207
208 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
209 kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]210 print(_pillar)
211 print(kvm01)
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]212
213 def errorOccured = false
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]214
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]215 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
216 def ctl01NodeProvider = _pillar['return'][0].values()[0]
217
218 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
219 def ctl02NodeProvider = _pillar['return'][0].values()[0]
220
221 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
222 def ctl03NodeProvider = _pillar['return'][0].values()[0]
223
224 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
225 def prx01NodeProvider = _pillar['return'][0].values()[0]
226
227 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
228 def prx02NodeProvider = _pillar['return'][0].values()[0]
229
230
231 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
232 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
233 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
234 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
235 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
236
237
238 try {
239 salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
240 } catch (Exception e) {
241 common.warningMsg('File already exists')
242 }
243 try {
244 salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
245 } catch (Exception e) {
246 common.warningMsg('File already exists')
247 }
248 try {
249 salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
250 } catch (Exception e) {
251 common.warningMsg('File already exists')
252 }
253 try {
254 salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
255 } catch (Exception e) {
256 common.warningMsg('File already exists')
257 }
258 try {
259 salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
260 } catch (Exception e) {
261 common.warningMsg('File already exists')
262 }
263
264
265 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
266 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
267 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
268 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
269 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
270
271
272 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
Jiri Broulik66529aa2017-05-25 13:54:53 +0200[diff] [blame]273 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]274
275 try {
276 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
277 } catch (Exception e) {
278 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
279 }
280
281 // salt 'kvm*' state.sls salt.control
282 salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
283
284 sleep(60)
285
286 // salt '*' saltutil.refresh_pillar
287 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
288 // salt '*' saltutil.sync_all
289 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
290
291 try {
292 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
293 } catch (Exception e) {
294 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
295 }
296 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
297
298 // salt 'ctl*' state.sls keepalived
299 // salt 'ctl*' state.sls haproxy
300 salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
301 // salt 'ctl*' service.restart rsyslog
302 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
303 // salt "ctl*" state.sls memcached
304 // salt "ctl*" state.sls keystone.server
305 try {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]306 try {
307 salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
308 } catch (Exception e) {
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]309 common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]310 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]311 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
312 }
313 // salt 'ctl01*' state.sls keystone.client
314 try {
315 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
316 } catch (Exception e) {
317 common.warningMsg('running keystone.client state again')
318 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
319 }
320 try {
321 salt.enforceState(saltMaster, 'ctl*', 'glance')
322 } catch (Exception e) {
323 common.warningMsg('running glance state again')
324 salt.enforceState(saltMaster, 'ctl*', 'glance')
325 } // salt 'ctl*' state.sls glusterfs.client
326 salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
327 // salt 'ctl*' state.sls keystone.server
328 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
329 // salt 'ctl*' state.sls nova
330 try {
331 salt.enforceState(saltMaster, 'ctl*', 'nova')
332 } catch (Exception e) {
333 common.warningMsg('running nova state again')
334 salt.enforceState(saltMaster, 'ctl*', 'nova')
335 }
336 // salt 'ctl*' state.sls cinder
337 try {
338 salt.enforceState(saltMaster, 'ctl*', 'cinder')
339 } catch (Exception e) {
340 common.warningMsg('running cinder state again')
341 salt.enforceState(saltMaster, 'ctl*', 'cinder')
342 }
343 try {
344 salt.enforceState(saltMaster, 'ctl*', 'neutron')
345 } catch (Exception e) {
346 common.warningMsg('running neutron state again')
347 salt.enforceState(saltMaster, 'ctl*', 'neutron')
348 }
349 // salt 'ctl*' state.sls heat
350 try {
351 salt.enforceState(saltMaster, 'ctl*', 'heat')
352 } catch (Exception e) {
353 common.warningMsg('running heat state again')
354 salt.enforceState(saltMaster, 'ctl*', 'heat')
355 }
356
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]357 } catch (Exception e) {
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]358 errorOccured = true
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]359 common.warningMsg('Some states that require syncdb failed. Restoring production databases')
360 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
361 if(databases && databases != ""){
362 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
363 for( i = 0; i < databasesList.size(); i++){
364 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
365 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
366 common.warningMsg("removing database ${databasesList[i]}")
367 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
368 }
369 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]370 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]371 }else{
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]372 common.errorMsg("No none _upgrade databases were returned. You have to restore production databases before running the real control upgrade again. This is because database schema for some services already happened. To do that delete the production databases, remove none upgrade database files from /root/mysql/flags/ and run salt 'I@mysql:client' state.sls mysql.client on the salt-master node")
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]373 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]374 common.errorMsg("Stage Real control upgrade failed")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]375 }
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]376 if(!errorOccured){
377 // salt 'cmp*' cmd.run 'service nova-compute restart'
378 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]379
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]380 // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog - TODO: proč? už to jednou projelo
381 // salt 'ctl*' state.sls keepalived
382 // salt 'prx*' state.sls keepalived
383 salt.enforceState(saltMaster, 'prx*', 'keepalived')
384 // salt 'prx*' state.sls horizon
385 salt.enforceState(saltMaster, 'prx*', 'horizon')
386 // salt 'prx*' state.sls nginx
387 salt.enforceState(saltMaster, 'prx*', 'nginx')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]388
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]389 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
390 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]391 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]392
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]393 if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
394 stage('Ask for manual confirmation') {
395 input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
396 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]397 }
398 }
399
400 if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
401 stage('Rollback upgrade') {
402
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]403 stage('Ask for manual confirmation') {
404 input message: "Do you really want to continue with the rollback?"
405 }
406
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]407 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
408 domain = _pillar['return'][0].values()[0].values()[0]
409 print(_pillar)
410 print(domain)
411
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]412 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
413 kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]414 print(_pillar)
415 print(kvm01)
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]416
417 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
418 def ctl01NodeProvider = _pillar['return'][0].values()[0]
419
420 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
421 def ctl02NodeProvider = _pillar['return'][0].values()[0]
422
423 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
424 def ctl03NodeProvider = _pillar['return'][0].values()[0]
425
426 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
427 def prx01NodeProvider = _pillar['return'][0].values()[0]
428
429 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
430 def prx02NodeProvider = _pillar['return'][0].values()[0]
431
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]432 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
433 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
434 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
435 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
436 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
437
438 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
439 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
440 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
441 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
442 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
443
444 try {
445 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
446 } catch (Exception e) {
447 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
448 }
449
450 databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep -v \'upgrade\' | grep -v \'schema\' | awk \'/-/ {print \$2}\'')
451 if(databases && databases != ""){
452 databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
453 for( i = 0; i < databasesList.size(); i++){
454 if(!databasesList[i].toLowerCase().contains('upgrade') && !databasesList[i].toLowerCase().contains('command execution')){
455 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
456 common.warningMsg("removing database ${databasesList[i]}")
457 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
458 }
459 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]460 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]461 }else{
462 common.errorMsg("No none _upgrade databases were returned")
463 }
464
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]465 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
466 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
467 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
468 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
469 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
470
471 // salt 'cmp*' cmd.run 'service nova-compute restart'
472 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
473
474 sleep(60)
475
476 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
477 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]478 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]479 }
480}