Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mk / mk-pipelines / 38b261d0b9ff760dd789ceefcf072ab4d8326e87 / . / openstack-control-upgrade.groovy
blob: c5219d8db9f130774f813806723c0b5cbb495421 [file] [log] [blame]
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]1/**
2 * Update packages on given nodes
3 *
4 * Expected parameters:
5 * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
6 * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]7 * STAGE_TEST_UPGRADE Run test upgrade stage (bool)
8 * STAGE_REAL_UPGRADE Run real upgrade stage (bool)
9 * STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]10 *
11**/
12
13def common = new com.mirantis.mk.Common()
14def salt = new com.mirantis.mk.Salt()
15
16
17def saltMaster
18
19timestamps {
20 node() {
21
22 stage('Connect to Salt API') {
23 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
24 }
25
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]26 if (STAGE_TEST_UPGRADE.toBoolean() == true) {
27 stage('Test upgrade') {
28
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]29
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]30 try {
31 salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
32 } catch (Exception e) {
Jiri Broulik34674322017-05-25 19:10:57 +0200[diff] [blame]33 common.warningMsg("Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]34 }
35
Jiri Broulik34674322017-05-25 19:10:57 +0200[diff] [blame]36 try {
37 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
38 } catch (Exception e) {
39 common.warningMsg("No response from some minions. We should continue to run")
40 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]41
Jiri Broulik34674322017-05-25 19:10:57 +0200[diff] [blame]42 try {
43 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
44 } catch (Exception e) {
45 common.warningMsg("No response from some minions. We should continue to run")
46 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]47
48 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
49 def domain = _pillar['return'][0].values()[0].values()[0]
50 print(_pillar)
51 print(domain)
52
53 // read backupninja variable
54 _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
55 def backupninja_backup_host = _pillar['return'][0].values()[0]
56 print(_pillar)
57 print(backupninja_backup_host)
58
59 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
60 def kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]61 print(_pillar)
62 print(kvm01)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]63
64 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
65 def upgNodeProvider = _pillar['return'][0].values()[0]
66 print(_pillar)
67 print(upgNodeProvider)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]68
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]69
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]70 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
71 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]72
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]73
74 try {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]75 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]76 } catch (Exception e) {
Jiri Broulikd6bc1382017-05-17 18:46:27 +0200[diff] [blame]77 common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]78 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]79
80
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]81 // salt 'kvm02*' state.sls salt.control
82 salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]83
Jiri Broulik9b73d6c2017-06-02 12:27:05 +0200[diff] [blame]84 sleep(70)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]85
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]86 // salt '*' saltutil.refresh_pillar
87 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
88 // salt '*' saltutil.sync_all
89 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]90
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]91 // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
92 try {
93 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
94 } catch (Exception e) {
95 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
96 }
97 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]98
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]99 // salt "upg*" state.sls rabbitmq
100 salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
101 try {
102 salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
103 } catch (Exception e) {
104 common.warningMsg('salt-minion was restarted. We should continue to run')
105 }
106 try {
107 salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
108 } catch (Exception e) {
109 common.warningMsg('salt-minion was restarted. We should continue to run')
110 }
111 // salt '*' state.apply salt.minion.grains
112 //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
113 // salt -C 'I@backupninja:server' state.sls backupninja
114 salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
115 // salt -C 'I@backupninja:client' state.sls backupninja
116 salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
117 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
Jiri Broulik90c6c7f2017-06-12 13:05:46 +0200[diff] [blame]118 try {
119 salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
120 } catch (Exception e) {
Jiri Broulik29cffe32017-06-12 14:15:19 +0200[diff] [blame]121 common.warningMsg('The ARP entry does not exist. We should continue to run.')
Jiri Broulik90c6c7f2017-06-12 13:05:46 +0200[diff] [blame]122 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]123 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
124 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
Jiri Broulik2be7f672017-05-25 11:07:06 +0200[diff] [blame]125 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]126
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]127 salt.enforceState(saltMaster, 'I@xtrabackup:server', 'xtrabackup')
128 salt.enforceState(saltMaster, 'I@xtrabackup:client', 'openssh.client')
129 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
130 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]131
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]132 def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
133 if(databases && databases != ""){
134 def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]135 for( i = 0; i < databasesList.size(); i++){
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]136 if(databasesList[i].toLowerCase().contains('upgrade')){
137 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
138 common.warningMsg("removing database ${databasesList[i]}")
139 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
140 }
141 }
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]142 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]143 }else{
144 common.errorMsg("No _upgrade databases were returned")
145 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]146
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]147 try {
148 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
Jiri Broulik29cffe32017-06-12 14:15:19 +0200[diff] [blame]149 salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]150 } catch (Exception e) {
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]151 common.warningMsg('Restarting Apache2')
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]152 salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]153 }
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]154 try {
155 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
156 } catch (Exception e) {
157 common.warningMsg('running keystone.client state again')
158 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
159 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]160 try {
161 salt.enforceState(saltMaster, 'upg*', 'glance')
162 } catch (Exception e) {
163 common.warningMsg('running glance state again')
164 salt.enforceState(saltMaster, 'upg*', 'glance')
165 }
166 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
167 try {
168 salt.enforceState(saltMaster, 'upg*', 'nova')
169 } catch (Exception e) {
170 common.warningMsg('running nova state again')
171 salt.enforceState(saltMaster, 'upg*', 'nova')
172 }
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]173 // run nova state again as sometimes nova does not enforce itself for some reason
174 try {
175 salt.enforceState(saltMaster, 'upg*', 'nova')
176 } catch (Exception e) {
177 common.warningMsg('running nova state again')
178 salt.enforceState(saltMaster, 'upg*', 'nova')
179 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]180 try {
181 salt.enforceState(saltMaster, 'upg*', 'cinder')
182 } catch (Exception e) {
183 common.warningMsg('running cinder state again')
184 salt.enforceState(saltMaster, 'upg*', 'cinder')
185 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]186 try {
187 salt.enforceState(saltMaster, 'upg*', 'neutron')
188 } catch (Exception e) {
189 common.warningMsg('running neutron state again')
190 salt.enforceState(saltMaster, 'upg*', 'neutron')
191 }
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]192 try {
193 salt.enforceState(saltMaster, 'upg*', 'heat')
194 } catch (Exception e) {
195 common.warningMsg('running heat state again')
196 salt.enforceState(saltMaster, 'upg*', 'heat')
197 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]198 salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]199
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]200 if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
201 stage('Ask for manual confirmation') {
202 input message: "Do you want to continue with upgrade?"
203 }
204 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]205 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]206 }
207
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]208 if (STAGE_REAL_UPGRADE.toBoolean() == true) {
209 stage('Real upgrade') {
210 // # actual upgrade
211
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]212 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
213 domain = _pillar['return'][0].values()[0].values()[0]
214 print(_pillar)
215 print(domain)
216
217 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
218 kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]219 print(_pillar)
220 print(kvm01)
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]221
222 def errorOccured = false
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]223
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]224 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
225 def ctl01NodeProvider = _pillar['return'][0].values()[0]
226
227 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
228 def ctl02NodeProvider = _pillar['return'][0].values()[0]
229
230 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
231 def ctl03NodeProvider = _pillar['return'][0].values()[0]
232
233 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
234 def prx01NodeProvider = _pillar['return'][0].values()[0]
235
236 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
237 def prx02NodeProvider = _pillar['return'][0].values()[0]
238
239
240 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
241 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
242 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
243 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
244 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
245
246
247 try {
248 salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
249 } catch (Exception e) {
250 common.warningMsg('File already exists')
251 }
252 try {
253 salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
254 } catch (Exception e) {
255 common.warningMsg('File already exists')
256 }
257 try {
258 salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
259 } catch (Exception e) {
260 common.warningMsg('File already exists')
261 }
262 try {
263 salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
264 } catch (Exception e) {
265 common.warningMsg('File already exists')
266 }
267 try {
268 salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
269 } catch (Exception e) {
270 common.warningMsg('File already exists')
271 }
272
273
274 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
275 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
276 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
277 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
278 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
279
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]280 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]281
282 try {
283 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
284 } catch (Exception e) {
285 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
286 }
287
288 // salt 'kvm*' state.sls salt.control
289 salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
290
Jiri Broulik9b73d6c2017-06-02 12:27:05 +0200[diff] [blame]291 sleep(70)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]292
293 // salt '*' saltutil.refresh_pillar
294 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
295 // salt '*' saltutil.sync_all
296 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
297
298 try {
299 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
300 } catch (Exception e) {
301 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
302 }
303 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
304
305 // salt 'ctl*' state.sls keepalived
306 // salt 'ctl*' state.sls haproxy
307 salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
308 // salt 'ctl*' service.restart rsyslog
309 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
310 // salt "ctl*" state.sls memcached
311 // salt "ctl*" state.sls keystone.server
312 try {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]313 try {
314 salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
Jiri Broulik29cffe32017-06-12 14:15:19 +0200[diff] [blame]315 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]316 } catch (Exception e) {
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]317 common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]318 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]319 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
320 }
321 // salt 'ctl01*' state.sls keystone.client
322 try {
323 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
324 } catch (Exception e) {
325 common.warningMsg('running keystone.client state again')
326 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
327 }
328 try {
329 salt.enforceState(saltMaster, 'ctl*', 'glance')
330 } catch (Exception e) {
331 common.warningMsg('running glance state again')
332 salt.enforceState(saltMaster, 'ctl*', 'glance')
333 } // salt 'ctl*' state.sls glusterfs.client
334 salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
335 // salt 'ctl*' state.sls keystone.server
336 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
337 // salt 'ctl*' state.sls nova
338 try {
339 salt.enforceState(saltMaster, 'ctl*', 'nova')
340 } catch (Exception e) {
341 common.warningMsg('running nova state again')
342 salt.enforceState(saltMaster, 'ctl*', 'nova')
343 }
344 // salt 'ctl*' state.sls cinder
345 try {
346 salt.enforceState(saltMaster, 'ctl*', 'cinder')
347 } catch (Exception e) {
348 common.warningMsg('running cinder state again')
349 salt.enforceState(saltMaster, 'ctl*', 'cinder')
350 }
351 try {
352 salt.enforceState(saltMaster, 'ctl*', 'neutron')
353 } catch (Exception e) {
354 common.warningMsg('running neutron state again')
355 salt.enforceState(saltMaster, 'ctl*', 'neutron')
356 }
357 // salt 'ctl*' state.sls heat
358 try {
359 salt.enforceState(saltMaster, 'ctl*', 'heat')
360 } catch (Exception e) {
361 common.warningMsg('running heat state again')
362 salt.enforceState(saltMaster, 'ctl*', 'heat')
363 }
364
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]365 } catch (Exception e) {
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]366 errorOccured = true
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]367 common.warningMsg('Some states that require syncdb failed. Restoring production databases')
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]368
369 // database restore section
370 try {
371 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
372 } catch (Exception er) {
373 common.warningMsg('Mysql service already stopped')
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]374 }
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]375 try {
376 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
377 } catch (Exception er) {
378 common.warningMsg('Mysql service already stopped')
379 }
380 try {
381 salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
382 } catch (Exception er) {
383 common.warningMsg('Files are not present')
384 }
385 try {
386 salt.cmdRun(saltMaster, 'I@galera:master', "mkdir /root/mysql/mysql.bak")
387 } catch (Exception er) {
388 common.warningMsg('Directory already exists')
389 }
390 try {
391 salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /root/mysql/mysql.bak/*")
392 } catch (Exception er) {
393 common.warningMsg('Directory already empty')
394 }
395 try {
396 salt.cmdRun(saltMaster, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
397 } catch (Exception er) {
398 common.warningMsg('Files were already moved')
399 }
400 try {
401 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
402 } catch (Exception er) {
403 common.warningMsg('File is not present')
404 }
405 salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
406 _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
407 backup_dir = _pillar['return'][0].values()[0]
408 if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
409 print(backup_dir)
410 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
411 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
412 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
413 sleep(5)
414 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
415 //
416
Jiri Broulik5a13a652017-05-18 14:33:32 +0200[diff] [blame]417 common.errorMsg("Stage Real control upgrade failed")
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]418 }
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]419 if(!errorOccured){
Jiri Broulik52a6b832017-06-16 13:05:08 +0200[diff] [blame]420
421 ceph = null
422
423 try {
424 ceph = salt.cmdRun(saltMaster, 'ctl*', "salt-call grains.item roles | grep ceph.client")
425
426 } catch (Exception er) {
427 common.infoMsg("Ceph is not used")
428 }
429
430 if(ceph != null) {
431 try {
432 salt.enforceState(saltMaster, 'ctl*', 'ceph.client')
433 } catch (Exception er) {
434 common.warningMsg("Ceph client state on controllers failed. Please fix it manually")
435 }
436 }
437
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]438 // salt 'cmp*' cmd.run 'service nova-compute restart'
439 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
Jiri Broulikf20c7672017-06-19 21:20:52 +0200[diff] [blame]440 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
441 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
442
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]443
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]444 // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]445 // salt 'ctl*' state.sls keepalived
446 // salt 'prx*' state.sls keepalived
447 salt.enforceState(saltMaster, 'prx*', 'keepalived')
448 // salt 'prx*' state.sls horizon
449 salt.enforceState(saltMaster, 'prx*', 'horizon')
450 // salt 'prx*' state.sls nginx
451 salt.enforceState(saltMaster, 'prx*', 'nginx')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]452
Jiri Broulik4df0c1f2017-06-23 11:29:29 +0200[diff] [blame]453 try {
454 salt.enforceHighstate(saltMaster, 'ctl*')
455 } catch (Exception er) {
456 common.errorMsg("Highstate was executed on controller nodes but something failed. Please check it and fix it accordingly.")
457 }
458
459 try {
460 salt.enforceHighstate(saltMaster, 'prx*')
461 } catch (Exception er) {
462 common.errorMsg("Highstate was executed on proxy nodes but something failed. Please check it and fix it accordingly.")
463 }
464
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]465 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
466 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]467 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]468
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]469 if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
470 stage('Ask for manual confirmation') {
471 input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
472 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]473 }
474 }
475
476 if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
477 stage('Rollback upgrade') {
478
Jiri Broulik08229c82017-05-22 23:35:32 +0200[diff] [blame]479 stage('Ask for manual confirmation') {
480 input message: "Do you really want to continue with the rollback?"
481 }
482
Jiri Broulike2f1e4a2017-05-17 23:59:52 +0200[diff] [blame]483 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
484 domain = _pillar['return'][0].values()[0].values()[0]
485 print(_pillar)
486 print(domain)
487
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]488 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
489 kvm01 = _pillar['return'][0].values()[0].values()[0]
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]490 print(_pillar)
491 print(kvm01)
Jiri Broulikd98364e2017-05-18 15:30:04 +0200[diff] [blame]492
493 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
494 def ctl01NodeProvider = _pillar['return'][0].values()[0]
495
496 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
497 def ctl02NodeProvider = _pillar['return'][0].values()[0]
498
499 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
500 def ctl03NodeProvider = _pillar['return'][0].values()[0]
501
502 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
503 def prx01NodeProvider = _pillar['return'][0].values()[0]
504
505 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
506 def prx02NodeProvider = _pillar['return'][0].values()[0]
507
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]508 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
509 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
510 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
511 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
512 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
513
514 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
515 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
516 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
517 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
518 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
519
520 try {
521 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
522 } catch (Exception e) {
523 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
524 }
525
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]526 // database restore section
527 try {
528 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
529 } catch (Exception e) {
530 common.warningMsg('Mysql service already stopped')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]531 }
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]532 try {
533 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
534 } catch (Exception e) {
535 common.warningMsg('Mysql service already stopped')
536 }
537 try {
538 salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
539 } catch (Exception e) {
540 common.warningMsg('Files are not present')
541 }
542 try {
543 salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /var/lib/mysql/*")
544 } catch (Exception e) {
545 common.warningMsg('Directory already empty')
546 }
547 try {
548 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
549 } catch (Exception e) {
550 common.warningMsg('File is not present')
551 }
552 salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
553 _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
554 backup_dir = _pillar['return'][0].values()[0]
555 if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
556 print(backup_dir)
557 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
558 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
559 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
560 sleep(5)
561 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
562 //
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]563
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]564 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
565 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
566 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
567 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
568 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
569
570 // salt 'cmp*' cmd.run 'service nova-compute restart'
571 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
572
Jiri Broulik9b73d6c2017-06-02 12:27:05 +0200[diff] [blame]573 sleep(70)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]574
Jiri Broulikf20c7672017-06-19 21:20:52 +0200[diff] [blame]575 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
576 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
577
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]578 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
579 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]580 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]581 }
582}