Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mk / mk-pipelines / ff24e585f0fdddb99dd3ae1f19170bee21706c3d / . / openstack-control-upgrade.groovy
blob: 9680f24faf1d415eaf5b9c4a8dcfc13fca12f6ea [file] [log] [blame]
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]1/**
2 * Update packages on given nodes
3 *
4 * Expected parameters:
5 * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
6 * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]7 * STAGE_TEST_UPGRADE Run test upgrade stage (bool)
8 * STAGE_REAL_UPGRADE Run real upgrade stage (bool)
9 * STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]10 *
11**/
12
13def common = new com.mirantis.mk.Common()
14def salt = new com.mirantis.mk.Salt()
15
16
17def saltMaster
18
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]19node() {
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]20
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]21 stage('Connect to Salt API') {
22 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
23 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]24
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]25 if (STAGE_TEST_UPGRADE.toBoolean() == true) {
26 stage('Test upgrade') {
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]27
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]28
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]29 try {
30 salt.enforceState(saltMaster, 'I@salt:master', 'reclass')
31 } catch (Exception e) {
32 common.warningMsg("Some parts of Reclass state failed. The most probable reasons were uncommited changes. We should continue to run")
33 }
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]34
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]35 try {
36 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
37 } catch (Exception e) {
38 common.warningMsg("No response from some minions. We should continue to run")
39 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]40
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]41 try {
42 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
43 } catch (Exception e) {
44 common.warningMsg("No response from some minions. We should continue to run")
45 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]46
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]47 def _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
48 def domain = _pillar['return'][0].values()[0].values()[0]
49 print(_pillar)
50 print(domain)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]51
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]52 // read backupninja variable
53 _pillar = salt.getPillar(saltMaster, 'I@backupninja:client', '_param:backupninja_backup_host')
54 def backupninja_backup_host = _pillar['return'][0].values()[0]
55 print(_pillar)
56 print(backupninja_backup_host)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]57
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]58 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
59 def kvm01 = _pillar['return'][0].values()[0].values()[0]
60 print(_pillar)
61 print(kvm01)
62
63 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:upg01:provider')
64 def upgNodeProvider = _pillar['return'][0].values()[0]
65 print(_pillar)
66 print(upgNodeProvider)
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]67
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]68
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]69 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.destroy', ["upg01.${domain}"], null, true)
70 salt.runSaltProcessStep(saltMaster, "${upgNodeProvider}", 'virt.undefine', ["upg01.${domain}"], null, true)
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]71
72
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]73 try {
74 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d upg01.${domain} -y")
75 } catch (Exception e) {
76 common.warningMsg("upg01.${domain} does not match any accepted, unaccepted or rejected keys. The key did not exist yet or was already removed. We should continue to run")
77 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]78
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]79
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]80 // salt 'kvm02*' state.sls salt.control
81 salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]82
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]83 // wait until upg node is registered in salt-key
84 salt.minionPresent(saltMaster, 'I@salt:master', 'upg01')
85
86 // salt '*' saltutil.refresh_pillar
87 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
88 // salt '*' saltutil.sync_all
89 salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.sync_all', [], null, true)
90
91 // salt "upg*" state.sls linux,openssh,salt.minion,ntp,rsyslog
92 try {
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]93 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]94 } catch (Exception e) {
95 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
96 }
97 salt.enforceState(saltMaster, 'upg*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]98
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]99 // salt "upg*" state.sls rabbitmq
100 salt.enforceState(saltMaster, 'upg*', ['rabbitmq', 'memcached'])
101 try {
102 salt.enforceState(saltMaster, 'I@backupninja:client', ['openssh.client', 'salt.minion'])
103 } catch (Exception e) {
104 common.warningMsg('salt-minion was restarted. We should continue to run')
105 }
106 try {
107 salt.enforceState(saltMaster, 'I@backupninja:server', ['salt.minion'])
108 } catch (Exception e) {
109 common.warningMsg('salt-minion was restarted. We should continue to run')
110 }
111 // salt '*' state.apply salt.minion.grains
112 //salt.enforceState(saltMaster, '*', 'salt.minion.grains')
113 // salt -C 'I@backupninja:server' state.sls backupninja
114 salt.enforceState(saltMaster, 'I@backupninja:server', 'backupninja')
115 // salt -C 'I@backupninja:client' state.sls backupninja
116 salt.enforceState(saltMaster, 'I@backupninja:client', 'backupninja')
117 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.rm_known_host', ["root", "${backupninja_backup_host}"], null, true)
118 try {
119 salt.cmdRun(saltMaster, 'I@backupninja:client', "arp -d ${backupninja_backup_host}")
120 } catch (Exception e) {
121 common.warningMsg('The ARP entry does not exist. We should continue to run.')
122 }
123 salt.runSaltProcessStep(saltMaster, 'I@backupninja:client', 'ssh.set_known_host', ["root", "${backupninja_backup_host}"], null, true)
124 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/101.mysql')
125 salt.cmdRun(saltMaster, 'I@backupninja:client', 'backupninja -n --run /etc/backup.d/200.backup.rsync > /tmp/backupninjalog')
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]126
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]127 salt.enforceState(saltMaster, 'I@xtrabackup:server', 'xtrabackup')
128 salt.enforceState(saltMaster, 'I@xtrabackup:client', 'openssh.client')
129 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
130 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]131
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]132 def databases = salt.cmdRun(saltMaster, 'I@mysql:client','salt-call mysql.db_list | grep upgrade | awk \'/-/ {print \$2}\'')
133 if(databases && databases != ""){
134 def databasesList = databases['return'][0].values()[0].trim().tokenize("\n")
135 for( i = 0; i < databasesList.size(); i++){
136 if(databasesList[i].toLowerCase().contains('upgrade')){
137 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'mysql.db_remove', ["${databasesList[i]}"], null, true)
138 common.warningMsg("removing database ${databasesList[i]}")
139 salt.runSaltProcessStep(saltMaster, 'I@mysql:client', 'file.remove', ["/root/mysql/flags/${databasesList[i]}-installed"], null, true)
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]140 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]141 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]142 salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client')
143 }else{
144 common.errorMsg("No _upgrade databases were returned")
145 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]146
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]147 try {
148 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
149 salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
150 } catch (Exception e) {
151 common.warningMsg('Restarting Apache2')
152 salt.runSaltProcessStep(saltMaster, 'upg*', 'service.restart', ['apache2'], null, true)
153 }
154 try {
155 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
156 } catch (Exception e) {
157 common.warningMsg('running keystone.client state again')
158 salt.enforceState(saltMaster, 'upg*', 'keystone.client')
159 }
160 try {
161 salt.enforceState(saltMaster, 'upg*', 'glance')
162 } catch (Exception e) {
163 common.warningMsg('running glance state again')
164 salt.enforceState(saltMaster, 'upg*', 'glance')
165 }
166 salt.enforceState(saltMaster, 'upg*', 'keystone.server')
167 try {
168 salt.enforceState(saltMaster, 'upg*', 'nova')
169 } catch (Exception e) {
170 common.warningMsg('running nova state again')
171 salt.enforceState(saltMaster, 'upg*', 'nova')
172 }
173 // run nova state again as sometimes nova does not enforce itself for some reason
174 try {
175 salt.enforceState(saltMaster, 'upg*', 'nova')
176 } catch (Exception e) {
177 common.warningMsg('running nova state again')
178 salt.enforceState(saltMaster, 'upg*', 'nova')
179 }
180 try {
181 salt.enforceState(saltMaster, 'upg*', 'cinder')
182 } catch (Exception e) {
183 common.warningMsg('running cinder state again')
184 salt.enforceState(saltMaster, 'upg*', 'cinder')
185 }
186 try {
187 salt.enforceState(saltMaster, 'upg*', 'neutron')
188 } catch (Exception e) {
189 common.warningMsg('running neutron state again')
190 salt.enforceState(saltMaster, 'upg*', 'neutron')
191 }
192 try {
193 salt.enforceState(saltMaster, 'upg*', 'heat')
194 } catch (Exception e) {
195 common.warningMsg('running heat state again')
196 salt.enforceState(saltMaster, 'upg*', 'heat')
197 }
198 salt.cmdRun(saltMaster, 'upg01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
199
200 if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
201 stage('Ask for manual confirmation') {
202 input message: "Do you want to continue with upgrade?"
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]203 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]204 }
205 }
206 }
207
208 if (STAGE_REAL_UPGRADE.toBoolean() == true) {
209 stage('Real upgrade') {
210 // # actual upgrade
211
212 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
213 domain = _pillar['return'][0].values()[0].values()[0]
214 print(_pillar)
215 print(domain)
216
217 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
218 kvm01 = _pillar['return'][0].values()[0].values()[0]
219 print(_pillar)
220 print(kvm01)
221
222 def errorOccured = false
223
224 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
225 def ctl01NodeProvider = _pillar['return'][0].values()[0]
226
227 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
228 def ctl02NodeProvider = _pillar['return'][0].values()[0]
229
230 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
231 def ctl03NodeProvider = _pillar['return'][0].values()[0]
232
233 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
234 def prx01NodeProvider = _pillar['return'][0].values()[0]
235
236 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
237 def prx02NodeProvider = _pillar['return'][0].values()[0]
238
239
240 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
241 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
242 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
243 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
244 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
245
246
247 try {
248 salt.cmdRun(saltMaster, "${prx01NodeProvider}", "[ ! -f /root/prx01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx01.${domain}/system.qcow2 ./prx01.${domain}.qcow2.bak")
249 } catch (Exception e) {
250 common.warningMsg('File already exists')
251 }
252 try {
253 salt.cmdRun(saltMaster, "${prx02NodeProvider}", "[ ! -f /root/prx02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/prx02.${domain}/system.qcow2 ./prx02.${domain}.qcow2.bak")
254 } catch (Exception e) {
255 common.warningMsg('File already exists')
256 }
257 try {
258 salt.cmdRun(saltMaster, "${ctl01NodeProvider}", "[ ! -f /root/ctl01.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl01.${domain}/system.qcow2 ./ctl01.${domain}.qcow2.bak")
259 } catch (Exception e) {
260 common.warningMsg('File already exists')
261 }
262 try {
263 salt.cmdRun(saltMaster, "${ctl02NodeProvider}", "[ ! -f /root/ctl02.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl02.${domain}/system.qcow2 ./ctl02.${domain}.qcow2.bak")
264 } catch (Exception e) {
265 common.warningMsg('File already exists')
266 }
267 try {
268 salt.cmdRun(saltMaster, "${ctl03NodeProvider}", "[ ! -f /root/ctl03.${domain}.qcow2.bak ] && cp /var/lib/libvirt/images/ctl03.${domain}/system.qcow2 ./ctl03.${domain}.qcow2.bak")
269 } catch (Exception e) {
270 common.warningMsg('File already exists')
271 }
272
273
274 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.undefine', ["prx01.${domain}"], null, true)
275 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.undefine', ["prx02.${domain}"], null, true)
276 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.undefine', ["ctl01.${domain}"], null, true)
277 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.undefine', ["ctl02.${domain}"], null, true)
278 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.undefine', ["ctl03.${domain}"], null, true)
279
280 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c '/usr/local/bin/innobackupex-runner.sh'")
281
282 try {
283 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
284 } catch (Exception e) {
285 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
286 }
287
288 // salt 'kvm*' state.sls salt.control
289 salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
290
291 // wait until ctl and prx nodes are registered in salt-key
292 salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
293 salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
294 salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
295 salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
296 salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
297
298
299 // salt '*' saltutil.refresh_pillar
300 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
301 // salt '*' saltutil.sync_all
302 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.sync_all', [], null, true)
303
304 try {
305 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
306 } catch (Exception e) {
307 common.warningMsg('Received no response because salt-minion was restarted. We should continue to run')
308 }
309 salt.enforceState(saltMaster, 'ctl* or prx* or ctl*', ['linux', 'openssh', 'salt.minion', 'ntp', 'rsyslog'])
310
311 // salt 'ctl*' state.sls keepalived
312 // salt 'ctl*' state.sls haproxy
313 salt.enforceState(saltMaster, 'ctl*', ['keepalived', 'haproxy'])
314 // salt 'ctl*' service.restart rsyslog
315 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['rsyslog'], null, true)
316 // salt "ctl*" state.sls memcached
317 // salt "ctl*" state.sls keystone.server
318 try {
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]319 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]320 salt.enforceState(saltMaster, 'ctl*', ['memcached', 'keystone.server'])
321 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
322 } catch (Exception e) {
323 common.warningMsg('Restarting Apache2 and enforcing keystone.server state again')
324 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['apache2'], null, true)
325 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
326 }
327 // salt 'ctl01*' state.sls keystone.client
328 try {
329 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]330 } catch (Exception e) {
331 common.warningMsg('running keystone.client state again')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]332 salt.enforceState(saltMaster, 'I@keystone:client and ctl*', 'keystone.client')
Jiri Broulik69de13a2017-05-18 13:53:58 +0200[diff] [blame]333 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]334 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]335 salt.enforceState(saltMaster, 'ctl*', 'glance')
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]336 } catch (Exception e) {
337 common.warningMsg('running glance state again')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]338 salt.enforceState(saltMaster, 'ctl*', 'glance')
339 } // salt 'ctl*' state.sls glusterfs.client
340 salt.enforceState(saltMaster, 'ctl*', 'glusterfs.client')
341 // salt 'ctl*' state.sls keystone.server
342 salt.enforceState(saltMaster, 'ctl*', 'keystone.server')
343 // salt 'ctl*' state.sls nova
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]344 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]345 salt.enforceState(saltMaster, 'ctl*', 'nova')
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]346 } catch (Exception e) {
347 common.warningMsg('running nova state again')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]348 salt.enforceState(saltMaster, 'ctl*', 'nova')
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]349 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]350 // salt 'ctl*' state.sls cinder
Jiri Broulik80d96752017-05-22 19:42:01 +0200[diff] [blame]351 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]352 salt.enforceState(saltMaster, 'ctl*', 'cinder')
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]353 } catch (Exception e) {
354 common.warningMsg('running cinder state again')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]355 salt.enforceState(saltMaster, 'ctl*', 'cinder')
356 }
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]357 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]358 salt.enforceState(saltMaster, 'ctl*', 'neutron')
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]359 } catch (Exception e) {
360 common.warningMsg('running neutron state again')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]361 salt.enforceState(saltMaster, 'ctl*', 'neutron')
Jiri Broulik5f83c292017-05-18 12:09:07 +0200[diff] [blame]362 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]363 // salt 'ctl*' state.sls heat
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]364 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]365 salt.enforceState(saltMaster, 'ctl*', 'heat')
Jiri Broulikbda0a212017-05-18 12:54:51 +0200[diff] [blame]366 } catch (Exception e) {
367 common.warningMsg('running heat state again')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]368 salt.enforceState(saltMaster, 'ctl*', 'heat')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]369 }
370
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]371 } catch (Exception e) {
372 errorOccured = true
373 common.warningMsg('Some states that require syncdb failed. Restoring production databases')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]374
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]375 // database restore section
376 try {
377 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]378 } catch (Exception er) {
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]379 common.warningMsg('Mysql service already stopped')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]380 }
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]381 try {
382 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]383 } catch (Exception er) {
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]384 common.warningMsg('Mysql service already stopped')
385 }
386 try {
387 salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]388 } catch (Exception er) {
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]389 common.warningMsg('Files are not present')
390 }
391 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]392 salt.cmdRun(saltMaster, 'I@galera:master', "mkdir /root/mysql/mysql.bak")
393 } catch (Exception er) {
394 common.warningMsg('Directory already exists')
395 }
396 try {
397 salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /root/mysql/mysql.bak/*")
398 } catch (Exception er) {
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]399 common.warningMsg('Directory already empty')
400 }
401 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]402 salt.cmdRun(saltMaster, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
403 } catch (Exception er) {
404 common.warningMsg('Files were already moved')
405 }
406 try {
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]407 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]408 } catch (Exception er) {
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]409 common.warningMsg('File is not present')
410 }
411 salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
412 _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
413 backup_dir = _pillar['return'][0].values()[0]
414 if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
415 print(backup_dir)
416 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
417 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
418 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
Jiri Broulik8dabbfd2017-07-25 10:49:45 +0200[diff] [blame]419
420 // wait until mysql service on galera master is up
421 salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
422
Jiri Broulik00d01092017-06-01 19:38:15 +0200[diff] [blame]423 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
424 //
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]425
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]426 common.errorMsg("Stage Real control upgrade failed")
427 }
428 if(!errorOccured){
429
430 ceph = null
431
432 try {
433 ceph = salt.cmdRun(saltMaster, 'ctl*', "salt-call grains.item roles | grep ceph.client")
434
435 } catch (Exception er) {
436 common.infoMsg("Ceph is not used")
437 }
438
439 if(ceph != null) {
440 try {
441 salt.enforceState(saltMaster, 'ctl*', 'ceph.client')
442 } catch (Exception er) {
443 common.warningMsg("Ceph client state on controllers failed. Please fix it manually")
444 }
445 }
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]446
447 // salt 'cmp*' cmd.run 'service nova-compute restart'
448 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
Jiri Broulikf20c7672017-06-19 21:20:52 +0200[diff] [blame]449 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
450 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
451
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]452
453 // salt 'prx*' state.sls linux,openssh,salt.minion,ntp,rsyslog
454 // salt 'ctl*' state.sls keepalived
455 // salt 'prx*' state.sls keepalived
456 salt.enforceState(saltMaster, 'prx*', 'keepalived')
457 // salt 'prx*' state.sls horizon
458 salt.enforceState(saltMaster, 'prx*', 'horizon')
459 // salt 'prx*' state.sls nginx
460 salt.enforceState(saltMaster, 'prx*', 'nginx')
461 // salt "prx*" state.sls memcached
462 salt.enforceState(saltMaster, 'prx*', 'memcached')
463
464 try {
465 salt.enforceHighstate(saltMaster, 'ctl*')
466 } catch (Exception er) {
467 common.errorMsg("Highstate was executed on controller nodes but something failed. Please check it and fix it accordingly.")
468 }
469
470 try {
471 salt.enforceHighstate(saltMaster, 'prx*')
472 } catch (Exception er) {
473 common.errorMsg("Highstate was executed on proxy nodes but something failed. Please check it and fix it accordingly.")
474 }
475
476 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
Jiri Broulik3792c962017-05-17 18:26:36 +0200[diff] [blame]477 }
Jiri Broulik6fc9d3f2017-05-12 07:11:29 +0200[diff] [blame]478 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400[diff] [blame]479
480 if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
481 stage('Ask for manual confirmation') {
482 input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click YES to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
483 }
484 }
485 }
486
487 if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
488 stage('Rollback upgrade') {
489
490 stage('Ask for manual confirmation') {
491 input message: "Do you really want to continue with the rollback?"
492 }
493
494 _pillar = salt.getGrain(saltMaster, 'I@salt:master', 'domain')
495 domain = _pillar['return'][0].values()[0].values()[0]
496 print(_pillar)
497 print(domain)
498
499 _pillar = salt.getGrain(saltMaster, 'I@salt:control', 'id')
500 kvm01 = _pillar['return'][0].values()[0].values()[0]
501 print(_pillar)
502 print(kvm01)
503
504 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl01:provider')
505 def ctl01NodeProvider = _pillar['return'][0].values()[0]
506
507 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl02:provider')
508 def ctl02NodeProvider = _pillar['return'][0].values()[0]
509
510 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:ctl03:provider')
511 def ctl03NodeProvider = _pillar['return'][0].values()[0]
512
513 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx01:provider')
514 def prx01NodeProvider = _pillar['return'][0].values()[0]
515
516 _pillar = salt.getPillar(saltMaster, "${kvm01}", 'salt:control:cluster:internal:node:prx02:provider')
517 def prx02NodeProvider = _pillar['return'][0].values()[0]
518
519 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.destroy', ["prx01.${domain}"], null, true)
520 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.destroy', ["prx02.${domain}"], null, true)
521 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.destroy', ["ctl01.${domain}"], null, true)
522 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.destroy', ["ctl02.${domain}"], null, true)
523 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.destroy', ["ctl03.${domain}"], null, true)
524
525 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'file.copy', ["/root/prx01.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx01.${domain}/system.qcow2"], null, true)
526 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'file.copy', ["/root/prx02.${domain}.qcow2.bak", "/var/lib/libvirt/images/prx02.${domain}/system.qcow2"], null, true)
527 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'file.copy', ["/root/ctl01.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl01.${domain}/system.qcow2"], null, true)
528 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'file.copy', ["/root/ctl02.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl02.${domain}/system.qcow2"], null, true)
529 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'file.copy', ["/root/ctl03.${domain}.qcow2.bak", "/var/lib/libvirt/images/ctl03.${domain}/system.qcow2"], null, true)
530
531 try {
532 salt.cmdRun(saltMaster, 'I@salt:master', "salt-key -d ctl01.${domain},ctl02.${domain},ctl03.${domain},prx01.${domain},prx02.${domain} -y")
533 } catch (Exception e) {
534 common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
535 }
536
537 // database restore section
538 try {
539 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.stop', ['mysql'], null, true)
540 } catch (Exception e) {
541 common.warningMsg('Mysql service already stopped')
542 }
543 try {
544 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.stop', ['mysql'], null, true)
545 } catch (Exception e) {
546 common.warningMsg('Mysql service already stopped')
547 }
548 try {
549 salt.cmdRun(saltMaster, 'I@galera:slave', "rm /var/lib/mysql/ib_logfile*")
550 } catch (Exception e) {
551 common.warningMsg('Files are not present')
552 }
553 try {
554 salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /var/lib/mysql/*")
555 } catch (Exception e) {
556 common.warningMsg('Directory already empty')
557 }
558 try {
559 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
560 } catch (Exception e) {
561 common.warningMsg('File is not present')
562 }
563 salt.cmdRun(saltMaster, 'I@galera:master', "sed -i '/gcomm/c\\wsrep_cluster_address=\"gcomm://\"' /etc/mysql/my.cnf")
564 _pillar = salt.getPillar(saltMaster, "I@galera:master", 'xtrabackup:client:backup_dir')
565 backup_dir = _pillar['return'][0].values()[0]
566 if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/mysql/xtrabackup' }
567 print(backup_dir)
568 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
569 salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
570 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
571
572 // wait until mysql service on galera master is up
573 salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
574
575 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
576 //
577
578 salt.runSaltProcessStep(saltMaster, "${prx01NodeProvider}", 'virt.start', ["prx01.${domain}"], null, true)
579 salt.runSaltProcessStep(saltMaster, "${prx02NodeProvider}", 'virt.start', ["prx02.${domain}"], null, true)
580 salt.runSaltProcessStep(saltMaster, "${ctl01NodeProvider}", 'virt.start', ["ctl01.${domain}"], null, true)
581 salt.runSaltProcessStep(saltMaster, "${ctl02NodeProvider}", 'virt.start', ["ctl02.${domain}"], null, true)
582 salt.runSaltProcessStep(saltMaster, "${ctl03NodeProvider}", 'virt.start', ["ctl03.${domain}"], null, true)
583
584 // salt 'cmp*' cmd.run 'service nova-compute restart'
585 salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
586
587 // wait until ctl and prx nodes are registered in salt-key
588 salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
589 salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
590 salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
591 salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
592 salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
593
594 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
595 salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
596
597 salt.cmdRun(saltMaster, 'ctl01*', '. /root/keystonerc; nova service-list; glance image-list; nova flavor-list; nova hypervisor-list; nova list; neutron net-list; cinder list; heat service-list')
598 }
Jiri Broulikbb447ac2017-05-04 15:36:22 +0200[diff] [blame]599 }
600}