Gitiles
Code Review Sign In
gerrit.mcp.mirantis.com / mk / mk-pipelines / f18fb15d6e52d8b27e5d9c3a35f0ff3e263e0cbb / . / heavy-lab-pipeline.groovy
blob: fa42445874c56749e04aec91680a0cba75657b58 [file] [log] [blame]
Ales Komarek47a29f12017-04-26 12:05:47 +0200[diff] [blame]1/**
2 *
3 * Launch heat stack with basic k8s
4 * Flow parameters:
5 * STACK_TYPE Orchestration engine: heat, ''
6 * INSTALL What should be installed (k8s, openstack, ...)
7 * TEST What should be tested (k8s, openstack, ...)
8 *
9 * Expected parameters:
10 *
11 * required for STACK_TYPE=heat
12 * HEAT_TEMPLATE_URL URL to git repo with Heat templates
13 * HEAT_TEMPLATE_CREDENTIALS Credentials to the Heat templates repo
14 * HEAT_TEMPLATE_BRANCH Heat templates repo branch
15 * HEAT_STACK_TEMPLATE Heat stack HOT template
16 * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
17 * HEAT_STACK_ZONE Heat stack availability zone
18 * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
19 * HEAT_STACK_DELETE Delete Heat stack when finished (bool)
20 * HEAT_STACK_CLEANUP_JOB Name of job for deleting Heat stack
21 * HEAT_STACK_REUSE Reuse Heat stack (don't create one)
22 * OPENSTACK_API_URL OpenStack API address
23 * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
24 * OPENSTACK_API_PROJECT OpenStack project to connect to
25 * OPENSTACK_API_CLIENT Versions of OpenStack python clients
26 * OPENSTACK_API_VERSION Version of the OpenStack API (2/3)
27 *
28 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
29 *
30 * required for STACK_TYPE=NONE or empty string
31 * SALT_MASTER_URL URL of Salt-API
32
33 * K8S_API_SERVER Kubernetes API address
34 * K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
35 *
36 * TEMPEST_IMAGE_LINK Tempest image link
37 *
38 */
39
40git = new com.mirantis.mk.Git()
41openstack = new com.mirantis.mk.Openstack()
42salt = new com.mirantis.mk.Salt()
43common = new com.mirantis.mk.Common()
44test = new com.mirantis.mk.Test()
45
46timestamps {
47 node {
48 try {
49 //
50 // Prepare machines
51 //
52 stage ('Create infrastructure') {
53
54 if (STACK_TYPE == 'heat') {
55 // value defaults
56 def openstackCloud
57 def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
58 def openstackEnv = "${env.WORKSPACE}/venv"
59
60 if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
61 error("If you want to reuse existing stack you need to provide it's name")
62 }
63
64 if (HEAT_STACK_REUSE.toBoolean() == false) {
65 // Don't allow to set custom heat stack name
66 wrap([$class: 'BuildUser']) {
67 if (env.BUILD_USER_ID) {
68 HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
69 } else {
70 HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
71 }
72 currentBuild.description = HEAT_STACK_NAME
73 }
74 }
75
76 // set description
77 currentBuild.description = "${HEAT_STACK_NAME}"
78
79 // get templates
80 git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
81
82 // create openstack env
83 openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
84 openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
85 openstack.getKeystoneToken(openstackCloud, openstackEnv)
86
87 // launch stack
88 if (HEAT_STACK_REUSE.toBoolean() == false) {
89 stage('Launch new Heat stack') {
90 // create stack
91 envParams = [
92 'instance_zone': HEAT_STACK_ZONE,
93 'public_net': HEAT_STACK_PUBLIC_NET
94 ]
95 openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
96 }
97 }
98
99 // get SALT_MASTER_URL
100 saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
101 currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
102
103 if (common.checkContains('INSTALL', 'kvm')) {
104 saltPort = 6969
105 } else {
106 saltPort = 6969
107 }
108
109 SALT_MASTER_URL = "http://${saltMasterHost}:${saltPort}"
110 }
111 }
112
113 //
114 // Connect to Salt master
115 //
116
117 def saltMaster
118 stage('Connect to Salt API') {
119 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
120 }
121
122 //
123 // Install
124 //
125
126 if (common.checkContains('INSTALL', 'core')) {
127 stage('Install core infrastructure') {
128 // orchestrate.installFoundationInfra(saltMaster)
129 salt.enforceState(saltMaster, 'I@salt:master', ['salt.master', 'reclass'], true)
130 salt.enforceState(saltMaster, '*', ['linux.system'], true)
131 salt.enforceState(saltMaster, '*', ['salt.minion'], true)
132 salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
133 salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
134 salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'openssh', 'salt.minion', 'ntp'], true)
135
136
137 if (common.checkContains('INSTALL', 'kvm')) {
138 // orchestrate.installInfraKvm(saltMaster)
139 // orchestrate.installFoundationInfra(saltMaster)
140 //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.refresh_pillar', [], null, true)
141 //salt.runSaltProcessStep(saltMaster, 'I@linux:system', 'saltutil.sync_all', [], null, true)
142
143 //salt.enforceState(saltMaster, 'I@salt:control', ['salt.minion', 'linux.system', 'linux.network', 'ntp'], true)
144 salt.enforceState(saltMaster, 'I@salt:control', 'libvirt', true)
145 salt.enforceState(saltMaster, 'I@salt:control', 'salt.control', true)
146
147 sleep(600)
148
149 salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.refresh_pillar', [], null, true)
150 salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'saltutil.sync_all', [], null, true)
151
152 // workaround - install apt-transport-https
153 //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'cmd.run', ['apt-get update -y && apt-get install -y apt-transport-https'], null, true)
154 //salt.runSaltProcessStep(saltMaster, '* and not kvm*', 'pkg.install', ['apt-transport-https', 'refresh=True'], null, true)
155 salt.enforceState(saltMaster, 'I@linux:system', ['linux', 'salt.minion'], true, false)
156 salt.enforceState(saltMaster, 'I@linux:system', ['openssh', 'salt.minion', 'ntp'], true, false)
157 }
158
159 // orchestrate.validateFoundationInfra(saltMaster)
160 salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['salt-key'], null, true)
161 salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'test.version', [], null, true)
162 salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'cmd.run', ['reclass-salt --top'], null, true)
163 salt.runSaltProcessStep(saltMaster, 'I@reclass:storage', 'reclass.inventory', [], null, true)
164 salt.runSaltProcessStep(saltMaster, 'I@salt:minion', 'state.show_top', [], null, true)
165 }
166 }
167
168 // install k8s
169 if (common.checkContains('INSTALL', 'k8s')) {
170 stage('Install Kubernetes infra') {
171 // orchestrate.installKubernetesInfra(saltMaster)
172
173 // Install glusterfs
174 salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
175
176 // Install keepalived
177 salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster and *01*', 'state.sls', ['keepalived'])
178 salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
179
180 // Setup glusterfs
181 salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server and *01*', 'state.sls', ['glusterfs.server.setup'])
182 salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
183 salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
184
185 // Install haproxy
186 salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
187 salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
188
189 // Install docker
190 salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'state.sls', ['docker.host'])
191 salt.runSaltProcessStep(saltMaster, 'I@docker:host', 'cmd.run', ['docker ps'])
192
193 // Install etcd
194 salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
195 salt.runSaltProcessStep(saltMaster, 'I@etcd:server', 'cmd.run', ['./var/lib/etcd/configenv && etcdctl cluster-health'])
196
197 }
198
199 stage('Install Kubernetes control') {
200 // orchestrate.installKubernetesControl(saltMaster)
201
202 // Install Kubernetes pool and Calico
203 salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes.master.kube-addons'])
204 salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
205
206 // Setup etcd server
207 salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['etcd.server.setup'])
208
209 // Run k8s without master.setup
210 salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])
211
212 // Run k8s master setup
213 salt.runSaltProcessStep(saltMaster, 'I@kubernetes:master and *01*', 'state.sls', ['kubernetes.master.setup'])
214
215 // Restart kubelet
216 salt.runSaltProcessStep(saltMaster, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
217 }
218
219 }
220
221 // install openstack
222 if (common.checkContains('INSTALL', 'openstack')) {
223 // install Infra and control, tests, ...
224
225 stage('Install OpenStack infra') {
226 // orchestrate.installOpenstackInfra(saltMaster, physical)
227
228 // Install glusterfs
229 salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
230
231 // Install keepaliveds
232 //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
233 salt.enforceState(saltMaster, 'I@keepalived:cluster and *01*', 'keepalived', true)
234 salt.enforceState(saltMaster, 'I@keepalived:cluster', 'keepalived', true)
235
236 // Check the keepalived VIPs
237 salt.runSaltProcessStep(saltMaster, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
238
239 salt.enforceState(saltMaster, 'I@glusterfs:server and *01*', 'glusterfs.server.setup', true)
240
241 salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'], null, true)
242 salt.runSaltProcessStep(saltMaster, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'], null, true)
243
244 // Install rabbitmq
245 withEnv(['ASK_ON_ERROR=false']){
246 retry(2) {
247 salt.enforceState(saltMaster, 'I@rabbitmq:server', 'rabbitmq', true)
248 }
249 }
250 // Check the rabbitmq status
251 salt.runSaltProcessStep(saltMaster, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
252
253 // Install galera
254 withEnv(['ASK_ON_ERROR=false']){
255 retry(2) {
256 salt.enforceState(saltMaster, 'I@galera:master', 'galera', true)
257 }
258 }
259 salt.enforceState(saltMaster, 'I@galera:slave', 'galera', true)
260
261 // Check galera status
262 salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'mysql.status')
263 salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'mysql.status')
264
265 // // Setup mysql client
266 // salt.enforceState(saltMaster, 'I@mysql:client', 'mysql.client', true)
267
268 // Install haproxy
269 salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy', true)
270 salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.status', ['haproxy'])
271 salt.runSaltProcessStep(saltMaster, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
272
273 // Install memcached
274 salt.enforceState(saltMaster, 'I@memcached:server', 'memcached', true)
275
276 }
277
278 stage('Install OpenStack control') {
279 // orchestrate.installOpenstackControl(saltMaster)
280
281 // Install horizon dashboard
282 salt.enforceState(saltMaster, 'I@horizon:server', 'horizon', true)
283 salt.enforceState(saltMaster, 'I@nginx:server', 'nginx', true)
284
285 // setup keystone service
286 //runSaltProcessStep(saltMaster, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
287 salt.enforceState(saltMaster, 'I@keystone:server and *01*', 'keystone.server', true)
288 salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
289 // populate keystone services/tenants/roles/users
290
291 // keystone:client must be called locally
292 //salt.runSaltProcessStep(saltMaster, 'I@keystone:client', 'cmd.run', ['salt-call state.sls keystone.client'], null, true)
293 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'service.restart', ['apache2'])
294 sleep(30)
295 salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
296 salt.enforceState(saltMaster, 'I@keystone:client', 'keystone.client', true)
297 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonercv3; openstack service list'], null, true)
298
299 // Install glance and ensure glusterfs clusters
300 //runSaltProcessStep(saltMaster, 'I@glance:server', 'state.sls', ['glance.server'], 1)
301 salt.enforceState(saltMaster, 'I@glance:server and *01*', 'glance.server', true)
302 salt.enforceState(saltMaster, 'I@glance:server', 'glance.server', true)
303 salt.enforceState(saltMaster, 'I@glance:server', 'glusterfs.client', true)
304
305 // Update fernet tokens before doing request on keystone server
306 salt.enforceState(saltMaster, 'I@keystone:server', 'keystone.server', true)
307
308 // Check glance service
309 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'], null, true)
310
311 // Install and check nova service
312 //runSaltProcessStep(saltMaster, 'I@nova:controller', 'state.sls', ['nova'], 1)
313 salt.enforceState(saltMaster, 'I@nova:controller and *01*', 'nova.controller', true)
314 salt.enforceState(saltMaster, 'I@nova:controller', 'nova.controller', true)
315 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'], null, true)
316
317 // Install and check cinder service
318 //runSaltProcessStep(saltMaster, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
319 salt.enforceState(saltMaster, 'I@cinder:controller and *01*', 'cinder', true)
320 salt.enforceState(saltMaster, 'I@cinder:controller', 'cinder', true)
321 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'], null, true)
322
323 // Install neutron service
324 //runSaltProcessStep(saltMaster, 'I@neutron:server', 'state.sls', ['neutron'], 1)
325
326 salt.enforceState(saltMaster, 'I@neutron:server and *01*', 'neutron.server', true)
327 salt.enforceState(saltMaster, 'I@neutron:server', 'neutron.server', true)
328 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'], null, true)
329
330 // Install heat service
331 //runSaltProcessStep(saltMaster, 'I@heat:server', 'state.sls', ['heat'], 1)
332 salt.enforceState(saltMaster, 'I@heat:server and *01*', 'heat', true)
333 salt.enforceState(saltMaster, 'I@heat:server', 'heat', true)
334 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'], null, true)
335
336 // Restart nova api
337 salt.runSaltProcessStep(saltMaster, 'I@nova:controller', 'service.restart', ['nova-api'])
338
339 }
340
341 stage('Install OpenStack network') {
342
343 if (common.checkContains('INSTALL', 'contrail')) {
344 // orchestrate.installContrailNetwork(saltMaster)
345 // Install opencontrail database services
346 //runSaltProcessStep(saltMaster, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
347 try {
348 salt.enforceState(saltMaster, 'I@opencontrail:database and *01*', 'opencontrail.database', true)
349 } catch (Exception e) {
350 common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database and *01*')
351 }
352
353 try {
354 salt.enforceState(saltMaster, 'I@opencontrail:database', 'opencontrail.database', true)
355 } catch (Exception e) {
356 common.warningMsg('Exception in state opencontrail.database on I@opencontrail:database')
357 }
358
359 // Install opencontrail control services
360 //runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
361 salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and *01*', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
362 salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
363 salt.runSaltProcessStep(saltMaster, 'I@opencontrail:collector', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
364
365 // Test opencontrail
366 salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'cmd.run', ['contrail-status'], null, true)
367 } else if (common.checkContains('INSTALL', 'ovs')) {
368 // orchestrate.installOpenstackNetwork(saltMaster)
369 // Apply gateway
370 salt.runSaltProcessStep(saltMaster, 'I@neutron:gateway', 'state.apply', [], null, true)
371 }
372
373 // Pring information
374 // where is this
375 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'], null, true)
376 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'], null, true)
377 }
378
379 stage('Install OpenStack compute') {
380 // orchestrate.installOpenstackCompute(saltMaster)
381 // Configure compute nodes
382 retry(2) {
383 salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'state.apply', [], null, true)
384 }
385
386 if (common.checkContains('INSTALL', 'contrail')) {
387 // orchestrate.installContrailCompute(saltMaster)
388 // Provision opencontrail control services
389 salt.enforceState(saltMaster, 'I@opencontrail:database:id:1', 'opencontrail.client', true)
390 // Provision opencontrail virtual routers
391 salt.enforceState(saltMaster, 'I@opencontrail:compute', 'opencontrail.client', true)
392
393 salt.runSaltProcessStep(saltMaster, 'I@nova:compute', 'cmd.run', ['exec 0>&-; exec 1>&-; exec 2>&-; nohup bash -c "ip link | grep vhost && echo no_reboot || sleep 5 && reboot & "'], null, true)
394 sleep(300)
395 }
396 }
397
398 }
399
400
401 if (common.checkContains('INSTALL', 'stacklight')) {
402 stage('Install StackLight') {
403 //TODO: orchestrate.installStacklightControl(saltMaster)
404 //TODO: orchestrate.installStacklightClient(saltMaster)
405
406 // infra install
407 // Install the StackLight backends
408 salt.enforceState(saltMaster, '*01* and I@elasticsearch:server', 'elasticsearch.server', true)
409 salt.enforceState(saltMaster, 'I@elasticsearch:server', 'elasticsearch.server', true)
410
411 salt.enforceState(saltMaster, '*01* and I@influxdb:server', 'influxdb', true)
412 salt.enforceState(saltMaster, 'I@influxdb:server', 'influxdb', true)
413
414 salt.enforceState(saltMaster, '*01* and I@kibana:server', 'kibana.server', true)
415 salt.enforceState(saltMaster, 'I@kibana:server', 'kibana.server', true)
416
417 salt.enforceState(saltMaster, '*01* and I@grafana:server','grafana.server', true)
418 salt.enforceState(saltMaster, 'I@grafana:server','grafana.server', true)
419
420 salt.enforceState(saltMaster, 'I@nagios:server', 'nagios.server', true)
421 salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client.service', true)
422 salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client.service', true)
423 // nw salt.enforceState('I@kibana:client or I@elasticsearch:client' --async service.restart salt-minion
424
425 sleep(10)
426
427 salt.runSaltProcessStep(saltMaster, 'I@elasticsearch.client', 'cmd.run', ['salt-call state.sls elasticsearch.client'], null, true)
428 // salt.enforceState(saltMaster, 'I@elasticsearch:client', 'elasticsearch.client', true)
429 salt.runSaltProcessStep(saltMaster, 'I@kibana.client', 'cmd.run', ['salt-call state.sls kibana.client'], null, true)
430 // salt.enforceState(saltMaster, 'I@kibana:client', 'kibana.client', true)
431
432 // install monitor
433 // Restart salt-minion to make sure that it uses the latest Jinja library
434 // no way: salt '*' --async service.restart salt-minion; sleep 15
435
436 // Start by flusing Salt Mine to make sure it is clean
437 // Also clean-up the grains files to make sure that we start from a clean state
438 // nw salt "*" mine.flush
439 // nw salt "*" file.remove /etc/salt/grains.d/collectd
440 // nw salt "*" file.remove /etc/salt/grains.d/grafana
441 // nw salt "*" file.remove /etc/salt/grains.d/heka
442 // nw salt "*" file.remove /etc/salt/grains.d/sensu
443 // nw salt "*" file.remove /etc/salt/grains
444
445 // Install collectd, heka and sensu services on the nodes, this will also
446 // generate the metadata that goes into the grains and eventually into Salt Mine
447 salt.enforceState(saltMaster, '*', 'collectd', true)
448 salt.enforceState(saltMaster, '*', 'salt.minion', true)
449 salt.enforceState(saltMaster, '*', 'heka', true)
450
451 // Gather the Grafana metadata as grains
452 salt.enforceState(saltMaster, 'I@grafana:collector', 'grafana.collector', true)
453
454 // Update Salt Mine
455 salt.enforceState(saltMaster, '*', 'salt.minion.grains', true)
456 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_modules', [], null, true)
457 salt.runSaltProcessStep(saltMaster, '*', 'mine.update', [], null, true)
458
459 sleep(5)
460
461 // Update Heka
462 salt.enforceState(saltMaster, 'I@heka:aggregator:enabled:True or I@heka:remote_collector:enabled:True', 'heka', true)
463
464 // Update collectd
465 salt.enforceState(saltMaster, 'I@collectd:remote_client:enabled:True', 'collectd', true)
466
467 // Update Nagios
468 salt.enforceState(saltMaster, 'I@nagios:server', 'nagios', true)
469 // Stop the Nagios service because the package starts it by default and it will
470 // started later only on the node holding the VIP address
471 salt.runSaltProcessStep(saltMaster, 'I@nagios:server', 'service.stop', ['nagios3'], null, true)
472
473 // Update Sensu
474 // TODO for stacklight team, should be fixed in model
475 //salt.enforceState(saltMaster, 'I@sensu:server', 'sensu', true)
476
477 // Finalize the configuration of Grafana (add the dashboards...)
478 salt.enforceState(saltMaster, 'I@grafana:client', 'grafana.client.service', true)
479 // nw salt -C 'I@grafana:client' --async service.restart salt-minion; sleep 10
480
481 salt.runSaltProcessStep(saltMaster, 'I@grafana.client and *01*', 'cmd.run', ['salt-call state.sls grafana.client'], null, true)
482 // salt.enforceState(saltMaster, 'I@grafana:client and *01*', 'grafana.client', true)
483
484 // Get the StackLight monitoring VIP addres
485 //vip=$(salt-call pillar.data _param:stacklight_monitor_address --out key|grep _param: |awk '{print $2}')
486 //vip=${vip:=172.16.10.253}
487 def pillar = salt.getPillar(saltMaster, 'ctl01*', '_param:stacklight_monitor_address')
488 print(common.prettyPrint(pillar))
489 def stacklight_vip = pillar['return'][0].values()[0]
490
491 if (stacklight_vip) {
492 // (re)Start manually the services that are bound to the monitoring VIP
493 common.infoMsg("restart services on node with IP: ${stacklight_vip}")
494 salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collectd'], null, true)
495 salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['remote_collector'], null, true)
496 salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['aggregator'], null, true)
497 salt.runSaltProcessStep(saltMaster, "G@ipv4:${stacklight_vip}", 'service.restart', ['nagios3'], null, true)
498 } else {
499 throw new Exception("Missing stacklight_vip")
500 }
501 }
502 }
503
504 //
505 // Test
506 //
507 def artifacts_dir = '_artifacts/'
508
509 if (common.checkContains('TEST', 'k8s')) {
510 stage('Run k8s bootstrap tests') {
511 def image = 'tomkukral/k8s-scripts'
512 def output_file = image.replaceAll('/', '-') + '.output'
513
514 // run image
515 test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
516
517 // collect output
518 sh "mkdir -p ${artifacts_dir}"
519 file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
520 writeFile file: "${artifacts_dir}${output_file}", text: file_content
521 sh "cat ${artifacts_dir}${output_file}"
522
523 // collect artifacts
524 archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
525 }
526
527 stage('Run k8s conformance e2e tests') {
528 //test.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
529
530 def image = K8S_CONFORMANCE_IMAGE
531 def output_file = image.replaceAll('/', '-') + '.output'
532
533 // run image
534 test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
535
536 // collect output
537 sh "mkdir -p ${artifacts_dir}"
538 file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
539 writeFile file: "${artifacts_dir}${output_file}", text: file_content
540 sh "cat ${artifacts_dir}${output_file}"
541
542 // collect artifacts
543 archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
544 }
545 }
546
547 if (common.checkContains('TEST', 'openstack')) {
548 stage('Run OpenStack tests') {
549 test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
550 }
551
552 stage('Copy Tempest results to config node') {
553 test.copyTempestResults(saltMaster)
554 }
555 }
556
557 stage('Finalize') {
558 if (INSTALL != '') {
559 try {
560 salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
561 } catch (Exception e) {
562 common.warningMsg('State apply failed but we should continue to run')
563 }
564 }
565 }
566 } catch (Throwable e) {
567 currentBuild.result = 'FAILURE'
568 throw e
569 } finally {
570
571
572 //
573 // Clean
574 //
575
576 if (STACK_TYPE == 'heat') {
577 // send notification
578 common.sendNotification(currentBuild.result, HEAT_STACK_NAME, ["slack"])
579
580 if (HEAT_STACK_DELETE.toBoolean() == true) {
581 common.errorMsg('Heat job cleanup triggered')
582 stage('Trigger cleanup job') {
583 build job: 'deploy-heat-cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
584 }
585 } else {
586 if (currentBuild.result == 'FAILURE') {
587 common.errorMsg("Deploy job FAILED and was not deleted. Please fix the problem and delete stack on you own.")
588
589 if (SALT_MASTER_URL) {
590 common.errorMsg("Salt master URL: ${SALT_MASTER_URL}")
591 }
592 }
593
594 }
595 }
596 }
597 }
598}