blob: 9fc42d3da8daabe3dc6119b75f55e49040261632 [file] [log] [blame]
Filip Pytloun0a07f702017-02-24 18:26:18 +01001/**
2 *
3 * Launch heat stack with CI/CD lab infrastructure
4 *
5 * Expected parameters:
6 * HEAT_TEMPLATE_URL URL to git repo with Heat templates
7 * HEAT_TEMPLATE_CREDENTIALS Credentials to the Heat templates repo
8 * HEAT_TEMPLATE_BRANCH Heat templates repo branch
9 * HEAT_STACK_NAME Heat stack name
10 * HEAT_STACK_TEMPLATE Heat stack HOT template
11 * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
12 * HEAT_STACK_ZONE Heat stack availability zone
13 * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
14 * HEAT_STACK_DELETE Delete Heat stack when finished (bool)
15 * HEAT_STACK_CLEANUP_JOB Name of job for deleting Heat stack
16 * HEAT_STACK_REUSE Reuse Heat stack (don't create one)
17 *
18 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
Filip Pytloune32fda82017-02-24 18:26:18 +010019 * SALT_MASTER_PORT Port of salt-api, defaults to 8000
Filip Pytloun0a07f702017-02-24 18:26:18 +010020 *
21 * OPENSTACK_API_URL OpenStack API address
22 * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
23 * OPENSTACK_API_PROJECT OpenStack project to connect to
24 * OPENSTACK_API_CLIENT Versions of OpenStack python clients
25 * OPENSTACK_API_VERSION Version of the OpenStack API (2/3)
26 *
27 */
28
Filip Pytlounad2b36b2017-03-04 20:33:41 +010029common = new com.mirantis.mk.Common()
Filip Pytloun0a07f702017-02-24 18:26:18 +010030git = new com.mirantis.mk.Git()
31openstack = new com.mirantis.mk.Openstack()
32salt = new com.mirantis.mk.Salt()
33orchestrate = new com.mirantis.mk.Orchestrate()
chnyda625f4b42017-10-11 14:10:31 +020034def python = new com.mirantis.mk.Python()
35
36def pepperEnv = "pepperEnv"
Jakub Josef458913d2017-05-10 15:37:56 +020037_MAX_PERMITTED_STACKS = 2
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040038
39node {
40 try {
41 // connection objects
42 def openstackCloud
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040043
44 // value defaults
45 def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
46 def openstackEnv = "${env.WORKSPACE}/venv"
47
Filip Pytlounbfce09d2017-03-01 19:00:43 +010048 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040049 sshPubKey = SSH_PUBLIC_KEY
50 } catch (MissingPropertyException e) {
51 sshPubKey = false
52 }
Filip Pytloun0a07f702017-02-24 18:26:18 +010053
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040054 if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
55 error("If you want to reuse existing stack you need to provide it's name")
56 }
Filip Pytloun0a07f702017-02-24 18:26:18 +010057
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040058 if (HEAT_STACK_REUSE.toBoolean() == false) {
59 // Don't allow to set custom heat stack name
60 wrap([$class: 'BuildUser']) {
61 if (env.BUILD_USER_ID) {
62 HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
Filip Pytloun0da421f2017-03-03 18:50:45 +010063 } else {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040064 HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
Filip Pytloun0da421f2017-03-03 18:50:45 +010065 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040066 currentBuild.description = HEAT_STACK_NAME
67 }
68 }
Filip Pytloun3eefd3d2017-03-03 14:13:41 +010069
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040070 //
71 // Bootstrap
72 //
73
74 stage ('Download Heat templates') {
75 git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
76 }
77
78 stage('Install OpenStack CLI') {
79 openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
80 }
81
82 stage('Connect to OpenStack cloud') {
83 openstackCloud = openstack.createOpenstackEnv(
84 OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS,
85 OPENSTACK_API_PROJECT, OPENSTACK_API_PROJECT_DOMAIN,
86 OPENSTACK_API_PROJECT_ID, OPENSTACK_API_USER_DOMAIN,
87 OPENSTACK_API_VERSION)
88 openstack.getKeystoneToken(openstackCloud, openstackEnv)
89 wrap([$class: 'BuildUser']) {
90 if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !HEAT_STACK_REUSE.toBoolean()) {
91 def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv)
92 if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
93 HEAT_STACK_DELETE = "false"
94 throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
95 }
Filip Pytloun3eefd3d2017-03-03 14:13:41 +010096 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +040097 }
98 }
Filip Pytloun3eefd3d2017-03-03 14:13:41 +010099
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400100 if (HEAT_STACK_REUSE.toBoolean() == false) {
101 stage('Launch new Heat stack') {
102 envParams = [
Jakub Josefb03557a2017-08-09 17:23:10 +0200103 'cluster_zone': HEAT_STACK_ZONE,
104 'cluster_public_net': HEAT_STACK_PUBLIC_NET
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400105 ]
106 openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
107 }
108 }
109
110 stage('Connect to Salt master') {
111 def saltMasterPort
112 try {
113 saltMasterPort = SALT_MASTER_PORT
114 } catch (MissingPropertyException e) {
115 saltMasterPort = 6969
116 }
117 saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
118 currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}"
119 saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}"
Dmitrii Kabanovf31c8962017-10-12 21:00:30 -0700120 python.setupPepperVirtualenv(pepperEnv, saltMasterUrl, SALT_MASTER_CREDENTIALS)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400121 }
122
123 //
124 // Install
125 //
126
127 stage('Install core infra') {
128 // salt.master, reclass
129 // refresh_pillar
130 // sync_all
131 // linux,openssh,salt.minion.ntp
132
chnyda625f4b42017-10-11 14:10:31 +0200133 orchestrate.installFoundationInfra(pepperEnv)
134 orchestrate.validateFoundationInfra(pepperEnv)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400135 }
136
137 stage("Deploy GlusterFS") {
chnyda625f4b42017-10-11 14:10:31 +0200138 salt.enforceState(pepperEnv, 'I@glusterfs:server', 'glusterfs.server.service', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400139 retry(2) {
chnyda625f4b42017-10-11 14:10:31 +0200140 salt.enforceState(pepperEnv, 'ci01*', 'glusterfs.server.setup', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400141 }
142 sleep(5)
chnyda625f4b42017-10-11 14:10:31 +0200143 salt.enforceState(pepperEnv, 'I@glusterfs:client', 'glusterfs.client', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400144
145 timeout(5) {
146 println "Waiting for GlusterFS volumes to get mounted.."
chnyda625f4b42017-10-11 14:10:31 +0200147 salt.cmdRun(pepperEnv, 'I@glusterfs:client', 'while true; do systemctl -a|grep "GlusterFS File System"|grep -v mounted >/dev/null || break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400148 }
chnyda625f4b42017-10-11 14:10:31 +0200149 print common.prettyPrint(salt.cmdRun(pepperEnv, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"'))
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400150 }
151
152 stage("Deploy GlusterFS") {
chnyda625f4b42017-10-11 14:10:31 +0200153 salt.enforceState(pepperEnv, 'I@haproxy:proxy', 'haproxy,keepalived')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400154 }
155
156 stage("Setup Docker Swarm") {
chnyda625f4b42017-10-11 14:10:31 +0200157 salt.enforceState(pepperEnv, 'I@docker:host', 'docker.host', true)
158 salt.enforceState(pepperEnv, 'I@docker:swarm:role:master', 'docker.swarm', true)
159 salt.enforceState(pepperEnv, 'I@docker:swarm:role:master', 'salt', true)
160 salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master', 'mine.flush')
161 salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master', 'mine.update')
162 salt.enforceState(pepperEnv, 'I@docker:swarm', 'docker.swarm', true)
163 print common.prettyPrint(salt.cmdRun(pepperEnv, 'I@docker:swarm:role:master', 'docker node ls'))
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400164 }
165
166 stage("Configure OSS services") {
chnyda625f4b42017-10-11 14:10:31 +0200167 salt.enforceState(pepperEnv, 'I@devops_portal:config', 'devops_portal.config')
168 salt.enforceState(pepperEnv, 'I@rundeck:server', 'rundeck.server')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400169 }
170
171 stage("Deploy Docker services") {
172 // We need /etc/aptly-publisher.yaml to be present before
173 // services are deployed
174 // XXX: for some weird unknown reason, refresh_pillar is
175 // required to execute here
chnyda625f4b42017-10-11 14:10:31 +0200176 salt.runSaltProcessStep(pepperEnv, 'I@aptly:publisher', 'saltutil.refresh_pillar', [], null, true)
177 salt.enforceState(pepperEnv, 'I@aptly:publisher', 'aptly.publisher', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400178 retry(3) {
179 sleep(5)
chnyda625f4b42017-10-11 14:10:31 +0200180 salt.enforceState(pepperEnv, 'I@docker:swarm:role:master', 'docker.client')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400181 }
182 // XXX: Workaround to have `/var/lib/jenkins` on all
183 // nodes where are jenkins_slave services are created.
chnyda625f4b42017-10-11 14:10:31 +0200184 salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm', 'cmd.run', ['mkdir -p /var/lib/jenkins'])
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400185 }
186
187 stage("Configure CI/CD services") {
chnyda625f4b42017-10-11 14:10:31 +0200188 salt.syncAll(pepperEnv, '*')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400189
190 // Aptly
191 timeout(10) {
192 println "Waiting for Aptly to come up.."
193 retry(2) {
194 // XXX: retry to workaround magical VALUE_TRIMMED
195 // response from salt master + to give slow cloud some
196 // more time to settle down
chnyda625f4b42017-10-11 14:10:31 +0200197 salt.cmdRun(pepperEnv, 'I@aptly:server', 'while true; do curl -sf http://172.16.10.254:8084/api/version >/dev/null && break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400198 }
199 }
chnyda625f4b42017-10-11 14:10:31 +0200200 salt.enforceState(pepperEnv, 'I@aptly:server', 'aptly', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400201
202 // OpenLDAP
203 timeout(10) {
204 println "Waiting for OpenLDAP to come up.."
chnyda625f4b42017-10-11 14:10:31 +0200205 salt.cmdRun(pepperEnv, 'I@openldap:client', 'while true; do curl -sf ldap://172.16.10.254 >/dev/null && break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400206 }
chnyda625f4b42017-10-11 14:10:31 +0200207 salt.enforceState(pepperEnv, 'I@openldap:client', 'openldap', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400208
209 // Gerrit
210 timeout(10) {
211 println "Waiting for Gerrit to come up.."
chnyda625f4b42017-10-11 14:10:31 +0200212 salt.cmdRun(pepperEnv, 'I@gerrit:client', 'while true; do curl -sf 172.16.10.254:8080 >/dev/null && break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400213 }
chnyda625f4b42017-10-11 14:10:31 +0200214 salt.enforceState(pepperEnv, 'I@gerrit:client', 'gerrit', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400215
216 // Jenkins
217 timeout(10) {
218 println "Waiting for Jenkins to come up.."
chnyda625f4b42017-10-11 14:10:31 +0200219 salt.cmdRun(pepperEnv, 'I@jenkins:client', 'while true; do curl -sf 172.16.10.254:8081 >/dev/null && break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400220 }
221 retry(2) {
222 // XXX: needs retry as first run installs python-jenkins
223 // thus make jenkins modules available for second run
chnyda625f4b42017-10-11 14:10:31 +0200224 salt.enforceState(pepperEnv, 'I@jenkins:client', 'jenkins', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400225 }
226
227 // Postgres client - initialize OSS services databases
228 timeout(300){
229 println "Waiting for postgresql database to come up.."
chnyda625f4b42017-10-11 14:10:31 +0200230 salt.cmdRun(pepperEnv, 'I@postgresql:client', 'while true; do if docker service logs postgresql_postgresql-db | grep "ready to accept"; then break; else sleep 5; fi; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400231 }
Jakub Josef182d7bd2017-08-15 16:45:28 +0200232 // XXX: first run usually fails on some inserts, but we need to create databases at first
chnyda625f4b42017-10-11 14:10:31 +0200233 salt.enforceState(pepperEnv, 'I@postgresql:client', 'postgresql.client', true, false)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400234
235 // Setup postgres database with integration between
236 // Pushkin notification service and Security Monkey security audit service
237 timeout(10) {
238 println "Waiting for Pushkin to come up.."
chnyda625f4b42017-10-11 14:10:31 +0200239 salt.cmdRun(pepperEnv, 'I@postgresql:client', 'while true; do curl -sf 172.16.10.254:8887/apps >/dev/null && break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400240 }
chnyda625f4b42017-10-11 14:10:31 +0200241 salt.enforceState(pepperEnv, 'I@postgresql:client', 'postgresql.client', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400242
243 // Rundeck
244 timeout(10) {
245 println "Waiting for Rundeck to come up.."
chnyda625f4b42017-10-11 14:10:31 +0200246 salt.cmdRun(pepperEnv, 'I@rundeck:client', 'while true; do curl -sf 172.16.10.254:4440 >/dev/null && break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400247 }
chnyda625f4b42017-10-11 14:10:31 +0200248 salt.enforceState(pepperEnv, 'I@rundeck:client', 'rundeck.client', true)
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400249
250 // Elasticsearch
251 timeout(10) {
252 println 'Waiting for Elasticsearch to come up..'
chnyda625f4b42017-10-11 14:10:31 +0200253 salt.cmdRun(pepperEnv, 'I@elasticsearch:client', 'while true; do curl -sf 172.16.10.254:9200 >/dev/null && break; done')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400254 }
Jakub Josefd6381832017-08-15 17:47:45 +0200255 retry(3){
256 sleep(10)
Jakub Josef182d7bd2017-08-15 16:45:28 +0200257 // XXX: first run sometimes fails on update indexes, so we need to wait
chnyda625f4b42017-10-11 14:10:31 +0200258 salt.enforceState(pepperEnv, 'I@elasticsearch:client', 'elasticsearch.client', true)
Jakub Josef182d7bd2017-08-15 16:45:28 +0200259 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400260 }
261
262 stage("Finalize") {
263 //
264 // Deploy user's ssh key
265 //
266 def adminUser
267 def authorizedKeysFile
chnyda625f4b42017-10-11 14:10:31 +0200268 def adminUserCmdOut = salt.cmdRun(pepperEnv, 'I@salt:master', "[ ! -d /home/ubuntu ] || echo 'ubuntu user exists'")
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400269 if (adminUserCmdOut =~ /ubuntu user exists/) {
270 adminUser = "ubuntu"
271 authorizedKeysFile = "/home/ubuntu/.ssh/authorized_keys"
272 } else {
273 adminUser = "root"
274 authorizedKeysFile = "/root/.ssh/authorized_keys"
275 }
276
277 if (sshPubKey) {
278 println "Deploying provided ssh key at ${authorizedKeysFile}"
chnyda625f4b42017-10-11 14:10:31 +0200279 salt.cmdRun(pepperEnv, '*', "echo '${sshPubKey}' | tee -a ${authorizedKeysFile}")
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400280 }
281
282 //
283 // Generate docs
284 //
285 try {
Filip Pytlounbfce09d2017-03-01 19:00:43 +0100286 try {
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400287 // Run sphinx state to install sphinx-build needed in
288 // upcomming orchestrate
chnyda625f4b42017-10-11 14:10:31 +0200289 salt.enforceState(pepperEnv, 'I@sphinx:server', 'sphinx')
Filip Pytlounbfce09d2017-03-01 19:00:43 +0100290 } catch (Throwable e) {
Filip Pytlounbfce09d2017-03-01 19:00:43 +0100291 true
292 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400293 retry(3) {
294 // TODO: fix salt.orchestrateSystem
chnyda625f4b42017-10-11 14:10:31 +0200295 // print salt.orchestrateSystem(pepperEnv, ['expression': '*', 'type': 'compound'], 'sphinx.orch.generate_doc')
296 def out = salt.cmdRun(pepperEnv, 'I@salt:master', 'salt-run state.orchestrate sphinx.orch.generate_doc || echo "Command execution failed"')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400297 print common.prettyPrint(out)
298 if (out =~ /Command execution failed/) {
299 throw new Exception("Command execution failed")
300 }
Filip Pytlounbd619272017-03-22 12:21:01 +0100301 }
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400302 } catch (Throwable e) {
303 // We don't want sphinx docs to ruin whole build, so possible
304 // errors are just ignored here
305 true
Filip Pytlounbfce09d2017-03-01 19:00:43 +0100306 }
chnyda625f4b42017-10-11 14:10:31 +0200307 salt.enforceState(pepperEnv, 'I@nginx:server', 'nginx')
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400308
chnyda625f4b42017-10-11 14:10:31 +0200309 def failedSvc = salt.cmdRun(pepperEnv, '*', """systemctl --failed | grep -E 'loaded[ \t]+failed' && echo 'Command execution failed' || true""")
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400310 if (failedSvc =~ /Command execution failed/) {
311 common.errorMsg("Some services are not running. Environment may not be fully functional!")
312 }
313
314 common.successMsg("""
315============================================================
316Your CI/CD lab has been deployed and you can enjoy it:
317Use sshuttle to connect to your private subnet:
318
319 sshuttle -r ${adminUser}@${saltMasterHost} 172.16.10.0/24
320
321And visit services running at 172.16.10.254 (vip address):
322
323 9600 HAProxy statistics
324 8080 Gerrit
325 8081 Jenkins
326 8089 LDAP administration
327 4440 Rundeck
328 8084 DevOps Portal
329 8091 Docker swarm visualizer
330 8090 Reclass-generated documentation
331
332If you provided SSH_PUBLIC_KEY, you can use it to login,
333otherwise you need to get private key connected to this
334heat template.
335
336DON'T FORGET TO TERMINATE YOUR STACK WHEN YOU DON'T NEED IT!
337============================================================""")
338 }
339 } catch (Throwable e) {
340 // If there was an error or exception thrown, the build failed
341 currentBuild.result = "FAILURE"
Jakub Josefd2efd7d2017-08-22 17:49:57 +0200342 currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
Ruslan Kamaldinov6feef402017-08-02 16:55:58 +0400343 throw e
344 } finally {
345 // Cleanup
346 if (HEAT_STACK_DELETE.toBoolean() == true) {
347 stage('Trigger cleanup job') {
348 build(job: 'deploy-stack-cleanup', parameters: [
349 [$class: 'StringParameterValue', name: 'STACK_NAME', value: HEAT_STACK_NAME],
350 [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
351 ])
Filip Pytlounfd6726a2017-02-28 19:31:16 +0100352 }
Filip Pytloun23741982017-02-27 17:43:00 +0100353 }
Filip Pytlounf6e877f2017-02-28 19:38:16 +0100354 }
Filip Pytloun0a07f702017-02-24 18:26:18 +0100355}