blob: 2ceba6d0c894b4c5187e7a4109f5d176d1e34282 [file] [log] [blame]
Ales Komarek24097402017-05-22 16:28:10 +02001/**
2 *
3 * Launch heat/cloudformation stack
4 *
5 * Expected parameters:
6 * STACK_NAME Infrastructure stack name
7 * STACK_TEMPLATE Stack HOT/CFN template
8 * STACK_TYPE Deploy OpenStack/AWS [heat/aws]
9 *
10 * STACK_TEMPLATE_URL URL to git repo with stack templates
11 * STACK_TEMPLATE_CREDENTIALS Credentials to the templates repo
12 * STACK_TEMPLATE_BRANCH Stack templates repo branch
13 *
14 * STACK_DELETE Delete stack when finished (bool)
15 * STACK_REUSE Reuse existing stack (don't create one)
16 * STACK_INSTALL What should be installed (k8s, openstack, ...)
17 * STACK_TEST Run tests (bool)
18 * STACK_CLEANUP_JOB Name of job for deleting stack
19 *
Tomáš Kukrál74f92862017-06-05 19:18:43 +020020 * STACK_COMPUTE_COUNT Number of compute nodes to launch
21 *
Ales Komarek24097402017-05-22 16:28:10 +020022 * AWS_STACK_REGION CloudFormation AWS region
23 * AWS_API_CREDENTIALS AWS Access key ID with AWS secret access key
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +020024 * AWS_SSH_KEY AWS key pair name (used for SSH access)
Ales Komarek24097402017-05-22 16:28:10 +020025 *
26 * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters
27 * HEAT_STACK_ZONE Heat stack availability zone
28 * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool
29 * OPENSTACK_API_URL OpenStack API address
30 * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API
31 * OPENSTACK_API_PROJECT OpenStack project to connect to
32 * OPENSTACK_API_CLIENT Versions of OpenStack python clients
33 * OPENSTACK_API_VERSION Version of the OpenStack API (2/3)
34 *
35 * SALT_MASTER_CREDENTIALS Credentials to the Salt API
36 * SALT_MASTER_URL URL of Salt master
37 *
38 * K8S_API_SERVER Kubernetes API address
39 * K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
Matthew Mosesohn13933442017-07-04 21:20:08 +030040 * SALT_OVERRIDES YAML with overrides for Salt deployment
Ales Komarek24097402017-05-22 16:28:10 +020041 *
42 * TEMPEST_IMAGE_LINK Tempest image link
43 *
44 */
45common = new com.mirantis.mk.Common()
46git = new com.mirantis.mk.Git()
47openstack = new com.mirantis.mk.Openstack()
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +020048aws = new com.mirantis.mk.Aws()
Ales Komarek24097402017-05-22 16:28:10 +020049orchestrate = new com.mirantis.mk.Orchestrate()
50salt = new com.mirantis.mk.Salt()
51test = new com.mirantis.mk.Test()
52
53_MAX_PERMITTED_STACKS = 2
54overwriteFile = "/srv/salt/reclass/classes/cluster/override.yml"
55
Tomáš Kukrála1d86b82017-07-10 17:24:01 +020056// Define global variables
Tomáš Kukrál31414a22017-06-09 14:17:53 +020057def saltMaster
Tomáš Kukrála1d86b82017-07-10 17:24:01 +020058def venv
Tomáš Kukrálbc4be252017-06-06 11:28:40 +020059
Tomáš Kukrálb19bdda2017-06-13 17:09:43 +020060if (STACK_TYPE == 'aws') {
Tomáš Kukrála1d86b82017-07-10 17:24:01 +020061 def aws_env_vars
Tomáš Kukrálb19bdda2017-06-13 17:09:43 +020062}
63
Ales Komarek24097402017-05-22 16:28:10 +020064timestamps {
65 node {
66 try {
Tomáš Kukrála1d86b82017-07-10 17:24:01 +020067 // Set build-specific variables
68 venv = "${env.WORKSPACE}/venv"
69
Ales Komarek24097402017-05-22 16:28:10 +020070 //
71 // Prepare machines
72 //
73 stage ('Create infrastructure') {
74
75 if (STACK_TYPE == 'heat') {
76 // value defaults
77 def openstackCloud
78 def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
Ales Komarek24097402017-05-22 16:28:10 +020079
80 if (STACK_REUSE.toBoolean() == true && STACK_NAME == '') {
81 error("If you want to reuse existing stack you need to provide it's name")
82 }
83
84 if (STACK_REUSE.toBoolean() == false) {
85 // Don't allow to set custom heat stack name
86 wrap([$class: 'BuildUser']) {
87 if (env.BUILD_USER_ID) {
88 STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
89 } else {
90 STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
91 }
92 currentBuild.description = STACK_NAME
93 }
94 }
95
96 // set description
97 currentBuild.description = "${STACK_NAME}"
98
99 // get templates
100 git.checkoutGitRepository('template', STACK_TEMPLATE_URL, STACK_TEMPLATE_BRANCH, STACK_TEMPLATE_CREDENTIALS)
101
102 // create openstack env
Tomáš Kukrál8fa9b672017-07-10 14:22:19 +0200103 openstack.setupOpenstackVirtualenv(venv, openstackVersion)
Ales Komarek24097402017-05-22 16:28:10 +0200104 openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
Tomáš Kukrál8fa9b672017-07-10 14:22:19 +0200105 openstack.getKeystoneToken(openstackCloud, venv)
Ales Komarek24097402017-05-22 16:28:10 +0200106 //
107 // Verify possibility of create stack for given user and stack type
108 //
109 wrap([$class: 'BuildUser']) {
110 if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !STACK_REUSE.toBoolean()) {
Tomáš Kukrál8fa9b672017-07-10 14:22:19 +0200111 def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", venv)
Ales Komarek24097402017-05-22 16:28:10 +0200112 if(existingStacks.size() >= _MAX_PERMITTED_STACKS){
113 STACK_DELETE = "false"
114 throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}")
115 }
116 }
117 }
118 // launch stack
119 if (STACK_REUSE.toBoolean() == false) {
120 stage('Launch new Heat stack') {
121 // create stack
122 envParams = [
123 'instance_zone': HEAT_STACK_ZONE,
124 'public_net': HEAT_STACK_PUBLIC_NET
125 ]
Tomáš Kukrál8fa9b672017-07-10 14:22:19 +0200126 openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, venv, false)
Ales Komarek24097402017-05-22 16:28:10 +0200127 }
128 }
129
130 // get SALT_MASTER_URL
Tomáš Kukrál8fa9b672017-07-10 14:22:19 +0200131 saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, STACK_NAME, 'salt_master_ip', venv)
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200132 currentBuild.description = "${STACK_NAME} ${saltMasterHost}"
Ales Komarek24097402017-05-22 16:28:10 +0200133
134 SALT_MASTER_URL = "http://${saltMasterHost}:6969"
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200135 } else if (STACK_TYPE == 'aws') {
Ales Komarek24097402017-05-22 16:28:10 +0200136
Tomáš Kukrál9af83062017-07-13 10:00:54 +0200137 // setup environment
138 aws.setupVirtualEnv(venv)
139
Tomáš Kukrála1d86b82017-07-10 17:24:01 +0200140 // set aws_env_vars
141 aws_env_vars = aws.getEnvVars(AWS_API_CREDENTIALS, AWS_STACK_REGION)
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200142
143 if (STACK_REUSE.toBoolean() == true && STACK_NAME == '') {
144 error("If you want to reuse existing stack you need to provide it's name")
145 }
146
147 if (STACK_REUSE.toBoolean() == false) {
148 // Don't allow to set custom stack name
149 wrap([$class: 'BuildUser']) {
150 if (env.BUILD_USER_ID) {
151 STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
152 } else {
153 STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
154 }
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200155 }
Tomáš Kukráldbf8ca82017-05-25 15:13:20 +0200156
157 // no underscore in STACK_NAME
158 STACK_NAME = STACK_NAME.replaceAll('_', '-')
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200159 }
160
161 // set description
Tomáš Kukráldbf8ca82017-05-25 15:13:20 +0200162 currentBuild.description = STACK_NAME
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200163
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200164 if (STACK_REUSE.toBoolean() == false) {
165 // get templates
166 git.checkoutGitRepository('template', STACK_TEMPLATE_URL, STACK_TEMPLATE_BRANCH, STACK_TEMPLATE_CREDENTIALS)
167
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200168 // start stack
Tomáš Kukrálad79d322017-06-05 15:40:14 +0200169 def stack_params = [
170 "ParameterKey=KeyName,ParameterValue=" + AWS_SSH_KEY,
171 "ParameterKey=CmpNodeCount,ParameterValue=" + STACK_COMPUTE_COUNT
172 ]
Tomáš Kukráldbb58402017-06-02 07:08:53 +0200173 def template_file = 'cfn/' + STACK_TEMPLATE + '.yml'
Tomáš Kukrála1d86b82017-07-10 17:24:01 +0200174 aws.createStack(venv, aws_env_vars, template_file, STACK_NAME, stack_params)
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200175 }
176
177 // wait for stack to be ready
Tomáš Kukrála1d86b82017-07-10 17:24:01 +0200178 aws.waitForStatus(venv, aws_env_vars, STACK_NAME, 'CREATE_COMPLETE')
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200179
180 // get outputs
Tomáš Kukrála1d86b82017-07-10 17:24:01 +0200181 saltMasterHost = aws.getOutputs(venv, aws_env_vars, STACK_NAME, 'SaltMasterIP')
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200182 currentBuild.description = "${STACK_NAME} ${saltMasterHost}"
Ales Komarek24097402017-05-22 16:28:10 +0200183 SALT_MASTER_URL = "http://${saltMasterHost}:6969"
Ales Komarek24097402017-05-22 16:28:10 +0200184
Tomáš Kukrál7a8a4fa2017-05-25 14:54:24 +0200185 } else {
186 throw new Exception("STACK_TYPE ${STACK_TYPE} is not supported")
187 }
Ales Komarek24097402017-05-22 16:28:10 +0200188
Tomáš Kukrál20707642017-06-09 13:35:13 +0200189 // Connect to Salt master
Ales Komarek24097402017-05-22 16:28:10 +0200190 saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
191 }
192
Matthew Mosesohn13933442017-07-04 21:20:08 +0300193 // Set up override params
194 if (env.getEnvironment().containsKey('SALT_OVERRIDES')) {
195 stage('Set Salt overrides') {
196 salt.setSaltOverrides(saltMaster, SALT_OVERRIDES)
197 }
198 }
199
Ales Komarek24097402017-05-22 16:28:10 +0200200 //
201 // Install
202 //
203
204 if (common.checkContains('STACK_INSTALL', 'core')) {
205 stage('Install core infrastructure') {
206 orchestrate.installFoundationInfra(saltMaster)
207
208 if (common.checkContains('STACK_INSTALL', 'kvm')) {
209 orchestrate.installInfraKvm(saltMaster)
210 orchestrate.installFoundationInfra(saltMaster)
211 }
212
213 orchestrate.validateFoundationInfra(saltMaster)
214 }
215 }
216
217 // install k8s
218 if (common.checkContains('STACK_INSTALL', 'k8s')) {
219 stage('Install Kubernetes infra') {
Tomáš Kukrál39c96cd2017-07-11 10:14:13 +0200220 if (STACK_TYPE == 'aws') {
221 // configure kubernetes_control_address - save loadbalancer
222 def kubernetes_control_address = aws.getOutputs(venv, aws_env_vars, STACK_NAME, 'ControlLoadBalancer')
223 print(kubernetes_control_address)
224 salt.runSaltProcessStep(saltMaster, 'I@salt:master', 'reclass.cluster_meta_set', ['kubernetes_control_address', kubernetes_control_address], null, true)
225 }
Tomáš Kukrálb19bdda2017-06-13 17:09:43 +0200226
Tomáš Kukrál4898be32017-06-19 09:54:15 +0200227 // ensure certificates are generated properly
228 salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
229 salt.enforceState(saltMaster, '*', ['salt.minion.cert'], true)
230
Ales Komarek24097402017-05-22 16:28:10 +0200231 orchestrate.installKubernetesInfra(saltMaster)
232 }
233
234 stage('Install Kubernetes control') {
235
Ales Komarek24097402017-05-22 16:28:10 +0200236 orchestrate.installKubernetesControl(saltMaster)
237 }
238
Tomáš Kukrál74f92862017-06-05 19:18:43 +0200239 stage('Scale Kubernetes computes') {
Tomáš Kukrálb9a7d6f2017-06-07 14:05:31 +0200240 if (STACK_COMPUTE_COUNT > 0) {
241 if (STACK_TYPE == 'aws') {
Tomáš Kukrálb9a7d6f2017-06-07 14:05:31 +0200242 // get stack info
Tomáš Kukrála1d86b82017-07-10 17:24:01 +0200243 def scaling_group = aws.getOutputs(venv, aws_env_vars, STACK_NAME, 'ComputesScalingGroup')
Tomáš Kukrál74f92862017-06-05 19:18:43 +0200244
Tomáš Kukrálb9a7d6f2017-06-07 14:05:31 +0200245 //update autoscaling group
Tomáš Kukrála1d86b82017-07-10 17:24:01 +0200246 aws.updateAutoscalingGroup(venv, aws_env_vars, scaling_group, ["--desired-capacity " + STACK_COMPUTE_COUNT])
Tomáš Kukrál8130a562017-06-06 11:09:22 +0200247
Tomáš Kukrálb9a7d6f2017-06-07 14:05:31 +0200248 // wait for computes to boot up
Tomáš Kukrála1d86b82017-07-10 17:24:01 +0200249 aws.waitForAutoscalingInstances(venv, aws_env_vars, scaling_group)
Tomáš Kukrálf31a5912017-06-13 15:47:25 +0200250 sleep(60)
Tomáš Kukrálb9a7d6f2017-06-07 14:05:31 +0200251 }
252
253 orchestrate.installKubernetesCompute(saltMaster)
Tomáš Kukrál74f92862017-06-05 19:18:43 +0200254 }
Tomáš Kukrál74f92862017-06-05 19:18:43 +0200255 }
Ales Komarek24097402017-05-22 16:28:10 +0200256
257 if (common.checkContains('STACK_INSTALL', 'contrail')) {
Tomáš Kukrál391df0d2017-06-13 10:37:02 +0200258 stage('Install Contrail for Kubernetes') {
Ales Komarek24097402017-05-22 16:28:10 +0200259 orchestrate.installContrailNetwork(saltMaster)
260 orchestrate.installContrailCompute(saltMaster)
261 }
262 }
263 }
264
265 // install openstack
266 if (common.checkContains('STACK_INSTALL', 'openstack')) {
267 // install Infra and control, tests, ...
268
269 stage('Install OpenStack infra') {
270 orchestrate.installOpenstackInfra(saltMaster)
271 }
272
273 stage('Install OpenStack control') {
274 orchestrate.installOpenstackControl(saltMaster)
275 }
276
277 stage('Install OpenStack network') {
278
279 if (common.checkContains('STACK_INSTALL', 'contrail')) {
280 orchestrate.installContrailNetwork(saltMaster)
281 } else if (common.checkContains('STACK_INSTALL', 'ovs')) {
282 orchestrate.installOpenstackNetwork(saltMaster)
283 }
284
285 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
286 salt.runSaltProcessStep(saltMaster, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
287 }
288
289 stage('Install OpenStack compute') {
290 orchestrate.installOpenstackCompute(saltMaster)
291
292 if (common.checkContains('STACK_INSTALL', 'contrail')) {
293 orchestrate.installContrailCompute(saltMaster)
294 }
295 }
296
297 }
298
299
Martin Polreich46971102017-07-11 12:56:40 +0200300 if (common.checkContains('STACK_INSTALL', 'sl-legacy')) {
301 stage('Install StackLight v1') {
302 orchestrate.installStacklightv1Control(saltMaster)
303 orchestrate.installStacklightv1Client(saltMaster)
304 }
305 }
306
Ales Komarek24097402017-05-22 16:28:10 +0200307 if (common.checkContains('STACK_INSTALL', 'stacklight')) {
308 stage('Install StackLight') {
Martin Polreich46971102017-07-11 12:56:40 +0200309 orchestrate.installDockerSwarm(saltMaster)
310 orchestrate.installStacklight(saltMaster)
Ales Komarek24097402017-05-22 16:28:10 +0200311 }
312 }
313
314 //
315 // Test
316 //
317 def artifacts_dir = '_artifacts/'
318
319 if (common.checkContains('STACK_TEST', 'k8s')) {
320 stage('Run k8s bootstrap tests') {
321 def image = 'tomkukral/k8s-scripts'
322 def output_file = image.replaceAll('/', '-') + '.output'
323
324 // run image
325 test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
326
327 // collect output
328 sh "mkdir -p ${artifacts_dir}"
329 file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
330 writeFile file: "${artifacts_dir}${output_file}", text: file_content
331 sh "cat ${artifacts_dir}${output_file}"
332
333 // collect artifacts
334 archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
335 }
336
337 stage('Run k8s conformance e2e tests') {
338 //test.runConformanceTests(saltMaster, K8S_API_SERVER, K8S_CONFORMANCE_IMAGE)
339
340 def image = K8S_CONFORMANCE_IMAGE
341 def output_file = image.replaceAll('/', '-') + '.output'
342
343 // run image
344 test.runConformanceTests(saltMaster, K8S_API_SERVER, image)
345
346 // collect output
347 sh "mkdir -p ${artifacts_dir}"
348 file_content = salt.getFileContent(saltMaster, 'ctl01*', '/tmp/' + output_file)
349 writeFile file: "${artifacts_dir}${output_file}", text: file_content
350 sh "cat ${artifacts_dir}${output_file}"
351
352 // collect artifacts
353 archiveArtifacts artifacts: "${artifacts_dir}${output_file}"
354 }
355 }
356
357 if (common.checkContains('STACK_TEST', 'openstack')) {
358 stage('Run deployment tests') {
359 test.runTempestTests(saltMaster, TEMPEST_IMAGE_LINK)
360 }
361
362 stage('Copy test results to config node') {
363 test.copyTempestResults(saltMaster)
364 }
365 }
366
367 stage('Finalize') {
368 if (STACK_INSTALL != '') {
369 try {
370 salt.runSaltProcessStep(saltMaster, '*', 'state.apply', [], null, true)
371 } catch (Exception e) {
372 common.warningMsg('State apply failed but we should continue to run')
373 }
374 }
375 }
376 } catch (Throwable e) {
377 currentBuild.result = 'FAILURE'
378 throw e
379 } finally {
380
381
382 //
383 // Clean
384 //
385
Tomáš Kukrál439c62e2017-06-02 11:42:36 +0200386 if (STACK_NAME && STACK_NAME != '') {
Ales Komarek24097402017-05-22 16:28:10 +0200387 // send notification
388 common.sendNotification(currentBuild.result, STACK_NAME, ["slack"])
Tomáš Kukrál439c62e2017-06-02 11:42:36 +0200389 }
Ales Komarek24097402017-05-22 16:28:10 +0200390
Tomáš Kukrál439c62e2017-06-02 11:42:36 +0200391 if (STACK_DELETE.toBoolean() == true) {
392 stage('Trigger cleanup job') {
393 common.errorMsg('Stack cleanup job triggered')
394 build(job: STACK_CLEANUP_JOB, parameters: [
395 [$class: 'StringParameterValue', name: 'STACK_NAME', value: STACK_NAME],
396 [$class: 'StringParameterValue', name: 'STACK_TYPE', value: STACK_TYPE]
397 ])
398 }
399 } else {
400 if (currentBuild.result == 'FAILURE') {
401 common.errorMsg("Deploy job FAILED and was not deleted. Please fix the problem and delete stack on you own.")
402 if (SALT_MASTER_URL) {
403 common.errorMsg("Salt master URL: ${SALT_MASTER_URL}")
Ales Komarek24097402017-05-22 16:28:10 +0200404 }
405 }
406 }
407 }
408 }
409}