| /** |
| * |
| * Launch heat stack with CI/CD lab infrastructure |
| * |
| * Expected parameters: |
| * HEAT_TEMPLATE_URL URL to git repo with Heat templates |
| * HEAT_TEMPLATE_CREDENTIALS Credentials to the Heat templates repo |
| * HEAT_TEMPLATE_BRANCH Heat templates repo branch |
| * HEAT_STACK_NAME Heat stack name |
| * HEAT_STACK_TEMPLATE Heat stack HOT template |
| * HEAT_STACK_ENVIRONMENT Heat stack environmental parameters |
| * HEAT_STACK_ZONE Heat stack availability zone |
| * HEAT_STACK_PUBLIC_NET Heat stack floating IP pool |
| * HEAT_STACK_DELETE Delete Heat stack when finished (bool) |
| * HEAT_STACK_CLEANUP_JOB Name of job for deleting Heat stack |
| * HEAT_STACK_REUSE Reuse Heat stack (don't create one) |
| * |
| * SALT_MASTER_CREDENTIALS Credentials to the Salt API |
| * SALT_MASTER_PORT Port of salt-api, defaults to 8000 |
| * |
| * OPENSTACK_API_URL OpenStack API address |
| * OPENSTACK_API_CREDENTIALS Credentials to the OpenStack API |
| * OPENSTACK_API_PROJECT OpenStack project to connect to |
| * OPENSTACK_API_CLIENT Versions of OpenStack python clients |
| * OPENSTACK_API_VERSION Version of the OpenStack API (2/3) |
| * |
| */ |
| |
| common = new com.mirantis.mk.Common() |
| git = new com.mirantis.mk.Git() |
| openstack = new com.mirantis.mk.Openstack() |
| salt = new com.mirantis.mk.Salt() |
| orchestrate = new com.mirantis.mk.Orchestrate() |
| def python = new com.mirantis.mk.Python() |
| |
| def pepperEnv = "pepperEnv" |
| _MAX_PERMITTED_STACKS = 2 |
| timeout(time: 12, unit: 'HOURS') { |
| node { |
| try { |
| // connection objects |
| def openstackCloud |
| |
| // value defaults |
| def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty' |
| def openstackEnv = "${env.WORKSPACE}/venv" |
| |
| try { |
| sshPubKey = SSH_PUBLIC_KEY |
| } catch (MissingPropertyException e) { |
| sshPubKey = false |
| } |
| |
| if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') { |
| error("If you want to reuse existing stack you need to provide it's name") |
| } |
| |
| if (HEAT_STACK_REUSE.toBoolean() == false) { |
| // Don't allow to set custom heat stack name |
| wrap([$class: 'BuildUser']) { |
| if (env.BUILD_USER_ID) { |
| HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}" |
| } else { |
| HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}" |
| } |
| currentBuild.description = HEAT_STACK_NAME |
| } |
| } |
| |
| // |
| // Bootstrap |
| // |
| |
| stage ('Download Heat templates') { |
| git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS) |
| } |
| |
| stage('Install OpenStack CLI') { |
| openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion) |
| } |
| |
| stage('Connect to OpenStack cloud') { |
| openstackCloud = openstack.createOpenstackEnv(openstackEnv, |
| OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, |
| OPENSTACK_API_PROJECT, OPENSTACK_API_PROJECT_DOMAIN, |
| OPENSTACK_API_PROJECT_ID, OPENSTACK_API_USER_DOMAIN, |
| OPENSTACK_API_VERSION) |
| openstack.getKeystoneToken(openstackCloud, openstackEnv) |
| wrap([$class: 'BuildUser']) { |
| if (env.BUILD_USER_ID && !env.BUILD_USER_ID.equals("jenkins") && !HEAT_STACK_REUSE.toBoolean()) { |
| def existingStacks = openstack.getStacksForNameContains(openstackCloud, "${env.BUILD_USER_ID}-${JOB_NAME}", openstackEnv) |
| if(existingStacks.size() >= _MAX_PERMITTED_STACKS){ |
| HEAT_STACK_DELETE = "false" |
| throw new Exception("You cannot create new stack, you already have ${_MAX_PERMITTED_STACKS} stacks of this type (${JOB_NAME}). \nStack names: ${existingStacks}") |
| } |
| } |
| } |
| } |
| |
| if (HEAT_STACK_REUSE.toBoolean() == false) { |
| stage('Launch new Heat stack') { |
| envParams = [ |
| 'cluster_zone': HEAT_STACK_ZONE, |
| 'cluster_public_net': HEAT_STACK_PUBLIC_NET |
| ] |
| openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv) |
| } |
| } |
| |
| stage('Connect to Salt master') { |
| def saltMasterPort |
| try { |
| saltMasterPort = SALT_MASTER_PORT |
| } catch (MissingPropertyException e) { |
| saltMasterPort = 6969 |
| } |
| saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv) |
| currentBuild.description = "${HEAT_STACK_NAME}: ${saltMasterHost}" |
| saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}" |
| python.setupPepperVirtualenv(pepperEnv, saltMasterUrl, SALT_MASTER_CREDENTIALS) |
| } |
| |
| // |
| // Install |
| // |
| |
| stage('Install core infra') { |
| // salt.master, reclass |
| // refresh_pillar |
| // sync_all |
| // linux,openssh,salt.minion.ntp |
| |
| orchestrate.installFoundationInfra(pepperEnv) |
| orchestrate.validateFoundationInfra(pepperEnv) |
| } |
| |
| stage("Deploy GlusterFS") { |
| salt.enforceState(pepperEnv, 'I@glusterfs:server', 'glusterfs.server.service', true) |
| retry(2) { |
| salt.enforceState(pepperEnv, 'ci01*', 'glusterfs.server.setup', true) |
| } |
| sleep(5) |
| salt.enforceState(pepperEnv, 'I@glusterfs:client', 'glusterfs.client', true) |
| |
| timeout(5) { |
| println "Waiting for GlusterFS volumes to get mounted.." |
| salt.cmdRun(pepperEnv, 'I@glusterfs:client', 'while true; do systemctl -a|grep "GlusterFS File System"|grep -v mounted >/dev/null || break; done') |
| } |
| print common.prettyPrint(salt.cmdRun(pepperEnv, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"')) |
| } |
| |
| stage("Deploy GlusterFS") { |
| salt.enforceState(pepperEnv, 'I@haproxy:proxy', 'haproxy,keepalived') |
| } |
| |
| stage("Setup Docker Swarm") { |
| salt.enforceState(pepperEnv, 'I@docker:host', 'docker.host', true) |
| salt.enforceState(pepperEnv, 'I@docker:swarm:role:master', 'docker.swarm', true) |
| salt.enforceState(pepperEnv, 'I@docker:swarm:role:master', 'salt', true) |
| salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master', 'mine.flush') |
| salt.runSaltProcessStep(pepperEnv, 'I@docker:swarm:role:master', 'mine.update') |
| salt.enforceState(pepperEnv, 'I@docker:swarm', 'docker.swarm', true) |
| print common.prettyPrint(salt.cmdRun(pepperEnv, 'I@docker:swarm:role:master', 'docker node ls')) |
| } |
| |
| stage("Configure OSS services") { |
| salt.enforceState(pepperEnv, 'I@devops_portal:config', 'devops_portal.config') |
| salt.enforceState(pepperEnv, 'I@rundeck:server', 'rundeck.server') |
| } |
| |
| stage("Deploy Docker services") { |
| // We need /etc/aptly-publisher.yaml to be present before |
| // services are deployed |
| // XXX: for some weird unknown reason, refresh_pillar is |
| // required to execute here |
| salt.runSaltProcessStep(pepperEnv, 'I@aptly:publisher', 'saltutil.refresh_pillar', [], null, true) |
| salt.enforceState(pepperEnv, 'I@aptly:publisher', 'aptly.publisher', true) |
| retry(3) { |
| sleep(5) |
| salt.enforceState(pepperEnv, 'I@docker:swarm:role:master', 'docker.client') |
| } |
| // XXX: Workaround to have `/var/lib/jenkins` on all |
| // nodes where are jenkins_slave services are created. |
| salt.cmdRun(pepperEnv, 'I@docker:swarm', "mkdir -p /var/lib/jenkins") |
| } |
| |
| stage("Configure CI/CD services") { |
| salt.syncAll(pepperEnv, '*') |
| |
| // Aptly |
| timeout(10) { |
| println "Waiting for Aptly to come up.." |
| retry(2) { |
| // XXX: retry to workaround magical VALUE_TRIMMED |
| // response from salt master + to give slow cloud some |
| // more time to settle down |
| salt.cmdRun(pepperEnv, 'I@aptly:server', 'while true; do curl -sf http://apt.mcp.mirantis.net:8081/api/version >/dev/null && break; done') |
| } |
| } |
| salt.enforceState(pepperEnv, 'I@aptly:server', 'aptly', true) |
| |
| // OpenLDAP |
| timeout(10) { |
| println "Waiting for OpenLDAP to come up.." |
| salt.cmdRun(pepperEnv, 'I@openldap:client', 'while true; do curl -sf ldap://172.16.10.254 >/dev/null && break; done') |
| } |
| salt.enforceState(pepperEnv, 'I@openldap:client', 'openldap', true) |
| |
| // Gerrit |
| timeout(10) { |
| println "Waiting for Gerrit to come up.." |
| salt.cmdRun(pepperEnv, 'I@gerrit:client', 'while true; do curl -sf 172.16.10.254:8080 >/dev/null && break; done') |
| } |
| salt.enforceState(pepperEnv, 'I@gerrit:client', 'gerrit', true) |
| |
| // Jenkins |
| timeout(10) { |
| println "Waiting for Jenkins to come up.." |
| salt.cmdRun(pepperEnv, 'I@jenkins:client', 'while true; do curl -sf 172.16.10.254:8081 >/dev/null && break; done') |
| } |
| retry(2) { |
| // XXX: needs retry as first run installs python-jenkins |
| // thus make jenkins modules available for second run |
| salt.enforceState(pepperEnv, 'I@jenkins:client', 'jenkins', true) |
| } |
| |
| // Postgres client - initialize OSS services databases |
| timeout(300){ |
| println "Waiting for postgresql database to come up.." |
| salt.cmdRun(pepperEnv, 'I@postgresql:client', 'while true; do if docker service logs postgresql_postgresql-db | grep "ready to accept"; then break; else sleep 5; fi; done') |
| } |
| // XXX: first run usually fails on some inserts, but we need to create databases at first |
| salt.enforceState(pepperEnv, 'I@postgresql:client', 'postgresql.client', true, false) |
| |
| // Setup postgres database with integration between |
| // Pushkin notification service and Security Monkey security audit service |
| timeout(10) { |
| println "Waiting for Pushkin to come up.." |
| salt.cmdRun(pepperEnv, 'I@postgresql:client', 'while true; do curl -sf 172.16.10.254:8887/apps >/dev/null && break; done') |
| } |
| salt.enforceState(pepperEnv, 'I@postgresql:client', 'postgresql.client', true) |
| |
| // Rundeck |
| timeout(10) { |
| println "Waiting for Rundeck to come up.." |
| salt.cmdRun(pepperEnv, 'I@rundeck:client', 'while true; do curl -sf 172.16.10.254:4440 >/dev/null && break; done') |
| } |
| salt.enforceState(pepperEnv, 'I@rundeck:client', 'rundeck.client', true) |
| |
| // Elasticsearch |
| timeout(10) { |
| println 'Waiting for Elasticsearch to come up..' |
| salt.cmdRun(pepperEnv, 'I@elasticsearch:client', 'while true; do curl -sf 172.16.10.254:9200 >/dev/null && break; done') |
| } |
| retry(3){ |
| sleep(10) |
| // XXX: first run sometimes fails on update indexes, so we need to wait |
| salt.enforceState(pepperEnv, 'I@elasticsearch:client', 'elasticsearch.client', true) |
| } |
| } |
| |
| stage("Finalize") { |
| // |
| // Deploy user's ssh key |
| // |
| def adminUser |
| def authorizedKeysFile |
| def adminUserCmdOut = salt.cmdRun(pepperEnv, 'I@salt:master', "[ ! -d /home/ubuntu ] || echo 'ubuntu user exists'") |
| if (adminUserCmdOut =~ /ubuntu user exists/) { |
| adminUser = "ubuntu" |
| authorizedKeysFile = "/home/ubuntu/.ssh/authorized_keys" |
| } else { |
| adminUser = "root" |
| authorizedKeysFile = "/root/.ssh/authorized_keys" |
| } |
| |
| if (sshPubKey) { |
| println "Deploying provided ssh key at ${authorizedKeysFile}" |
| salt.cmdRun(pepperEnv, '*', "echo '${sshPubKey}' | tee -a ${authorizedKeysFile}") |
| } |
| |
| // |
| // Generate docs |
| // |
| try { |
| try { |
| // Run sphinx state to install sphinx-build needed in |
| // upcomming orchestrate |
| salt.enforceState(pepperEnv, 'I@sphinx:server', 'sphinx') |
| } catch (Throwable e) { |
| true |
| } |
| retry(3) { |
| // TODO: fix salt.orchestrateSystem |
| // print salt.orchestrateSystem(pepperEnv, ['expression': '*', 'type': 'compound'], 'sphinx.orch.generate_doc') |
| def out = salt.cmdRun(pepperEnv, 'I@salt:master', 'salt-run state.orchestrate sphinx.orch.generate_doc || echo "Command execution failed"') |
| print common.prettyPrint(out) |
| if (out =~ /Command execution failed/) { |
| throw new Exception("Command execution failed") |
| } |
| } |
| } catch (Throwable e) { |
| // We don't want sphinx docs to ruin whole build, so possible |
| // errors are just ignored here |
| true |
| } |
| salt.enforceState(pepperEnv, 'I@nginx:server', 'nginx') |
| |
| def failedSvc = salt.cmdRun(pepperEnv, '*', """systemctl --failed | grep -E 'loaded[ \t]+failed' && echo 'Command execution failed' || true""") |
| if (failedSvc =~ /Command execution failed/) { |
| common.errorMsg("Some services are not running. Environment may not be fully functional!") |
| } |
| |
| common.successMsg(""" |
| ============================================================ |
| Your CI/CD lab has been deployed and you can enjoy it: |
| Use sshuttle to connect to your private subnet: |
| |
| sshuttle -r ${adminUser}@${saltMasterHost} 172.16.10.0/24 |
| |
| And visit services running at 172.16.10.254 (vip address): |
| |
| 9600 HAProxy statistics |
| 8080 Gerrit |
| 8081 Jenkins |
| 8089 LDAP administration |
| 4440 Rundeck |
| 8084 DevOps Portal |
| 8091 Docker swarm visualizer |
| 8090 Reclass-generated documentation |
| |
| If you provided SSH_PUBLIC_KEY, you can use it to login, |
| otherwise you need to get private key connected to this |
| heat template. |
| |
| DON'T FORGET TO TERMINATE YOUR STACK WHEN YOU DON'T NEED IT! |
| ============================================================""") |
| } |
| } catch (Throwable e) { |
| // If there was an error or exception thrown, the build failed |
| currentBuild.result = "FAILURE" |
| currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message |
| throw e |
| } finally { |
| // Cleanup |
| if (HEAT_STACK_DELETE.toBoolean() == true) { |
| stage('Trigger cleanup job') { |
| build(job: 'deploy-stack-cleanup', parameters: [ |
| [$class: 'StringParameterValue', name: 'STACK_NAME', value: HEAT_STACK_NAME], |
| [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT], |
| ]) |
| } |
| } |
| } |
| } |
| } |