cicd-lab-pipeline.groovy

/**
 *
 * Launch heat stack with CI/CD lab infrastructure
 *
 * Expected parameters:
 *   HEAT_TEMPLATE_URL          URL to git repo with Heat templates
 *   HEAT_TEMPLATE_CREDENTIALS  Credentials to the Heat templates repo
 *   HEAT_TEMPLATE_BRANCH       Heat templates repo branch
 *   HEAT_STACK_NAME            Heat stack name
 *   HEAT_STACK_TEMPLATE        Heat stack HOT template
 *   HEAT_STACK_ENVIRONMENT     Heat stack environmental parameters
 *   HEAT_STACK_ZONE            Heat stack availability zone
 *   HEAT_STACK_PUBLIC_NET      Heat stack floating IP pool
 *   HEAT_STACK_DELETE          Delete Heat stack when finished (bool)
 *   HEAT_STACK_CLEANUP_JOB     Name of job for deleting Heat stack
 *   HEAT_STACK_REUSE           Reuse Heat stack (don't create one)
 *
 *   SALT_MASTER_CREDENTIALS    Credentials to the Salt API
 *   SALT_MASTER_PORT           Port of salt-api, defaults to 8000
 *
 *   OPENSTACK_API_URL          OpenStack API address
 *   OPENSTACK_API_CREDENTIALS  Credentials to the OpenStack API
 *   OPENSTACK_API_PROJECT      OpenStack project to connect to
 *   OPENSTACK_API_CLIENT       Versions of OpenStack python clients
 *   OPENSTACK_API_VERSION      Version of the OpenStack API (2/3)
 *
 */

common = new com.mirantis.mk.Common()
git = new com.mirantis.mk.Git()
openstack = new com.mirantis.mk.Openstack()
salt = new com.mirantis.mk.Salt()
orchestrate = new com.mirantis.mk.Orchestrate()

def waitForServices(saltMaster) {
    retry(30) {
        out = salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', """/bin/bash -c 'docker service ls | grep -E "0/[0-9]+"' && echo 'Some services are not running'""")
        for (int a = 0; a < out['return'].size(); a++) {
            def entry = out['return'].get(a)
            for (int i = 0; i < entry.size(); i++) {
                def node = entry.get(i)
                if (node) {
                    if (node.value =~ /Some services are not running/) {
                        sleep(10)
                        throw new Exception("$node.key: $node.value")
                    } else {
                        print out
                    }
                }
            }
        }
    }
}

timestamps {
    node {
        try {
            // connection objects
            def openstackCloud
            def saltMaster

            // value defaults
            def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
            def openstackEnv = "${env.WORKSPACE}/venv"

            try {
                sshPubKey = SSH_PUBLIC_KEY
            } catch (MissingPropertyException e) {
                sshPubKey = false
            }

            if (HEAT_STACK_REUSE.toBoolean() == true && HEAT_STACK_NAME == '') {
                error("If you want to reuse existing stack you need to provide it's name")
            }

            if (HEAT_STACK_REUSE.toBoolean() == false) {
                // Don't allow to set custom heat stack name
                wrap([$class: 'BuildUser']) {
                    if (env.BUILD_USER_ID) {
                        HEAT_STACK_NAME = "${env.BUILD_USER_ID}-${JOB_NAME}-${BUILD_NUMBER}"
                    } else {
                        HEAT_STACK_NAME = "jenkins-${JOB_NAME}-${BUILD_NUMBER}"
                    }
                    currentBuild.description = HEAT_STACK_NAME
                }
            }

            //
            // Bootstrap
            //

            stage ('Download Heat templates') {
                git.checkoutGitRepository('template', HEAT_TEMPLATE_URL, HEAT_TEMPLATE_BRANCH, HEAT_TEMPLATE_CREDENTIALS)
            }

            stage('Install OpenStack CLI') {
                openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
            }

            stage('Connect to OpenStack cloud') {
                openstackCloud = openstack.createOpenstackEnv(OPENSTACK_API_URL, OPENSTACK_API_CREDENTIALS, OPENSTACK_API_PROJECT)
                openstack.getKeystoneToken(openstackCloud, openstackEnv)
            }

            if (HEAT_STACK_REUSE.toBoolean() == false) {
                stage('Launch new Heat stack') {
                    envParams = [
                        'instance_zone': HEAT_STACK_ZONE,
                        'public_net': HEAT_STACK_PUBLIC_NET
                    ]
                    openstack.createHeatStack(openstackCloud, HEAT_STACK_NAME, HEAT_STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, openstackEnv)
                }
            }

            stage('Connect to Salt master') {
                def saltMasterPort
                try {
                    saltMasterPort = SALT_MASTER_PORT
                } catch (MissingPropertyException e) {
                    saltMasterPort = 8000
                }
                saltMasterHost = openstack.getHeatStackOutputParam(openstackCloud, HEAT_STACK_NAME, 'salt_master_ip', openstackEnv)
                saltMasterUrl = "http://${saltMasterHost}:${saltMasterPort}"
                saltMaster = salt.connection(saltMasterUrl, SALT_MASTER_CREDENTIALS)
            }

            //
            // Install
            //

            stage('Install core infra') {
                // salt.master, reclass
                // refresh_pillar
                // sync_all
                // linux,openssh,salt.minion.ntp

                orchestrate.installFoundationInfra(saltMaster)
                orchestrate.validateFoundationInfra(saltMaster)
            }

            stage("Deploy GlusterFS") {
                salt.enforceState(saltMaster, 'I@glusterfs:server', 'glusterfs.server.service', true)
                salt.enforceState(saltMaster, 'ci01*', 'glusterfs.server.setup', true)
                sleep(5)
                salt.enforceState(saltMaster, 'I@glusterfs:client', 'glusterfs.client', true)
                print salt.cmdRun(saltMaster, 'I@glusterfs:client', 'mount|grep fuse.glusterfs || echo "Command failed"')
            }

            stage("Deploy GlusterFS") {
                salt.enforceState(saltMaster, 'I@haproxy:proxy', 'haproxy,keepalived')
            }

            stage("Setup Docker Swarm") {
                salt.enforceState(saltMaster, 'I@docker:host', 'docker.host', true)
                salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.swarm', true)
                salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'salt', true)
                salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.flush')
                salt.runSaltProcessStep(saltMaster, 'I@docker:swarm:role:master', 'mine.update')
                salt.enforceState(saltMaster, 'I@docker:swarm', 'docker.swarm', true)
                print salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'docker node ls')
            }

            stage("Deploy Docker services") {
                salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')

                // XXX: Hack to fix dependency of gerrit on mysql
                print salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', "docker service rm gerrit; sleep 5; rm -rf /srv/volumes/gerrit/*")
                waitForServices(saltMaster)

                timeout(10) {
                    salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'apt-get install -y mysql-client')
                    println "Waiting for MySQL to come up.."
                    salt.cmdRun(saltMaster, 'I@docker:swarm:role:master', 'while true; do mysql -h172.16.10.254 -ppassword -e"show status;" >/dev/null && break; done')
                }
                salt.enforceState(saltMaster, 'I@docker:swarm:role:master', 'docker.client')
                // ---- cut here (end of hack) ----

                waitForServices(saltMaster)
            }

            stage("Configure CI/CD services") {
                // Aptly
                salt.enforceState(saltMaster, 'I@aptly:server', 'aptly', true)

                // Gerrit
                timeout(10) {
                    println "Waiting for Gerrit to come up.."
                    salt.cmdRun(saltMaster, 'I@gerrit:client', 'while true; do curl -svf 172.16.10.254:8080 >/dev/null && break; done')
                }
                retry(2) {
                    // Needs to run twice to pass __virtual__ method of gerrit module
                    // after installation of dependencies
                    try {
                        salt.enforceState(saltMaster, 'I@gerrit:client', 'gerrit', true)
                    } catch (Exception e) {
                        common.infoMsg("Restarting Salt minion")
                        salt.cmdRun(saltMaster, 'I@gerrit:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &")
                        sleep(5)
                        throw e
                    }
                }

                // Jenkins
                timeout(10) {
                    println "Waiting for Jenkins to come up.."
                    salt.cmdRun(saltMaster, 'I@jenkins:client', 'while true; do curl -svf 172.16.10.254:8081 >/dev/null && break; done')
                }
                retry(2) {
                    // Same for jenkins
                    try {
                        salt.enforceState(saltMaster, 'I@jenkins:client', 'jenkins', true)
                    } catch (Exception e) {
                        common.infoMsg("Restarting Salt minion")
                        salt.cmdRun(saltMaster, 'I@jenkins:client', "exec 0>&-; exec 1>&-; exec 2>&-; nohup /bin/sh -c 'salt-call --local service.restart salt-minion' &")
                        sleep(5)
                        throw e
                    }
                }
            }

            stage("Finalize") {
                //
                // Deploy user's ssh key
                //
                def adminUser
                def authorizedKeysFile
                def adminUserCmdOut = salt.cmdRun(saltMaster, 'I@salt:master', "[ -d /home/ubuntu ] && echo 'ubuntu user exists'")
                if (adminUserCmdOut =~ /ubuntu user exists/) {
                    adminUser = "ubuntu"
                    authorizedKeysFile = "/home/ubuntu/.ssh/authorized_keys"
                } else {
                    adminUser = "root"
                    authorizedKeysFile = "/root/.ssh/authorized_keys"
                }

                if (sshPubKey) {
                    println "Deploying provided ssh key at ${authorizedKeysFile}"
                    salt.cmdRun(saltMaster, '*', "echo '${sshPubKey}' | tee -a ${authorizedKeysFile}")
                }

                //
                // Generate docs
                //
                try {
                    try {
                        // Run sphinx state to install sphinx-build needed in
                        // upcomming orchestrate
                        salt.enforceState(saltMaster, 'I@sphinx:server', 'sphinx')
                    } catch (Throwable e) {
                        true
                    }
                    retry(3) {
                        // TODO: fix salt.orchestrateSystem
                        // print salt.orchestrateSystem(saltMaster, ['expression': '*', 'type': 'compound'], 'sphinx.orch.generate_doc')
                        def out = salt.cmdRun(saltMaster, 'I@salt:master', 'salt-run state.orchestrate sphinx.orch.generate_doc || echo "Command execution failed"')
                        print out
                        if (out =~ /Command execution failed/) {
                            throw new Exception("Command execution failed")
                        }
                    }
                } catch (Throwable e) {
                    // We don't want sphinx docs to ruin whole build, so possible
                    // errors are just ignored here
                    true
                }
                salt.enforceState(saltMaster, 'I@nginx:server', 'nginx')

                common.successMsg("""
    ============================================================
    Your CI/CD lab has been deployed and you can enjoy it:
    Use sshuttle to connect to your private subnet:

        sshuttle -r ${adminUser}@${saltMasterHost} 172.16.10.0/24

    And visit services running at 172.16.10.254 (vip address):

        9600    haproxy stats
        8080    gerrit
        8081    jenkins
        8091    Docker swarm visualizer
        8090    Reclass-generated documentation

    If you provided SSH_PUBLIC_KEY, you can use it to login,
    otherwise you need to get private key connected to this
    heat template.

    DON'T FORGET TO TERMINATE YOUR STACK WHEN YOU DON'T NEED IT!
    ============================================================""")
            }
        } catch (Throwable e) {
            // If there was an error or exception thrown, the build failed
            currentBuild.result = "FAILURE"
            throw e
        } finally {
            // Cleanup
            if (HEAT_STACK_DELETE.toBoolean() == true) {
                stage('Trigger cleanup job') {
                    build job: 'deploy-heat-cleanup', parameters: [[$class: 'StringParameterValue', name: 'HEAT_STACK_NAME', value: HEAT_STACK_NAME]]
                }
            }
        }
    }
}