packer-image-buid.groovy

/**
 *
 * Build mirror/aptly/apt01 image pipeline
 *
 *  PACKER_URL       = 'https://releases.hashicorp.com/packer/1.2.4/packer_1.2.4_linux_amd64.zip'
 *  PACKER_ZIP_MD5   = '997274e80ae41102eecf9df2e5b67860'
 *  PACKER_ARGS      = '-debug'
 *  BUILD_ONLY       = 'openstack|qemu'
 *  CLEANUP_AFTER    = wipe workspace AFTER build
 *  IMAGE_NAME       = Name of the result image.
 *  TIMESTAMP_INAME  = True|false. If true - image will be uploaded to backend with IMAGE_NAME + timestamp
 *
 * OPENSTACK_OPENRC_YAML: aml of key:value variables required for openstack venv
 * example:
 *            ---
 *             OS_TENANT_ID: 'xxxx'
 *             OS_TENANT_NAME: "mcp-oscore-ci"
 *
 * EXTRA_VARIABLES_YAML - Yaml of key:value variables required for template.json
 * example:
 *            ---
 *            IMAGE_NAME: 'ubuntu-16.04-proposed'
 *
 *  CREDENTIALS_ID  = Global jenkins cred for clone DEFAULT_GIT_URL
 *  DEFAULT_GIT_URL
 *  REFSPEC
 *
 *  OS_VERSION         = OpenStack version
 *  OS_CREDENTIALS_ID  = ID of credentials for OpenStack API stored in Jenkins.
 *  OS_URL             = Keystone auth endpoint of the OpenStack.
 *  OS_PROJECT         =
 *
 *  PUBLISH_BACKEND = local,glance,http
 *  UPLOAD_URL      = URL of an WebDAV used to upload the image after creating.(Only in case PUBLISH_BACKEND == http)
 */

// Load shared libs
def common = new com.mirantis.mk.Common()
def openstack = new com.mirantis.mk.Openstack()
def gerrit = new com.mirantis.mk.Gerrit()
def date = new Date()
def dateTime = date.format("ddMMyyyy-HHmmss")
//
def job_env = env.getEnvironment().findAll { k, v -> v }
gerritCredentials = env.CREDENTIALS_ID ?: 'gerrit'
gerritCredentialsHttp = env.GERRIT_HTTP_CREDENTIALS_ID ?: 'mcp-jenkins-http'

/////
extra_vars = readYaml text: job_env.get('EXTRA_VARIABLES_YAML','').trim()
// FIXME: os_openrc should be refactored.
os_openrc = readYaml text: job_env.get('OPENSTACK_OPENRC_YAML','').trim()
if (job_env.get('TIMESTAMP_INAME', false).toBoolean()) {
    imageName = job_env.IMAGE_NAME + "-" + dateTime
} else {
    imageName = job_env.IMAGE_NAME
}
// Overwrite IMAGE_NAME in template.json with expected
extra_vars['IMAGE_NAME'] = imageName
// Fix some variables
job_env['CLEANUP_AFTER'] = job_env.CLEANUP_AFTER.toBoolean()
job_env['SKIP_UPLOAD'] = job_env.SKIP_UPLOAD.toBoolean()
job_env['CLEANUP_AFTER'] = job_env.CLEANUP_AFTER.toBoolean()
job_env['BUILD_ONLY'] = job_env.BUILD_ONLY.toLowerCase()
job_env['PUBLISH_BACKEND'] = job_env.PUBLISH_BACKEND.toLowerCase()
publishBackends = job_env['PUBLISH_BACKEND'].split(',')
//
defaultGitRef = job_env.get('REFSPEC', 'HEAD')
defaultGitUrl = job_env.get('DEFAULT_GIT_URL', null)
slaveNode = (env.SLAVE_NODE ?: 'virtual')

// Self-check
for (String req_v : ['BUILD_OS', 'BUILD_ONLY','IMAGE_NAME'] ) {
    if (!job_env.get(req_v, false)) {
        throw new Exception("${req_v} not set!")
    }
}

def MapToList(input_map) {
/**
 * Convert dict in bash-like list
 */
    def data_list = []
    for (i = 0; i < input_map.size(); i++) {
        data = ''
        data = input_map.keySet()[i] + "=" + input_map.values()[i]
        data_list.add(data)
    }
    return data_list
}
// Necessary to have ~100G of volume on node for build offline image. Code below chooses huge available slave.
def getHugeSlave() {
    if (os_openrc['OPENSTACK_ENVIRONMENT'] == "internal_cloud_v2_eu") {
        return nodesByLabel("mcp-offline-builder-eu")[0]
    }
    slaves = nodesByLabel("virtual")
    def targetSlave = ''
    for (slave in slaves) {
        node(slave){
            outputMsg = sh(returnStdout: true, script: '''#!/bin/bash
                        df --block-size=G / | awk '{print $4}' | sed -n 2p | sed 's/.$//'
                        ''')
        }
        if(outputMsg.toInteger() >= 120){
            targetSlave = slave
            break
        }
    }
    return targetSlave
}

nodeForBuild = getHugeSlave()

timeout(time: 9, unit: 'HOURS') {
    node(nodeForBuild) {
        def workspace = common.getWorkspace()
        def openstackEnv = "${workspace}/venv"
        def rcFile = ''
        creds = common.getPasswordCredentials(job_env.CREDENTIALS_ID)

        stage("checkout") {
            if (defaultGitRef && defaultGitUrl) {
                checkouted = gerrit.gerritPatchsetCheckout(defaultGitUrl, defaultGitRef, "HEAD", job_env.CREDENTIALS_ID)
            } else {
                throw new Exception("Cannot checkout gerrit patchset: DEFAULT_GIT_URL is null")
            }
        }

        if (job_env.BUILD_ONLY == 'openstack' || 'glance' in publishBackends) {
            rcFile = openstack.createOpenstackEnv(workspace, os_openrc['OS_AUTH_URL'], os_openrc['OS_CREDENTIALS_ID'], os_openrc['OS_PROJECT_NAME'], "default", "", "default", "3", "")
        }

        try {
            def _artifact_dir = "${workspace}/artifacts"
            def _artifact_list = []
            def ImagesCacheFolder = "${workspace}/../${env.JOB_NAME}_cache/"
            stage("Prepare env") {
                if (!fileExists("${workspace}/tmp")) {
                    sh "mkdir -p ${workspace}/tmp"
                }
                if (!fileExists(ImagesCacheFolder)) {
                    sh "mkdir -p ${ImagesCacheFolder}"
                }
                if (!fileExists(_artifact_dir)) {
                    sh "mkdir -p ${_artifact_dir}"
                }
                if (!fileExists("bin")) {
                    common.infoMsg("Downloading packer")
                    sh "mkdir -p bin"
                    dir("bin") {
                        def zipname = sh(script: "basename ${job_env.PACKER_URL}", returnStdout: true).trim()
                        sh(script: "wget --quiet ${job_env.PACKER_URL}", returnStdout: true)
                        sh "echo \"${job_env.PACKER_ZIP_MD5} ${zipname}\" >> md5sum"
                        sh(script: "md5sum -c --status md5sum", returnStdout: true)
                        sh "unzip ${zipname}"
                    }
                }
                if (!fileExists("${job_env.BUILD_OS}/images")) {
                    // clean images dir before building
                    sh(script: "rm -rf ${job_env.BUILD_OS}/images/*", returnStatus: true)
                }
            }

            stage("Build Instance") {
                def _packer_args = job_env.get('PACKER_ARGS', '')
                def _packer_log = "${workspace}/packer.log"
                // clean old log, for correct status grepping
                if (fileExists(_packer_log)) {
                    sh "rm -v ${_packer_log}"
                }

                dir("${workspace}/${job_env.BUILD_OS}/") {
                    if (fileExists("config-drive")) {
                        def model = extra_vars.get('CLUSTER_MODEL', '')
                        if (model != "") {
                            checkout([
                                    $class             : 'GitSCM',
                                    branches           : [[name: 'FETCH_HEAD']],
                                    extensions         : [
                                        [$class: 'RelativeTargetDirectory', relativeTargetDir: 'config-drive/model'],
                                        [$class: 'SubmoduleOption', disableSubmodules: false, parentCredentials: true, recursiveSubmodules: true, trackingSubmodules: false],
                                    ],
                                    userRemoteConfigs  : [[url: model, refspec: extra_vars.get('CLUSTER_MODEL_REF', 'master'), credentialsId: gerritCredentials]]
                            ])

                            def clusterName = extra_vars.get('CLUSTER_NAME', '')
                            def mcpVersion = extra_vars.get('MCP_VERSION', '')
                            if (clusterName == 'mcp-offline' && mcpVersion != '') {
                                def filePath = 'classes/cluster/mcp-offline/infra/apt01_dummy.yml'
                                sh "sed -i 's/mcp_version: .*/mcp_version: ${mcpVersion}/' config-drive/model/${filePath}"
                                // speed up build by using internal mirrors in the same region
                                if (os_openrc.get('CLOUD_REGION', 'US').matches("[Ee][Uu]")) {
                                    sh "sed -i 's/debmirror_mirrors_host: .*/debmirror_mirrors_host: mirror-eu.mcp.mirantis.net/' config-drive/model/${filePath}"
                                }
                                filePath = 'classes/cluster/mcp-offline/infra/mirror/init.yml'
                                withCredentials([usernamePassword(
                                    credentialsId: gerritCredentialsHttp,
                                    usernameVariable: 'GERRIT_HTTP_USER',
                                    passwordVariable: 'GERRIT_HTTP_PASS')
                                ]) {
                                    sh "sed -i 's/%STACKLIGHT_PYTEST_GIT_HTTPS_USER%/${GERRIT_HTTP_USER}/' config-drive/model/${filePath}"
                                    sh "sed -i 's/%STACKLIGHT_PYTEST_GIT_HTTPS_PASS%/${GERRIT_HTTP_PASS}/' config-drive/model/${filePath}"
                                }
                            }

                            def reclass_url = extra_vars.get('RECLASS_SYSTEM_URL', '')
                            def reclass_ref = extra_vars.get('RECLASS_SYSTEM_REF', '')
                            if (reclass_url && reclass_ref) {
                                if (fileExists('config-drive/model/classes/system')) {
                                    dir('config-drive/model/classes/system') {
                                        checkout([
                                                $class             : 'GitSCM',
                                                branches           : [[name: 'FETCH_HEAD']],
                                                userRemoteConfigs  : [[url: reclass_url, refspec: reclass_ref, credentialsId: gerritCredentials]]
                                        ])
                                    }
                                }
                            }
                        }
                        def scripts = extra_vars.get('GIT_SALT_FORMULAS_SCRIPTS', '')
                        if (scripts != "") {
                            checkout([
                                    $class           : 'GitSCM',
                                    branches         : [[name: 'FETCH_HEAD']],
                                    extensions       : [[$class: 'RelativeTargetDirectory', relativeTargetDir: 'config-drive/salt_scripts']],
                                    userRemoteConfigs: [[url: scripts, refspec: extra_vars.get('SCRIPTS_REF', 'master'), credentialsId: gerritCredentials]]
                            ])
                        }

                        common.infoMsg("Creating cloud-config drive")
                        def isoFile = "config-drive/cloudata.iso"
                        if (fileExists(isoFile)) {
                            sh "rm -v ${isoFile}"
                        }
                        // This is left for backward-compatibility
                        if (fileExists("config-drive/user-data.yaml")) {
                            sh "mv config-drive/user-data.yaml config-drive/user-data"
                            if (!fileExists("config-drive/meta-data")) {
                                sh "echo 'hostname: ubuntu' > config-drive/meta-data"
                            }
                        }
                        sh "mkisofs -o ${isoFile} -V cidata -r -J --quiet config-drive"
                        archiveArtifacts artifacts: "${isoFile}"
                    }
                }

                if (job_env.BUILD_ONLY == "openstack") {
                    dir("${workspace}/${job_env.BUILD_OS}/") {
                        extra_vars_list = MapToList(extra_vars)
                        withEnv(["PATH=${env.PATH}:${workspace}/bin",
                                 "PACKER_LOG_PATH=${_packer_log}",
                                 "PACKER_LOG=1",
                                 "TMPDIR=${workspace}/tmp",
                                 "OS_USERNAME=${creds.username.toString()}",
                                 "OS_PASSWORD=${creds.password.toString()}"] + extra_vars_list) {

                            common.infoMsg("Run build with:")
                            sh(script: 'printenv|sort')
                            common.infoMsg("Free disk space by 'df -h' command:")
                            sh(script: 'df -h')
                            sh(script: "set -xe; packer build -only='openstack' ${_packer_args} -parallel=false template.json")
                            _os_private = "${workspace}/${job_env.BUILD_OS}/os_${job_env.BUILD_OS}.pem"
                            if (fileExists(_os_private)) {
                                common.infoMsg("Packer private key:")
                                sh "cat ${_os_private}"
                            }
                            def packerStatus = sh(script: "grep \"Some builds didn't complete successfully and had errors\" ${_packer_log}", returnStatus: true)
                            // grep returns 0 if find something
                            if (packerStatus != 0) {
                                common.infoMsg("Openstack instance build complete")
                            } else {
                                throw new Exception("Openstack Packer build failed")
                            }

                            common.retry(3, 5) {
                                common.infoMsg("Attempt download openstack image..")
                                openstack.runOpenstackCommand("openstack image save --file ${_artifact_dir}/${imageName}.qcow2 ${imageName}", rcFile, openstackEnv)
                            }
                        }
                    }

                } else if (job_env.BUILD_ONLY == 'qemu') {

                    dir("${workspace}/${job_env.BUILD_OS}/") {
                        extra_vars_list = MapToList(extra_vars)
                        withEnv(["PATH=${env.PATH}:${workspace}/bin",
                                 "PACKER_LOG_PATH=${_packer_log}",
                                 "PACKER_LOG=1",
                                 "TMPDIR=${workspace}/tmp",
                                 "PACKER_IMAGES_CACHE=${ImagesCacheFolder}"] + extra_vars_list) {
                            common.infoMsg("Run build with:")
                            sh(script: 'printenv|sort')
                            common.infoMsg("Free disk space by 'df -h' command:")
                            sh(script: 'df -h')
                            sh(script: "set -xe ; packer build -on-error=ask -only='qemu' ${_packer_args} -parallel=false template.json".toString())

                            def packerStatus = sh(script: "grep \"Some builds didn't complete successfully and had errors\" ${PACKER_LOG_PATH}", returnStatus: true)
                            // grep returns 0 if find something
                            if (packerStatus != 0) {
                                common.infoMsg("qemu instance build completed successfully")
                            } else {
                                throw new Exception("qemu instance build failed")
                            }
                            // collect artifacts
                            // TODO make it possible, process multiply artifacts by one run.
                            dir('images/') {
                                sh(script: 'find .')
                                def _files = findFiles(glob: "*qemu*/*${imageName}*.qcow2")
                                if (_files.size() > 1) {
                                    common.warningMsg("Multiply artifacts detected!Only first one will be processed!")
                                } else if (_files.size() == 0) {
                                    throw new Exception("No artifacts detected!BUILD_ONLY=${env.BUILD_ONLY} failed!")
                                }
                                for (String x : _files) {
                                    _file = sh(script: "set -x ; readlink -f ${x}", returnStdout: true)
                                    sh(script: "mv -v ${x} ${_artifact_dir}/${imageName}.qcow2")
                                    // Save filename to list
                                    _artifact_list.add("${imageName}.qcow2")
                                }
                            }
                        }
                    }

                } else {
                    throw new Exception("Unexpected BUILD_ONLY=${env.BUILD_ONLY} target!")
                }
            }

            stage("Publish artifacts") {
                dir(_artifact_dir) {
                    def published = false
                    common.infoMsg("Processing md5 for artifacts")
                    for (String x : _artifact_list) {
                        _md5 = sh(script: "md5sum ${x} > ${x}.md5; cat ${x}.md5", returnStdout: true).trim()
                        _size = sh(script: "ls -alh ${x}", returnStdout: true).trim()
                        common.infoMsg("Artifact file: ${_size}\n${_md5}")
                    }
                    if ('local' in publishBackends) {
                        common.infoMsg("Uploading to: local")
                        common.infoMsg("For local publish target - nothing to do, all files in: ${_artifact_dir}")
                        if (job_env.get('CLEANUP_AFTER', false)) {
                            common.warningMsg("You are trying to use 'local' publish method, along with enabled CLEANUP_AFTER! ")
                            common.warningMsg("Disabling CLEANUP_AFTER option, to save you'r data ;) ")
                            job_env.CLEANUP_AFTER = false
                        }
                        published = true
                    }
                    if ('glance' in publishBackends) {
                        common.infoMsg("Uploading to: glance-openstack")
                        if (fileExists("${workspace}/venv")) {
                            common.infoMsg("cleaning virtualenv at:${workspace}/venv")
                            sh(script: "rm -rf ${workspace}/venv", returnStatus: true)
                        }
                        openstack.setupOpenstackVirtualenv(openstackEnv)
                        for (String x : findFiles(glob: "*.*")) {
                            if (x.getName().endsWith('.qcow2')) {
                                _md5sum = sh(script: "cat ${x}.md5", returnStdout: true).trim().split()[0]
                                _property = "--property data=${dateTime} --property md5sum=${_md5sum} --property build='${env.BUILD_URL}' --container-format bare --disk-format qcow2"
                                _cmdline = String.format("glance image-create %s --name '%s' --file %s", _property, imageName, x)
                                openstack.runOpenstackCommand(_cmdline, rcFile, openstackEnv)
                            }
                        }
                        currentBuild.description = "${imageName}.qcow2 uploaded tenant: ${os_openrc['OS_PROJECT_NAME']}"
                        published = true
                    }
                    if ('http' in publishBackends) {
                        for (String u_file : findFiles(glob: '*.*')) {
                            common.infoMsg("Uploading image ${u_file}")
                            def uploadImageStatus = ''
                            common.retry(3, 5) {
                                uploadImageStatus = sh(script: "curl -f -T ${u_file} ${job_env.UPLOAD_URL}", returnStatus: true)
                                if (uploadImageStatus != 0) {
                                    throw new Exception("Uploading file: ${u_file} failed!")
                                }
                            }
                            // Fixme for correct path ?
                            currentBuild.description = "<a href='http://images.mcp.mirantis.net/${imageName}.qcow2'>${imageName}.qcow2</a>"
                        }
                        published = true
                    }
                    if (! published) {
                        throw new Exception("Unsupported publish backend: ${publishBackends}")
                    }

                }
            }
        } catch (Throwable e) {
            // If there was an error or exception thrown, the build failed
            currentBuild.result = "FAILURE"
            throw e
        } finally {
            if (job_env.get('CLEANUP_AFTER', false)) {
                dir(workspace) {
                    sh "find . -mindepth 1 -delete || true"
                }
                if (job_env.BUILD_ONLY == 'openstack') {
                    common.warningMsg("openstack Env cleanup not implemented yet!")
                }
            } else {
                common.warningMsg("Env has not been cleaned!Please cleanup it manualy!")
            }
        }
    }
}