Jenkinsfile

import java.util.regex.Pattern
/**
 *
 * OS images build pipeline
 *
 * Target build slave node:
 * target-build-slave: [ qemu ]
 *
 * This pipeline require to have slave with such dep's:
 * ubuntu-pkgs: [ qemu-kvm, libvirt-bin, cloud-image-utils ]
 *
 * Expected parameters:
 *    BUILD_OS
 *    BUILD_ONLY
 *    PACKER_DEBUG
 *    PACKER_URL
 *    PACKER_ZIP
 *    PACKER_ZIP_MD5
 *    PACKER_ARGS
 *    UPLOAD_URL
 *    SKIP_UPLOAD
 *    CLEANUP_OLD
 *    CLEANUP_KEEP
 *    PIPELINE_LIBS_URL
 *    PIPELINE_LIBS_BRANCH
 *    PIPELINE_LIBS_CREDENTIALS_ID
 *    GLANCE_UPLOAD
 *    GLANCE_IMG_TYPES
 *    GLANCE_URL
 *    GLANCE_CREDENTIALS_ID
 *    GLANCE_PROJECT
 *    GLANCE_ARGS
 *    OPENSTACK_API_CLIENT
 *    IMAGE_NAME - Name of the result image.
 *    EXTRA_VARIABLES - list of key:value variables required by template.json
 */

// Load shared libs
common = new com.mirantis.mk.Common()
def date = new Date()
def dateTime = date.format("ddMMyyyy-HHmmss")

ArrayList extra_vars = EXTRA_VARIABLES.readLines()
IMAGE_NAME = IMAGE_NAME + "-" + dateTime

@NonCPS
def getCleanupImageList(remoteImagesString, imageType, osImage) {
    def remoteImages = remoteImagesString.tokenize("\n")
    def imageTypeForRegex = Pattern.quote(imageType)
    def osImageForRegex = Pattern.quote(osImage.replaceAll(/\./,"-"))
    def remoteImagesSameType = remoteImages.findAll { it ->
        it =~ /${imageTypeForRegex}$/
    }
    def imagesToClean = remoteImagesSameType.toSorted().findAll { it ->
        it =~ /^${osImageForRegex}-/
    }
    // dont cleanup non timestamp images
    return imagesToClean.findAll { it ->
        it =~ /${osImageForRegex}-x(64|32)-\d+${imageTypeForRegex}/
    }
}

timeout(time: 12, unit: 'HOURS') {
    node('qemu') {
        // Define global variables
        def workspace = common.getWorkspace()
        def buildTypes = BUILD_ONLY.tokenize(" ")
        def createdImages=[]
        def uploadedImages=[]
        def cleanedImages=[]

        checkout scm
        try {
            stage("prepare") {
                if (!fileExists("${workspace}/tmp")) {
                    sh "mkdir -p ${workspace}/tmp"
                }
                if (!fileExists("${workspace}/images")) {
                    sh "mkdir ${workspace}/images"
                }
            }
            if (!fileExists("bin")) {
                common.infoMsg("Downloading packer")
                sh "mkdir bin"
                dir("bin") {
                    sh "wget -O ${PACKER_ZIP} ${PACKER_URL}"
                    sh "echo \"${PACKER_ZIP_MD5} ${PACKER_ZIP}\" >> md5sum"
                    sh "md5sum -c --status md5sum"
                    sh "unzip ${PACKER_ZIP}"
                }
            }
            // clean images dir before building
            sh(script: String.format("rm -rf %s/images/*", BUILD_OS), returnStatus: true)
            // clean virtualenv is exists
            sh(script: String.format("rm -rf %s/venv", workspace), returnStatus: true)

            stage("build") {
                dir(BUILD_OS) {
                    withEnv(extra_vars + [String.format("PATH=%s:%s/bin", env.PATH, workspace),
                        "PACKER_LOG_PATH=${workspace}/packer.log",
                        "PACKER_LOG=1",
                        "IMAGE_NAME=${IMAGE_NAME}",
                        "TMPDIR=${workspace}/tmp"
                    ]) {
                        if (PACKER_DEBUG == 'true') {
                            PACKER_ARGS = "${PACKER_ARGS} -debug"
                        }

                        if (fileExists("config-drive/user-data.yaml")) {
                            common.infoMsg("Creating cloud-config drive")
                            if (fileExists("config-drive/cloudata.iso")) {
                            sh "rm -v config-drive/cloudata.iso"
                            }
                            sh "cloud-localds config-drive/cloudata.iso  config-drive/user-data.yaml"
                        }
                        sh "packer build -only=${BUILD_ONLY} ${PACKER_ARGS} -parallel=false template.json"

                        def packerStatus = sh(script: "grep \"Some builds didn't complete successfully and had errors\" ${PACKER_LOG_PATH}", returnStatus: true)
                        // grep returns 0 if find something
                        if (packerStatus != 0) {
                            if (buildTypes.contains("qemu")) {
                                def imageQemu = sh(script: "find images/ | grep -- '-qemu-' | tail -1", returnStdout: true).trim()
                                if (imageQemu != null && imageQemu != "") {
                                    def qemuConvertStatus = sh(script: "qemu-img convert -c -O qcow2 ${imageQemu} ${imageQemu}.qcow2", returnStatus:true)
                                    if(qemuConvertStatus == 0){
                                        def imageDir = imageQemu.substring(0, imageQemu.lastIndexOf("/") + 1)
                                        def imageQemuName = imageQemu.substring(imageQemu.lastIndexOf("/") + 1)
                                        def moveResult = sh(script: "mv ${imageQemu}.qcow2  ${imageDir}..", returnStatus: true)
                                        if(moveResult == 0){
                                            sh "rm -rf ${imageDir}"
                                            sh "rm -f ${imageQemu}"
                                            createdImages.add(imageQemuName+".qcow2")
                                        }
                                    }else{
                                        throw new Exception("Qemu image convert failed")
                                    }
                                } else {
                                        throw new Exception("No Qemu images with '-qemu-' in its path")
                                }
                            }
                            if (buildTypes.contains("docker")) {
                                def imageDocker = sh(script: "find images/ | grep -- '-docker-' | grep '.tar\$' | tail -1", returnStdout: true).trim()
                                if (imageDocker != null && imageDocker != "") {
                                    def pbZip2Status = sh(script: "pbzip2 ${imageDocker}", returnStatus: true)
                                    if(pbZip2Status == 0){
                                        sh "rm -f ${imageDocker}"
                                        createdImages.add(imageDocker+".bz2")
                                    }else{
                                        throw new Exception("pbzip2 image convert failed")
                                    }
                                } else {
                                        throw new Exception("No Docker images with '-docker-' in its path")
                                }
                            }

                        } else {
                          throw new Exception("Packer build failed")
                        }
                    }
                }
            }
            stage("upload"){
                dir(BUILD_OS + "/images") {
                    def images = findFiles(glob: "*.*")
                    def imageBuilds = [:]
                    def openstack = new com.mirantis.mk.Openstack()
                    def openstackEnv = String.format("%s/venv", workspace);
                    def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
                    def rcFile = openstack.createOpenstackEnv(workspace, GLANCE_URL, GLANCE_CREDENTIALS_ID, GLANCE_PROJECT)
                    def glanceImgTypes = GLANCE_IMG_TYPES.tokenize(" ")
                    openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
                    openstack.runOpenstackCommand("pip install python-glanceclient==1.0.0", rcFile, openstackEnv)
                    for (int i = 0; i < images.size(); i++) {
                        def imageName = images[i].name
                        def imageNameList = imageName.tokenize(".")
                        def imageType = "." + imageNameList[imageNameList.size() - 1]
                        if(imageType.equals(".md5")){
                            continue;
                        }

                        imageBuilds["build${i}"]={
                            if (SKIP_UPLOAD != 'true') {
                                sh "md5sum ${imageName} > ${imageName}.md5"
                                common.infoMsg("Uploading image " + imageName)
                                def uploadImageStatus = sh(script: "curl -f -T ${imageName} ${UPLOAD_URL}", returnStatus: true)
                                def uploadMd5Status = sh(script: "curl -f -T ${imageName}.md5 ${UPLOAD_URL}", returnStatus: true)
                                // upload latest
                                def latestImageName = imageName.substring(0, imageName.lastIndexOf("-")) + "-latest" + imageType
                                common.infoMsg("Uploading image ${imageName} as latest")
                                def uploadLatestStatus = sh(script: "curl -f -T ${imageName} ${UPLOAD_URL}${latestImageName}", returnStatus: true)
                                def uploadLatestMd5Status = sh(script: "curl -f -T ${imageName}.md5 ${UPLOAD_URL}${latestImageName}.md5", returnStatus: true)
                                if(uploadLatestStatus != 0 || uploadLatestMd5Status != 0){
                                    common.errorMsg("Latest image upload failed")
                                }
                                if (GLANCE_UPLOAD == 'true' && glanceImgTypes.contains(imageType.substring(1))) {
                                    def glanceRunArgs = String.format("%s --disk-format %s --container-format bare", GLANCE_ARGS, imageType.substring(1))
                                    if (GLANCE_PUBLIC == 'true') {
                                        glanceRunArgs += " --visibility public"
                                    }

                                    def imageShortName = imageNameList.get(0)
                                    openstack.runOpenstackCommand(String.format("glance image-create --name '%s' %s --file %s", imageShortName, glanceRunArgs, imageName), rcFile, openstackEnv)
                                }
                                if(uploadImageStatus==0 && uploadMd5Status == 0){
                                    uploadedImages.add(imageName)
                                    sh(String.format("rm -r %s %s.md5",imageName, imageName))
                                    createdImages.remove(imageName)
                                }else{
                                    throw new Exception("Image upload failed")
                                }
                            }
                            if (CLEANUP_OLD == 'true') {
                                def remoteImages = sh(script: "curl -f -sss ${UPLOAD_URL} | grep -Eo '>.*\\.(qcow2|box|tar\\.bz2)</a>' | sed -e 's,>,,g' -e 's,</a,,g'", returnStdout: true)
                                if (remoteImages != "") {
                                    def cleanupImages = getCleanupImageList(remoteImages, imageType, BUILD_OS)
                                    def deleteCount = cleanupImages.size() - Integer.parseInt(CLEANUP_KEEP)
                                    if (deleteCount > 0) {
                                        for (int j = 0; j < deleteCount; j++) {
                                            common.infoMsg(String.format("Deleting image %s from aptly", cleanupImages[j]))
                                            sh "curl -f -X DELETE ${UPLOAD_URL}" + cleanupImages[j]
                                            sh "curl -f -X DELETE ${UPLOAD_URL}" + cleanupImages[j] + ".md5"
                                            cleanedImages.add(cleanupImages[j])
                                        }
                                    }
                                }
                            }
                        }
                    }
                    parallel imageBuilds
                    common.infoMsg(String.format("Uploaded %s images with names %s", uploadedImages.size(), uploadedImages.toString()))
                    common.infoMsg(String.format("Cleaned %s images with names %s", cleanedImages.size(), cleanedImages.toString()))
                }
            }
        } catch (Throwable e) {
            // If there was an error or exception thrown, the build failed
            currentBuild.result = "FAILURE"
            throw e
        } finally {
            common.sendNotification(currentBuild.result, "", ["slack"])
            if (buildTypes.contains("docker")) {
                withEnv(["PACKER_LOG_PATH=${workspace}/packer.log"]) {
                    sh "docker rmi --force \$(grep \"docker: Image ID:\" ${PACKER_LOG_PATH} | cut -d : -f 6 | head -1 | sed s,\\ ,,g) || true"
                }
            }
            // clean created images if error occured
            if(!createdImages.isEmpty()){
                dir(BUILD_OS + "/images"){
                    for(int i=0;i<createdImages.size();i++){
                        sh String.format("rm -f %s",createdImages.get(i))
                    }
                }
            }
        }
    }
}