Jenkinsfile

import java.util.regex.Pattern
/**
 *
 * OS images build pipeline
 *
 * Target build slave node:
 * target-build-slave: [ qemu ]
 *
 * This pipeline require to have slave with such dep's:
 * ubuntu-pkgs: [ qemu-kvm, libvirt-bin, cloud-image-utils ]
 *
 * Expected parameters:
 *    BUILD_OS
 *    BUILD_ONLY
 *    PACKER_DEBUG
 *    PACKER_URL
 *    PACKER_ZIP
 *    PACKER_ZIP_MD5
 *    PACKER_ARGS
 *    UPLOAD_URL
 *    SKIP_UPLOAD
 *    CLEANUP_OLD
 *    CLEANUP_KEEP
 *    PIPELINE_LIBS_URL
 *    PIPELINE_LIBS_BRANCH
 *    PIPELINE_LIBS_CREDENTIALS_ID
 *    GLANCE_UPLOAD
 *    GLANCE_IMG_TYPES
 *    GLANCE_URL
 *    GLANCE_CREDENTIALS_ID
 *    GLANCE_PROJECT
 *    GLANCE_ARGS
 *    OPENSTACK_API_CLIENT
 */

// Load shared libs
common = new com.mirantis.mk.Common()

node('qemu') {
    // Define global variables
    def workspace = common.getWorkspace()
    def buildTypes = BUILD_ONLY.tokenize(" ")
    def createdImages=[]
    def uploadedImages=[]
    def cleanedImages=[]

    checkout scm
    try {
        stage("prepare") {
            if (!fileExists("${workspace}/tmp")) {
                sh "mkdir -p ${workspace}/tmp"
            }
            if (!fileExists("${workspace}/images")) {
                sh "mkdir ${workspace}/images"
            }
        }
        if (!fileExists("bin")) {
            common.infoMsg("Downloading packer")
            sh "mkdir bin"
            dir("bin") {
                sh "wget -O ${PACKER_ZIP} ${PACKER_URL}"
                sh "echo \"${PACKER_ZIP_MD5} ${PACKER_ZIP}\" >> md5sum"
                sh "md5sum -c --status md5sum"
                sh "unzip ${PACKER_ZIP}"
            }
        }
        // clean images dir before building
        sh(script: String.format("rm -rf %s/images/*", BUILD_OS), returnStatus: true)
        // clean virtualenv is exists
        sh(script: String.format("rm -rf %s/venv", workspace), returnStatus: true)

        stage("build") {
            dir(BUILD_OS) {
                withEnv([String.format("PATH=%s:%s/bin", env.PATH, workspace),
                    "PACKER_LOG_PATH=${workspace}/packer.log",
                    "PACKER_LOG=1",
                    "TMPDIR=${workspace}/tmp"
                ]) {
                    if (PACKER_DEBUG == 'true') {
                        PACKER_ARGS = "${PACKER_ARGS} -debug"
                    }

                    if (fileExists("config-drive/user-data.yaml")) {
                        common.infoMsg("Creating cloud-config drive")
                        if (fileExists("config-drive/cloudata.iso")) {
                        sh "rm -v config-drive/cloudata.iso"
                        }
                        sh "cloud-localds config-drive/cloudata.iso  config-drive/user-data.yaml"
                    }
                    sh "packer build -only=${BUILD_ONLY} ${PACKER_ARGS} -parallel=false template.json"

                    def packerStatus = sh(script: "grep \"Some builds didn't complete successfully and had errors\" ${PACKER_LOG_PATH}", returnStatus: true)
                    // grep returns 0 if find something
                    if (packerStatus != 0) {
                        if (buildTypes.contains("qemu")) {
                            def imageQemu = sh(script: "find images/ | grep -- '-qemu-' | tail -1", returnStdout: true).trim()
                            if (imageQemu != null && imageQemu != "") {
                                def qemuConvertStatus = sh(script: "qemu-img convert -c -O qcow2 ${imageQemu} ${imageQemu}.qcow2", returnStatus:true)
                                if(qemuConvertStatus == 0){
                                    def imageDir = imageQemu.substring(0, imageQemu.lastIndexOf("/") + 1)
                                    def imageQemuName = imageQemu.substring(imageQemu.lastIndexOf("/") + 1)
                                    def moveResult = sh(script: "mv ${imageQemu}.qcow2  ${imageDir}..", returnStatus: true)
                                    if(moveResult == 0){
                                        sh "rm -rf ${imageDir}"
                                        sh "rm -f ${imageQemu}"
                                        createdImages.add(imageQemuName+".qcow2")
                                    }
                                }else{
                                    throw new Exception("Qemu image convert failed")
                                }
                            }
                        }
                        if (buildTypes.contains("docker")) {
                            def imageDocker = sh(script: "find images/ | grep -- '-docker-' | grep '.tar\$' | tail -1", returnStdout: true).trim()
                            if (imageDocker != null && imageDocker != "") {
                                def pbZip2Status = sh(script: "pbzip2 ${imageDocker}", returnStatus: true)
                                if(pbZip2Status == 0){
                                    sh "rm -f ${imageDocker}"
                                    createdImages.add(imageDocker+".bz2")
                                }else{
                                    throw new Exception("pbzip2 image convert failed")
                                }
                            }
                        }
                    } else {
                      throw new Exception("Packer build failed")
                    }
                }
            }
        }
        stage("upload"){
            dir(BUILD_OS + "/images") {
                def images = findFiles(glob: "*.*")
                def imageBuilds = [:]
                def openstack = new com.mirantis.mk.Openstack()
                def openstackEnv = String.format("%s/venv", workspace);
                def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty'
                def rcFile = openstack.createOpenstackEnv(GLANCE_URL, GLANCE_CREDENTIALS_ID, GLANCE_PROJECT)
                def glanceImgTypes = GLANCE_IMG_TYPES.tokenize(" ")
                openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion)
                openstack.runOpenstackCommand("pip install python-glanceclient==1.0.0", rcFile, openstackEnv)
                for (int i = 0; i < images.size(); i++) {
                    def imageName = images[i].name
                    def imageNameList = imageName.tokenize(".")
                    def imageType = "." + imageNameList[imageNameList.size() - 1]
                    if(imageType.equals(".md5")){
                        continue;
                    }

                    imageBuilds["build${i}"]={
                        if (SKIP_UPLOAD != 'true') {
                            sh "md5sum ${imageName} > ${imageName}.md5"
                            common.infoMsg("Uploading image " + imageName)
                            def uploadImageStatus = sh(script: "curl -f -T ${imageName} ${UPLOAD_URL}", returnStatus: true)
                            def uploadMd5Status = sh(script: "curl -f -T ${imageName}.md5 ${UPLOAD_URL}", returnStatus: true)
                            // upload latest
                            def latestImageName = imageName.substring(0, imageName.lastIndexOf("-")) + "-latest" + imageType
                            common.infoMsg("Uploading image ${imageName} as latest")
                            def uploadLatestStatus = sh(script: "curl -f -T ${imageName} ${UPLOAD_URL}${latestImageName}", returnStatus: true)
                            def uploadLatestMd5Status = sh(script: "curl -f -T ${imageName}.md5 ${UPLOAD_URL}${latestImageName}.md5", returnStatus: true)
                            if(uploadLatestStatus != 0 || uploadLatestMd5Status != 0){
                                common.errorMsg("Latest image upload failed")
                            }
                            if (GLANCE_UPLOAD == 'true' && glanceImgTypes.contains(imageType.substring(1))) {
                                def glanceRunArgs = String.format("%s --disk-format %s --container-format bare", GLANCE_ARGS, imageType.substring(1))
                                if (GLANCE_PUBLIC == 'true') {
                                    glanceRunArgs += " --visibility public"
                                }

                                def imageShortName = imageNameList.get(0)
                                openstack.runOpenstackCommand(String.format("glance image-create --name '%s' %s --file %s", imageShortName, glanceRunArgs, imageName), rcFile, openstackEnv)
                            }
                            if(uploadImageStatus==0 && uploadMd5Status == 0){
                                uploadedImages.add(imageName)
                                sh(String.format("rm -r %s %s.md5",imageName, imageName))
                                createdImages.remove(imageName)
                            }else{
                                throw new Exception("Image upload failed")
                            }
                        }
                        if (CLEANUP_OLD == 'true') {
                            def remoteImages = sh(script: "curl -f -sss ${UPLOAD_URL} | grep -Eo '>.*\\.(qcow2|box|tar\\.bz2)</a>' | sed -e 's,>,,g' -e 's,</a,,g'", returnStdout: true)
                            if (remoteImages != "") {
                                def cleanupImages = getCleanupImageList(remoteImages, imageType, BUILD_OS)
                                def deleteCount = cleanupImages.size() - Integer.parseInt(CLEANUP_KEEP)
                                if (deleteCount > 0) {
                                    for (int j = 0; j < deleteCount; j++) {
                                        common.infoMsg(String.format("Deleting image %s from aptly", cleanupImages[j]))
                                        sh "curl -f -X DELETE ${UPLOAD_URL}" + cleanupImages[j]
                                        sh "curl -f -X DELETE ${UPLOAD_URL}" + cleanupImages[j] + ".md5"
                                        cleanedImages.add(cleanupImages[j])
                                    }
                                }
                            }
                        }
                    }
                }
                parallel imageBuilds
                common.infoMsg(String.format("Uploaded %s images with names %s", uploadedImages.size(), uploadedImages.toString()))
                common.infoMsg(String.format("Cleaned %s images with names %s", cleanedImages.size(), cleanedImages.toString()))
            }
        }
    } catch (Throwable e) {
        // If there was an error or exception thrown, the build failed
        currentBuild.result = "FAILURE"
        throw e
    } finally {
        common.sendNotification(currentBuild.result, "", ["slack"])
        if (buildTypes.contains("docker")) {
            withEnv(["PACKER_LOG_PATH=${workspace}/packer.log"]) {
                sh "docker rmi --force \$(grep \"docker: Image ID:\" ${PACKER_LOG_PATH} | cut -d : -f 6 | head -1 | sed s,\\ ,,g) || true"
            }
        }
        // clean created images if error occured
        if(!createdImages.isEmpty()){
            dir(BUILD_OS + "/images"){
                for(int i=0;i<createdImages.size();i++){
                    sh String.format("rm -f %s",createdImages.get(i))
                }
            }
        }
    }
}

@NonCPS
def getCleanupImageList(remoteImagesString, imageType, osImage) {
    def remoteImages = remoteImagesString.tokenize("\n")
    def imageTypeForRegex = Pattern.quote(imageType)
    def osImageForRegex = Pattern.quote(osImage.replaceAll(/\./,"-"))
    def remoteImagesSameType = remoteImages.findAll { it ->
        it =~ /${imageTypeForRegex}$/
    }
    def imagesToClean = remoteImagesSameType.toSorted().findAll { it ->
        it =~ /^${osImageForRegex}-/
    }
    // dont cleanup non timestamp images
    return imageToClean.findAll { it ->
        it =~ /${osImageForRegex}-x(64|32)-\d+${imageTypeForRegex}/
    }
}