| import java.util.regex.Pattern |
| /** |
| * |
| * OS images build pipeline |
| * |
| * Target build slave node: |
| * target-build-slave: [ qemu ] |
| * |
| * This pipeline require to have slave with such dep's: |
| * ubuntu-pkgs: [ qemu-kvm, libvirt-bin, cloud-image-utils ] |
| * |
| * Expected parameters: |
| * BUILD_OS |
| * BUILD_ONLY |
| * PACKER_DEBUG |
| * PACKER_URL |
| * PACKER_ZIP |
| * PACKER_ZIP_MD5 |
| * PACKER_ARGS |
| * UPLOAD_URL |
| * SKIP_UPLOAD |
| * CLEANUP_OLD |
| * CLEANUP_KEEP |
| * PIPELINE_LIBS_URL |
| * PIPELINE_LIBS_BRANCH |
| * PIPELINE_LIBS_CREDENTIALS_ID |
| * GLANCE_UPLOAD |
| * GLANCE_IMG_TYPES |
| * GLANCE_URL |
| * GLANCE_CREDENTIALS_ID |
| * GLANCE_PROJECT |
| * GLANCE_ARGS |
| * OPENSTACK_API_CLIENT |
| * IMAGE_NAME - Name of the result image. |
| * EXTRA_VARIABLES - list of key:value variables required by template.json |
| */ |
| |
| // Load shared libs |
| common = new com.mirantis.mk.Common() |
| def date = new Date() |
| def dateTime = date.format("ddMMyyyy-HHmmss") |
| |
| ArrayList extra_vars = EXTRA_VARIABLES.readLines() |
| IMAGE_NAME = IMAGE_NAME + "-" + dateTime |
| |
| @NonCPS |
| def getCleanupImageList(remoteImagesString, imageType, osImage) { |
| def remoteImages = remoteImagesString.tokenize("\n") |
| def imageTypeForRegex = Pattern.quote(imageType) |
| def osImageForRegex = Pattern.quote(osImage.replaceAll(/\./,"-")) |
| def remoteImagesSameType = remoteImages.findAll { it -> |
| it =~ /${imageTypeForRegex}$/ |
| } |
| def imagesToClean = remoteImagesSameType.toSorted().findAll { it -> |
| it =~ /^${osImageForRegex}-/ |
| } |
| // dont cleanup non timestamp images |
| return imagesToClean.findAll { it -> |
| it =~ /${osImageForRegex}-x(64|32)-\d+${imageTypeForRegex}/ |
| } |
| } |
| |
| timeout(time: 12, unit: 'HOURS') { |
| node('qemu') { |
| // Define global variables |
| def workspace = common.getWorkspace() |
| def buildTypes = BUILD_ONLY.tokenize(" ") |
| def createdImages=[] |
| def uploadedImages=[] |
| def cleanedImages=[] |
| |
| checkout scm |
| try { |
| stage("prepare") { |
| if (!fileExists("${workspace}/tmp")) { |
| sh "mkdir -p ${workspace}/tmp" |
| } |
| if (!fileExists("${workspace}/images")) { |
| sh "mkdir ${workspace}/images" |
| } |
| } |
| if (!fileExists("bin")) { |
| common.infoMsg("Downloading packer") |
| sh "mkdir bin" |
| dir("bin") { |
| sh "wget -O ${PACKER_ZIP} ${PACKER_URL}" |
| sh "echo \"${PACKER_ZIP_MD5} ${PACKER_ZIP}\" >> md5sum" |
| sh "md5sum -c --status md5sum" |
| sh "unzip ${PACKER_ZIP}" |
| } |
| } |
| // clean images dir before building |
| sh(script: String.format("rm -rf %s/images/*", BUILD_OS), returnStatus: true) |
| // clean virtualenv is exists |
| sh(script: String.format("rm -rf %s/venv", workspace), returnStatus: true) |
| |
| stage("build") { |
| dir(BUILD_OS) { |
| withEnv(extra_vars + [String.format("PATH=%s:%s/bin", env.PATH, workspace), |
| "PACKER_LOG_PATH=${workspace}/packer.log", |
| "PACKER_LOG=1", |
| "IMAGE_NAME=${IMAGE_NAME}", |
| "TMPDIR=${workspace}/tmp" |
| ]) { |
| if (PACKER_DEBUG == 'true') { |
| PACKER_ARGS = "${PACKER_ARGS} -debug" |
| } |
| |
| if (fileExists("config-drive/user-data.yaml")) { |
| common.infoMsg("Creating cloud-config drive") |
| if (fileExists("config-drive/cloudata.iso")) { |
| sh "rm -v config-drive/cloudata.iso" |
| } |
| sh "cloud-localds config-drive/cloudata.iso config-drive/user-data.yaml" |
| } |
| sh "packer build -only=${BUILD_ONLY} ${PACKER_ARGS} -parallel=false template.json" |
| |
| def packerStatus = sh(script: "grep \"Some builds didn't complete successfully and had errors\" ${PACKER_LOG_PATH}", returnStatus: true) |
| // grep returns 0 if find something |
| if (packerStatus != 0) { |
| if (buildTypes.contains("qemu")) { |
| def imageQemu = sh(script: "find images/ | grep -- '-qemu-' | tail -1", returnStdout: true).trim() |
| if (imageQemu != null && imageQemu != "") { |
| def qemuConvertStatus = sh(script: "qemu-img convert -c -O qcow2 ${imageQemu} ${imageQemu}.qcow2", returnStatus:true) |
| if(qemuConvertStatus == 0){ |
| def imageDir = imageQemu.substring(0, imageQemu.lastIndexOf("/") + 1) |
| def imageQemuName = imageQemu.substring(imageQemu.lastIndexOf("/") + 1) |
| def moveResult = sh(script: "mv ${imageQemu}.qcow2 ${imageDir}..", returnStatus: true) |
| if(moveResult == 0){ |
| sh "rm -rf ${imageDir}" |
| sh "rm -f ${imageQemu}" |
| createdImages.add(imageQemuName+".qcow2") |
| } |
| }else{ |
| throw new Exception("Qemu image convert failed") |
| } |
| } else { |
| throw new Exception("No Qemu images with '-qemu-' in its path") |
| } |
| } |
| if (buildTypes.contains("docker")) { |
| def imageDocker = sh(script: "find images/ | grep -- '-docker-' | grep '.tar\$' | tail -1", returnStdout: true).trim() |
| if (imageDocker != null && imageDocker != "") { |
| def pbZip2Status = sh(script: "pbzip2 ${imageDocker}", returnStatus: true) |
| if(pbZip2Status == 0){ |
| sh "rm -f ${imageDocker}" |
| createdImages.add(imageDocker+".bz2") |
| }else{ |
| throw new Exception("pbzip2 image convert failed") |
| } |
| } else { |
| throw new Exception("No Docker images with '-docker-' in its path") |
| } |
| } |
| |
| } else { |
| throw new Exception("Packer build failed") |
| } |
| } |
| } |
| } |
| stage("upload"){ |
| dir(BUILD_OS + "/images") { |
| def images = findFiles(glob: "*.*") |
| def imageBuilds = [:] |
| def openstack = new com.mirantis.mk.Openstack() |
| def openstackEnv = String.format("%s/venv", workspace); |
| def openstackVersion = OPENSTACK_API_CLIENT ? OPENSTACK_API_CLIENT : 'liberty' |
| def rcFile = openstack.createOpenstackEnv(workspace, GLANCE_URL, GLANCE_CREDENTIALS_ID, GLANCE_PROJECT) |
| def glanceImgTypes = GLANCE_IMG_TYPES.tokenize(" ") |
| openstack.setupOpenstackVirtualenv(openstackEnv, openstackVersion) |
| openstack.runOpenstackCommand("pip install python-glanceclient==1.0.0", rcFile, openstackEnv) |
| for (int i = 0; i < images.size(); i++) { |
| def imageName = images[i].name |
| def imageNameList = imageName.tokenize(".") |
| def imageType = "." + imageNameList[imageNameList.size() - 1] |
| if(imageType.equals(".md5")){ |
| continue; |
| } |
| |
| imageBuilds["build${i}"]={ |
| if (SKIP_UPLOAD != 'true') { |
| sh "md5sum ${imageName} > ${imageName}.md5" |
| common.infoMsg("Uploading image " + imageName) |
| def uploadImageStatus = sh(script: "curl -f -T ${imageName} ${UPLOAD_URL}", returnStatus: true) |
| def uploadMd5Status = sh(script: "curl -f -T ${imageName}.md5 ${UPLOAD_URL}", returnStatus: true) |
| // upload latest |
| def latestImageName = imageName.substring(0, imageName.lastIndexOf("-")) + "-latest" + imageType |
| common.infoMsg("Uploading image ${imageName} as latest") |
| def uploadLatestStatus = sh(script: "curl -f -T ${imageName} ${UPLOAD_URL}${latestImageName}", returnStatus: true) |
| def uploadLatestMd5Status = sh(script: "curl -f -T ${imageName}.md5 ${UPLOAD_URL}${latestImageName}.md5", returnStatus: true) |
| if(uploadLatestStatus != 0 || uploadLatestMd5Status != 0){ |
| common.errorMsg("Latest image upload failed") |
| } |
| if (GLANCE_UPLOAD == 'true' && glanceImgTypes.contains(imageType.substring(1))) { |
| def glanceRunArgs = String.format("%s --disk-format %s --container-format bare", GLANCE_ARGS, imageType.substring(1)) |
| if (GLANCE_PUBLIC == 'true') { |
| glanceRunArgs += " --visibility public" |
| } |
| |
| def imageShortName = imageNameList.get(0) |
| openstack.runOpenstackCommand(String.format("glance image-create --name '%s' %s --file %s", imageShortName, glanceRunArgs, imageName), rcFile, openstackEnv) |
| } |
| if(uploadImageStatus==0 && uploadMd5Status == 0){ |
| uploadedImages.add(imageName) |
| sh(String.format("rm -r %s %s.md5",imageName, imageName)) |
| createdImages.remove(imageName) |
| }else{ |
| throw new Exception("Image upload failed") |
| } |
| } |
| if (CLEANUP_OLD == 'true') { |
| def remoteImages = sh(script: "curl -f -sss ${UPLOAD_URL} | grep -Eo '>.*\\.(qcow2|box|tar\\.bz2)</a>' | sed -e 's,>,,g' -e 's,</a,,g'", returnStdout: true) |
| if (remoteImages != "") { |
| def cleanupImages = getCleanupImageList(remoteImages, imageType, BUILD_OS) |
| def deleteCount = cleanupImages.size() - Integer.parseInt(CLEANUP_KEEP) |
| if (deleteCount > 0) { |
| for (int j = 0; j < deleteCount; j++) { |
| common.infoMsg(String.format("Deleting image %s from aptly", cleanupImages[j])) |
| sh "curl -f -X DELETE ${UPLOAD_URL}" + cleanupImages[j] |
| sh "curl -f -X DELETE ${UPLOAD_URL}" + cleanupImages[j] + ".md5" |
| cleanedImages.add(cleanupImages[j]) |
| } |
| } |
| } |
| } |
| } |
| } |
| parallel imageBuilds |
| common.infoMsg(String.format("Uploaded %s images with names %s", uploadedImages.size(), uploadedImages.toString())) |
| common.infoMsg(String.format("Cleaned %s images with names %s", cleanedImages.size(), cleanedImages.toString())) |
| } |
| } |
| } catch (Throwable e) { |
| // If there was an error or exception thrown, the build failed |
| currentBuild.result = "FAILURE" |
| throw e |
| } finally { |
| common.sendNotification(currentBuild.result, "", ["slack"]) |
| if (buildTypes.contains("docker")) { |
| withEnv(["PACKER_LOG_PATH=${workspace}/packer.log"]) { |
| sh "docker rmi --force \$(grep \"docker: Image ID:\" ${PACKER_LOG_PATH} | cut -d : -f 6 | head -1 | sed s,\\ ,,g) || true" |
| } |
| } |
| // clean created images if error occured |
| if(!createdImages.isEmpty()){ |
| dir(BUILD_OS + "/images"){ |
| for(int i=0;i<createdImages.size();i++){ |
| sh String.format("rm -f %s",createdImages.get(i)) |
| } |
| } |
| } |
| } |
| } |
| } |