MCP pipeline library merged with ccp-pipeline-libs repo.
Change-Id: I74080e18c5a482b7bf44b5516228b7bfe1fe3586
diff --git a/src/com/mirantis/mk/Aptly.groovy b/src/com/mirantis/mk/Aptly.groovy
new file mode 100644
index 0000000..c1197a5
--- /dev/null
+++ b/src/com/mirantis/mk/Aptly.groovy
@@ -0,0 +1,98 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Aptly functions
+ *
+ */
+
+/**
+ * Upload package into local repo
+ *
+ * @param file File path
+ * @param server Server host
+ * @param repo Repository name
+ */
+def uploadPackage(file, server, repo, skipExists=false) {
+ def pkg = file.split('/')[-1].split('_')[0]
+ def jobName = currentBuild.build().environment.JOB_NAME
+
+ sh("curl -v -f -F file=@${file} ${server}/api/files/${pkg}")
+ sh("curl -v -o curl_out_${pkg}.log -f -X POST ${server}/api/repos/${repo}/file/${pkg}")
+
+ try {
+ sh("cat curl_out_${pkg}.log | json_pp | grep 'Unable to add package to repo' && exit 1 || exit 0")
+ } catch (err) {
+ sh("curl -s -f -X DELETE ${server}/api/files/${pkg}")
+ if (skipExists == true) {
+ println "[WARN] Package ${pkg} already exists in repo so skipping it's upload as requested"
+ } else {
+ error("Package ${pkg} already exists in repo, did you forget to add changelog entry and raise version?")
+ }
+ }
+}
+
+/**
+ * Build step to upload package. For use with eg. parallel
+ *
+ * @param file File path
+ * @param server Server host
+ * @param repo Repository name
+ */
+def uploadPackageStep(file, server, repo, skipExists=false) {
+ return {
+ uploadPackage(
+ file,
+ server,
+ repo,
+ skipExists
+ )
+ }
+}
+
+def snapshotRepo(server, repo, timestamp = null) {
+ // XXX: timestamp parameter is obsoleted, time of snapshot creation is
+ // what we should always use, not what calling pipeline provides
+ def now = new Date();
+ def ts = now.format("yyyyMMddHHmmss", TimeZone.getTimeZone('UTC'));
+
+ def snapshot = "${repo}-${ts}"
+ sh("curl -f -X POST -H 'Content-Type: application/json' --data '{\"Name\":\"$snapshot\"}' ${server}/api/repos/${repo}/snapshots")
+}
+
+def cleanupSnapshots(server, config='/etc/aptly-publisher.yaml', opts='-d --timeout 600') {
+ sh("aptly-publisher -c ${config} ${opts} --url ${server} cleanup")
+}
+
+def diffPublish(server, source, target, components=null, opts='--timeout 600') {
+ if (components) {
+ def componentsStr = components.join(' ')
+ opts = "${opts} --components ${componentsStr}"
+ }
+ sh("aptly-publisher --dry --url ${server} promote --source ${source} --target ${target} --diff ${opts}")
+}
+
+def promotePublish(server, source, target, recreate=false, components=null, packages=null, diff=false, opts='-d --timeout 600') {
+ if (components) {
+ def componentsStr = components.join(' ')
+ opts = "${opts} --components ${componentsStr}"
+ }
+ if (packages) {
+ def packagesStr = packages.join(' ')
+ opts = "${opts} --packages ${packagesStr}"
+ }
+ if (recreate.toBoolean() == true) {
+ opts = "${opts} --recreate"
+ }
+ if (diff.toBoolean() == true) {
+ opts = "--dry --diff"
+ }
+ sh("aptly-publisher --url ${server} promote --source ${source} --target ${target} ${opts}")
+}
+
+def publish(server, config='/etc/aptly-publisher.yaml', recreate=false, opts='-d --timeout 600') {
+ if (recreate == true) {
+ opts = "${opts} --recreate"
+ }
+ sh("aptly-publisher --url ${server} -c ${config} ${opts} publish")
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/artifactory.groovy b/src/com/mirantis/mk/Artifactory.groovy
similarity index 93%
rename from src/com/mirantis/mk/artifactory.groovy
rename to src/com/mirantis/mk/Artifactory.groovy
index 494552e..d1eb218 100644
--- a/src/com/mirantis/mk/artifactory.groovy
+++ b/src/com/mirantis/mk/Artifactory.groovy
@@ -161,8 +161,9 @@
* @param outRepo Output repository name used in context of this
* connection
* @param credentialsID ID of credentials store entry
+ * @param serverName Artifactory server name (optional)
*/
-def connection(url, dockerRegistryBase, dockerRegistrySsl, outRepo, credentialsId = "artifactory") {
+def connection(url, dockerRegistryBase, dockerRegistrySsl, outRepo, credentialsId = "artifactory", serverName = null) {
params = [
"url": url,
"credentialsId": credentialsId,
@@ -179,6 +180,11 @@
} else {
params["docker"]["proto"] = "http"
}
+
+ if (serverName ?: null) {
+ params['server'] = Artifactory.server(serverName)
+ }
+
params["docker"]["url"] = "${params.docker.proto}://${params.outRepo}.${params.docker.base}"
return params
@@ -308,6 +314,11 @@
return deleted
}
+@NonCPS
+def convertProperties(properties) {
+ return properties.collect { k,v -> "$k=$v" }.join(';')
+}
+
/**
* Upload debian package
*
@@ -316,31 +327,30 @@
* @param properties Map with additional artifact properties
* @param timestamp Image tag
*/
-def uploadDebian(art, file, properties, distribution, component, timestamp, data = null) {
- def fh
- if (file instanceof java.io.File) {
- fh = file
- } else {
- fh = new File(file)
- }
- def arch = fh.name.split('_')[-1].split('\\.')[0]
- if (data) {
- restPut(art, "/${art.outRepo}/pool/${fh.name};deb.distribution=${distribution};deb.component=${component};deb.architecture=${arch}", data)
- } else {
- restPut(art, "/${art.outRepo}/pool/${fh.name};deb.distribution=${distribution};deb.component=${component};deb.architecture=${arch}", fh)
- }
+def uploadDebian(art, file, properties, distribution, component, timestamp) {
+ def arch = file.split('_')[-1].split('\\.')[0]
/* Set artifact properties */
properties["build.number"] = currentBuild.build().environment.BUILD_NUMBER
properties["build.name"] = currentBuild.build().environment.JOB_NAME
properties["timestamp"] = timestamp
- setProperty(
- art,
- "pool/${fh.name}",
- timestamp,
- properties
- )
+
+ properties["deb.distribution"] = distribution
+ properties["deb.component"] = component
+ properties["deb.architecture"] = arch
+ props = convertProperties(properties)
+
+ def uploadSpec = """{
+ "files": [
+ {
+ "pattern": "${file}",
+ "target": "${art.outRepo}",
+ "props": "${props}"
+ }
+ ]
+ }"""
+ art.server.upload(uploadSpec)
}
/**
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
new file mode 100644
index 0000000..02fcada
--- /dev/null
+++ b/src/com/mirantis/mk/Common.groovy
@@ -0,0 +1,483 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Common functions
+ *
+ */
+
+/**
+ * Generate current timestamp
+ *
+ * @param format Defaults to yyyyMMddHHmmss
+ */
+def getDatetime(format="yyyyMMddHHmmss") {
+ def now = new Date();
+ return now.format(format, TimeZone.getTimeZone('UTC'));
+}
+
+/**
+ * Parse HEAD of current directory and return commit hash
+ */
+def getGitCommit() {
+ git_commit = sh (
+ script: 'git rev-parse HEAD',
+ returnStdout: true
+ ).trim()
+ return git_commit
+}
+
+/**
+ * Return workspace.
+ * Currently implemented by calling pwd so it won't return relevant result in
+ * dir context
+ */
+def getWorkspace() {
+ def workspace = sh script: 'pwd', returnStdout: true
+ workspace = workspace.trim()
+ return workspace
+}
+
+/**
+ * Get credentials from store
+ *
+ * @param id Credentials name
+ */
+def getCredentials(id) {
+ def creds = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
+ com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials.class,
+ jenkins.model.Jenkins.instance
+ )
+
+ for (Iterator<String> credsIter = creds.iterator(); credsIter.hasNext();) {
+ c = credsIter.next();
+ if ( c.id == id ) {
+ return c;
+ }
+ }
+
+ throw new Exception("Could not find credentials for ID ${id}")
+}
+
+/**
+ * Abort build, wait for some time and ensure we will terminate
+ */
+def abortBuild() {
+ currentBuild.build().doStop()
+ sleep(180)
+ // just to be sure we will terminate
+ throw new InterruptedException()
+}
+
+/**
+ * Print informational message
+ *
+ * @param msg
+ * @param color Colorful output or not
+ */
+def infoMsg(msg, color = true) {
+ printMsg(msg, "cyan")
+}
+
+/**
+ * Print error message
+ *
+ * @param msg
+ * @param color Colorful output or not
+ */
+def errorMsg(msg, color = true) {
+ printMsg(msg, "red")
+}
+
+/**
+ * Print success message
+ *
+ * @param msg
+ * @param color Colorful output or not
+ */
+def successMsg(msg, color = true) {
+ printMsg(msg, "green")
+}
+
+/**
+ * Print warning message
+ *
+ * @param msg
+ * @param color Colorful output or not
+ */
+def warningMsg(msg, color = true) {
+ printMsg(msg, "blue")
+}
+
+/**
+ * Print message
+ *
+ * @param msg Message to be printed
+ * @param level Level of message (default INFO)
+ * @param color Color to use for output or false (default)
+ */
+def printMsg(msg, color = false) {
+ colors = [
+ 'red' : '\u001B[31m',
+ 'black' : '\u001B[30m',
+ 'green' : '\u001B[32m',
+ 'yellow': '\u001B[33m',
+ 'blue' : '\u001B[34m',
+ 'purple': '\u001B[35m',
+ 'cyan' : '\u001B[36m',
+ 'white' : '\u001B[37m',
+ 'reset' : '\u001B[0m'
+ ]
+ if (color != false) {
+ wrap([$class: 'AnsiColorBuildWrapper']) {
+ print "${colors[color]}${msg}${colors.reset}"
+ }
+ } else {
+ print "[${level}] ${msg}"
+ }
+}
+
+/**
+ * Traverse directory structure and return list of files
+ *
+ * @param path Path to search
+ * @param type Type of files to search (groovy.io.FileType.FILES)
+ */
+@NonCPS
+def getFiles(path, type=groovy.io.FileType.FILES) {
+ files = []
+ new File(path).eachFile(type) {
+ files[] = it
+ }
+ return files
+}
+
+/**
+ * Helper method to convert map into form of list of [key,value] to avoid
+ * unserializable exceptions
+ *
+ * @param m Map
+ */
+@NonCPS
+def entries(m) {
+ m.collect {k, v -> [k, v]}
+}
+
+/**
+ * Opposite of build-in parallel, run map of steps in serial
+ *
+ * @param steps Map of String<name>: CPSClosure2<step>
+ */
+def serial(steps) {
+ stepsArray = entries(steps)
+ for (i=0; i < stepsArray.size; i++) {
+ s = stepsArray[i]
+ dummySteps = ["${s[0]}": s[1]]
+ parallel dummySteps
+ }
+}
+
+/**
+ * Get password credentials from store
+ *
+ * @param id Credentials name
+ */
+def getPasswordCredentials(id) {
+ def creds = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
+ com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials.class,
+ jenkins.model.Jenkins.instance
+ )
+
+ for (Iterator<String> credsIter = creds.iterator(); credsIter.hasNext();) {
+ c = credsIter.next();
+ if ( c.id == id ) {
+ return c;
+ }
+ }
+
+ throw new Exception("Could not find credentials for ID ${id}")
+}
+
+/**
+ * Get SSH credentials from store
+ *
+ * @param id Credentials name
+ */
+def getSshCredentials(id) {
+ def creds = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
+ com.cloudbees.plugins.credentials.common.StandardUsernameCredentials.class,
+ jenkins.model.Jenkins.instance
+ )
+
+ for (Iterator<String> credsIter = creds.iterator(); credsIter.hasNext();) {
+ c = credsIter.next();
+ if ( c.id == id ) {
+ return c;
+ }
+ }
+
+ throw new Exception("Could not find credentials for ID ${id}")
+}
+/**
+ * Setup ssh agent and add private key
+ *
+ * @param credentialsId Jenkins credentials name to lookup private key
+ */
+def prepareSshAgentKey(credentialsId) {
+ c = getSshCredentials(credentialsId)
+ sh("test -d ~/.ssh || mkdir -m 700 ~/.ssh")
+ sh('pgrep -l -u $USER -f | grep -e ssh-agent\$ >/dev/null || ssh-agent|grep -v "Agent pid" > ~/.ssh/ssh-agent.sh')
+ sh("set +x; echo '${c.getPrivateKey()}' > ~/.ssh/id_rsa_${credentialsId} && chmod 600 ~/.ssh/id_rsa_${credentialsId}; set -x")
+ agentSh("ssh-add ~/.ssh/id_rsa_${credentialsId}")
+}
+
+/**
+ * Execute command with ssh-agent
+ *
+ * @param cmd Command to execute
+ */
+def agentSh(cmd) {
+ sh(". ~/.ssh/ssh-agent.sh && ${cmd}")
+}
+
+/**
+ * Ensure entry in SSH known hosts
+ *
+ * @param url url of remote host
+ */
+def ensureKnownHosts(url) {
+ def hostArray = getKnownHost(url)
+ sh "test -f ~/.ssh/known_hosts && grep ${hostArray[0]} ~/.ssh/known_hosts || ssh-keyscan -p ${hostArray[1]} ${hostArray[0]} >> ~/.ssh/known_hosts"
+}
+
+@NonCPS
+def getKnownHost(url){
+ // test for git@github.com:organization/repository like URLs
+ def p = ~/.+@(.+\..+)\:{1}.*/
+ def result = p.matcher(url)
+ def host = ""
+ if (result.matches()) {
+ host = result.group(1)
+ port = 22
+ } else {
+ parsed = new URI(url)
+ host = parsed.host
+ port = parsed.port && parsed.port > 0 ? parsed.port: 22
+ }
+ return [host,port]
+}
+
+/**
+ * Mirror git repository, merge target changes (downstream) on top of source
+ * (upstream) and push target or both if pushSource is true
+ *
+ * @param sourceUrl Source git repository
+ * @param targetUrl Target git repository
+ * @param credentialsId Credentials id to use for accessing source/target
+ * repositories
+ * @param branches List or comma-separated string of branches to sync
+ * @param followTags Mirror tags
+ * @param pushSource Push back into source branch, resulting in 2-way sync
+ * @param pushSourceTags Push target tags into source or skip pushing tags
+ * @param gitEmail Email for creation of merge commits
+ * @param gitName Name for creation of merge commits
+ */
+def mirrorGit(sourceUrl, targetUrl, credentialsId, branches, followTags = false, pushSource = false, pushSourceTags = false, gitEmail = 'jenkins@localhost', gitName = 'Jenkins') {
+ if (branches instanceof String) {
+ branches = branches.tokenize(',')
+ }
+
+ prepareSshAgentKey(credentialsId)
+ ensureKnownHosts(targetUrl)
+ sh "git config user.email '${gitEmail}'"
+ sh "git config user.name '${gitName}'"
+
+ sh "git remote | grep target || git remote add target ${TARGET_URL}"
+ agentSh "git remote update --prune"
+
+ for (i=0; i < branches.size; i++) {
+ branch = branches[i]
+ sh "git branch | grep ${branch} || git checkout -b ${branch} origin/${branch}"
+ sh "git branch | grep ${branch} && git checkout ${branch} && git reset --hard origin/${branch}"
+
+ sh "git ls-tree target/${branch} && git merge --no-edit --ff target/${branch} || echo 'Target repository is empty, skipping merge'"
+ followTagsArg = followTags ? "--follow-tags" : ""
+ agentSh "git push ${followTagsArg} target HEAD:${branch}"
+
+ if (pushSource == true) {
+ followTagsArg = followTags && pushSourceTags ? "--follow-tags" : ""
+ agentSh "git push ${followTagsArg} origin HEAD:${branch}"
+ }
+ }
+
+ if (followTags == true) {
+ agentSh "git push target --tags"
+
+ if (pushSourceTags == true) {
+ agentSh "git push origin --tags"
+ }
+ }
+}
+
+/**
+ * Tests Jenkins instance for existence of plugin with given name
+ * @param pluginName plugin short name to test
+ * @return boolean result
+ */
+@NonCPS
+def jenkinsHasPlugin(pluginName){
+ return Jenkins.instance.pluginManager.plugins.collect{p -> p.shortName}.contains(pluginName)
+}
+
+@NonCPS
+def _needNotification(notificatedTypes, buildStatus, jobName) {
+ if(notificatedTypes && notificatedTypes.contains("onchange")){
+ if(jobName){
+ def job = Jenkins.instance.getItem(jobName)
+ def numbuilds = job.builds.size()
+ if (numbuilds > 0){
+ //actual build is first for some reasons, so last finished build is second
+ def lastBuild = job.builds[1]
+ if(lastBuild){
+ if(lastBuild.result.toString().toLowerCase().equals(buildStatus)){
+ println("Build status didn't changed since last build, not sending notifications")
+ return false;
+ }
+ }
+ }
+ }
+ }else if(!notificatedTypes.contains(buildStatus)){
+ return false;
+ }
+ return true;
+}
+
+/**
+ * Send notification to all enabled notifications services
+ * @param buildStatus message type (success, warning, error), null means SUCCESSFUL
+ * @param msgText message text
+ * @param enabledNotifications list of enabled notification types, types: slack, hipchat, email, default empty
+ * @param notificatedTypes types of notifications will be sent, default onchange - notificate if current build result not equal last result;
+ * otherwise use - ["success","unstable","failed"]
+ * @param jobName optional job name param, if empty env.JOB_NAME will be used
+ * @param buildNumber build number param, if empty env.JOB_NAME will be used
+ * @param buildUrl build url param, if empty env.JOB_NAME will be used
+ * @param mailFrom mail FROM param, if empty "jenkins" will be used, it's mandatory for sending email notifications
+ * @param mailTo mail TO param, it's mandatory for sending email notifications
+ */
+def sendNotification(buildStatus, msgText="", enabledNotifications = [], notificatedTypes=["onchange"], jobName=null, buildNumber=null, buildUrl=null, mailFrom="jenkins", mailTo=null){
+ // Default values
+ def colorName = 'blue'
+ def colorCode = '#0000FF'
+ def buildStatusParam = buildStatus != null && buildStatus != "" ? buildStatus : "SUCCESS"
+ def jobNameParam = jobName != null && jobName != "" ? jobName : env.JOB_NAME
+ def buildNumberParam = buildNumber != null && buildNumber != "" ? buildNumber : env.BUILD_NUMBER
+ def buildUrlParam = buildUrl != null && buildUrl != "" ? buildUrl : env.BUILD_URL
+ def subject = "${buildStatusParam}: Job '${jobNameParam} [${buildNumberParam}]'"
+ def summary = "${subject} (${buildUrlParam})"
+
+ if(msgText != null && msgText != ""){
+ summary+="\n${msgText}"
+ }
+ if(buildStatusParam.toLowerCase().equals("success")){
+ colorCode = "#00FF00"
+ colorName = "green"
+ }else if(buildStatusParam.toLowerCase().equals("unstable")){
+ colorCode = "#FFFF00"
+ colorName = "yellow"
+ }else if(buildStatusParam.toLowerCase().equals("failure")){
+ colorCode = "#FF0000"
+ colorName = "red"
+ }
+ if(_needNotification(notificatedTypes, buildStatusParam.toLowerCase(), jobNameParam)){
+ if(enabledNotifications.contains("slack") && jenkinsHasPlugin("slack")){
+ try{
+ slackSend color: colorCode, message: summary
+ }catch(Exception e){
+ println("Calling slack plugin failed")
+ e.printStackTrace()
+ }
+ }
+ if(enabledNotifications.contains("hipchat") && jenkinsHasPlugin("hipchat")){
+ try{
+ hipchatSend color: colorName.toUpperCase(), message: summary
+ }catch(Exception e){
+ println("Calling hipchat plugin failed")
+ e.printStackTrace()
+ }
+ }
+ if(enabledNotifications.contains("email") && mailTo != null && mailTo != "" && mailFrom != null && mailFrom != ""){
+ try{
+ mail body: summary, from: mailFrom, subject: subject, to: mailTo
+ }catch(Exception e){
+ println("Sending mail plugin failed")
+ e.printStackTrace()
+ }
+ }
+ }
+}
+
+/**
+ * Execute git clone and checkout stage from gerrit review
+ *
+ * @param config LinkedHashMap
+ * config includes next parameters:
+ * - credentialsId, id of user which should make checkout
+ * - withMerge, prevent detached mode in repo
+ * - withWipeOut, wipe repository and force clone
+ *
+ * Usage example:
+ * //anonymous gerrit checkout
+ * def gitFunc = new com.mirantis.mcp.Git()
+ * gitFunc.gerritPatchsetCheckout([
+ * withMerge : true
+ * ])
+ *
+ * def gitFunc = new com.mirantis.mcp.Git()
+ * gitFunc.gerritPatchsetCheckout([
+ * credentialsId : 'mcp-ci-gerrit',
+ * withMerge : true
+ * ])
+ */
+def gerritPatchsetCheckout(LinkedHashMap config) {
+ def merge = config.get('withMerge', false)
+ def wipe = config.get('withWipeOut', false)
+ def credentials = config.get('credentialsId','')
+
+ // default parameters
+ def scmExtensions = [
+ [$class: 'CleanCheckout'],
+ [$class: 'BuildChooserSetting', buildChooser: [$class: 'GerritTriggerBuildChooser']]
+ ]
+ def scmUserRemoteConfigs = [
+ name: 'gerrit',
+ refspec: "${GERRIT_REFSPEC}"
+ ]
+
+ if (credentials == '') {
+ // then try to checkout in anonymous mode
+ scmUserRemoteConfigs.put('url',"https://${GERRIT_HOST}/${GERRIT_PROJECT}")
+ } else {
+ // else use ssh checkout
+ scmUserRemoteConfigs.put('url',"ssh://${GERRIT_NAME}@${GERRIT_HOST}:${GERRIT_PORT}/${GERRIT_PROJECT}.git")
+ scmUserRemoteConfigs.put('credentialsId',credentials)
+ }
+
+ // if we need to "merge" code from patchset to GERRIT_BRANCH branch
+ if (merge) {
+ scmExtensions.add([$class: 'LocalBranch', localBranch: "${GERRIT_BRANCH}"])
+ }
+ // we need wipe workspace before checkout
+ if (wipe) {
+ scmExtensions.add([$class: 'WipeWorkspace'])
+ }
+
+ checkout(
+ scm: [
+ $class: 'GitSCM',
+ branches: [[name: "${GERRIT_BRANCH}"]],
+ extensions: scmExtensions,
+ userRemoteConfigs: [scmUserRemoteConfigs]
+ ]
+ )
+}
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
new file mode 100644
index 0000000..acd225a
--- /dev/null
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -0,0 +1,154 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Debian functions
+ *
+ */
+
+def cleanup(image="debian:sid") {
+ def common = new com.mirantis.mk.Common()
+ def img = docker.image(image)
+
+ workspace = common.getWorkspace()
+ sh("docker run -e DEBIAN_FRONTEND=noninteractive -v ${workspace}:${workspace} -w ${workspace} --rm=true --privileged ${image} /bin/bash -c 'rm -rf build-area || true'")
+}
+
+/*
+ * Build binary Debian package from existing dsc
+ *
+ * @param file dsc file to build
+ * @param image Image name to use for build (default debian:sid)
+ */
+def buildBinary(file, image="debian:sid", extraRepoUrl=null, extraRepoKeyUrl=null) {
+ def common = new com.mirantis.mk.Common()
+ def pkg = file.split('/')[-1].split('_')[0]
+ def img = docker.image(image)
+
+ workspace = common.getWorkspace()
+ sh("""docker run -e DEBIAN_FRONTEND=noninteractive -v ${workspace}:${workspace} -w ${workspace} --rm=true --privileged ${image} /bin/bash -c '
+ which eatmydata || (apt-get update && apt-get install -y eatmydata) &&
+ export LD_LIBRARY_PATH=\${LD_LIBRARY_PATH:+"\$LD_LIBRARY_PATH:"}/usr/lib/libeatmydata &&
+ export LD_PRELOAD=\${LD_PRELOAD:+"\$LD_PRELOAD "}libeatmydata.so &&
+ [[ -z "${extraRepoUrl}" && "${extraRepoUrl}" != "null" ]] || echo "${extraRepoUrl}" >/etc/apt/sources.list.d/extra.list &&
+ [[ -z "${extraRepoKeyUrl}" && "${extraRepoKeyUrl}" != "null" ]] || (
+ which curl || (apt-get update && apt-get install -y curl) &&
+ curl --insecure -ss -f "${extraRepoKeyUrl}" | apt-key add -
+ ) &&
+ apt-get update && apt-get install -y build-essential devscripts equivs &&
+ dpkg-source -x ${file} build-area/${pkg} && cd build-area/${pkg} &&
+ mk-build-deps -t "apt-get -o Debug::pkgProblemResolver=yes -y" -i debian/control
+ debuild --no-lintian -uc -us -b'""")
+}
+
+/*
+ * Build source package from directory
+ *
+ * @param dir Tree to build
+ * @param image Image name to use for build (default debian:sid)
+ * @param snapshot Generate snapshot version (default false)
+ */
+def buildSource(dir, image="debian:sid", snapshot=false, gitEmail='jenkins@dummy.org', gitName='Jenkins', revisionPostfix="") {
+ def isGit
+ try {
+ sh("test -d ${dir}/.git")
+ isGit = true
+ } catch (Exception e) {
+ isGit = false
+ }
+
+ if (isGit == true) {
+ buildSourceGbp(dir, image, snapshot, gitEmail, gitName, revisionPostfix)
+ } else {
+ buildSourceUscan(dir, image)
+ }
+}
+
+/*
+ * Build source package, fetching upstream code using uscan
+ *
+ * @param dir Tree to build
+ * @param image Image name to use for build (default debian:sid)
+ */
+def buildSourceUscan(dir, image="debian:sid") {
+ def common = new com.mirantis.mk.Common()
+ def img = docker.image(image)
+ workspace = common.getWorkspace()
+ sh("""docker run -e DEBIAN_FRONTEND=noninteractive -v ${workspace}:${workspace} -w ${workspace} --rm=true --privileged ${image} /bin/bash -c '
+ apt-get update && apt-get install -y build-essential devscripts &&
+ cd ${dir} && uscan --download-current-version &&
+ dpkg-buildpackage -S -nc -uc -us'""")
+}
+
+/*
+ * Build source package using git-buildpackage
+ *
+ * @param dir Tree to build
+ * @param image Image name to use for build (default debian:sid)
+ * @param snapshot Generate snapshot version (default false)
+ */
+def buildSourceGbp(dir, image="debian:sid", snapshot=false, gitEmail='jenkins@dummy.org', gitName='Jenkins', revisionPostfix="") {
+ def common = new com.mirantis.mk.Common()
+ def jenkinsUID = sh (
+ script: 'id -u',
+ returnStdout: true
+ ).trim()
+ def jenkinsGID = sh (
+ script: 'id -g',
+ returnStdout: true
+ ).trim()
+
+ if (! revisionPostfix) {
+ revisionPostfix = ""
+ }
+
+ def img = docker.image(image)
+ workspace = common.getWorkspace()
+ sh("""docker run -e DEBIAN_FRONTEND=noninteractive -e DEBFULLNAME='${gitName}' -e DEBEMAIL='${gitEmail}' -v ${workspace}:${workspace} -w ${workspace} --rm=true --privileged ${image} /bin/bash -exc '
+ which eatmydata || (apt-get update && apt-get install -y eatmydata) &&
+ export LD_LIBRARY_PATH=\${LD_LIBRARY_PATH:+"\$LD_LIBRARY_PATH:"}/usr/lib/libeatmydata &&
+ export LD_PRELOAD=\${LD_PRELOAD:+"\$LD_PRELOAD "}libeatmydata.so &&
+ apt-get update && apt-get install -y build-essential git-buildpackage sudo &&
+ groupadd -g ${jenkinsGID} jenkins &&
+ useradd -s /bin/bash --uid ${jenkinsUID} --gid ${jenkinsGID} -m jenkins &&
+ cd ${dir} &&
+ sudo -H -u jenkins git config --global user.name "${gitName}" &&
+ sudo -H -u jenkins git config --global user.email "${gitEmail}" &&
+ [[ "${snapshot}" == "false" ]] || (
+ VERSION=`dpkg-parsechangelog --count 1 | grep Version: | sed "s,Version: ,,g"` &&
+ UPSTREAM_VERSION=`echo \$VERSION | cut -d "-" -f 1` &&
+ REVISION=`echo \$VERSION | cut -d "-" -f 2` &&
+ TIMESTAMP=`date +%Y%m%d%H%M` &&
+ if [[ "`cat debian/source/format`" = *quilt* ]]; then
+ UPSTREAM_BRANCH=`(grep upstream-branch debian/gbp.conf || echo master) | cut -d = -f 2 | tr -d " "` &&
+ UPSTREAM_REV=`git rev-parse --short origin/\$UPSTREAM_BRANCH` &&
+ NEW_UPSTREAM_VERSION="\$UPSTREAM_VERSION+\$TIMESTAMP.\$UPSTREAM_REV" &&
+ NEW_VERSION=\$NEW_UPSTREAM_VERSION-\$REVISION$revisionPostfix &&
+ echo "Generating new upstream version \$NEW_UPSTREAM_VERSION" &&
+ sudo -H -u jenkins git tag \$NEW_UPSTREAM_VERSION origin/\$UPSTREAM_BRANCH &&
+ sudo -H -u jenkins git merge -X theirs \$NEW_UPSTREAM_VERSION
+ else
+ NEW_VERSION=\$VERSION+\$TIMESTAMP.`git rev-parse --short HEAD`$revisionPostfix
+ fi &&
+ sudo -H -u jenkins gbp dch --auto --multimaint-merge --ignore-branch --new-version=\$NEW_VERSION --distribution `lsb_release -c -s` --force-distribution &&
+ sudo -H -u jenkins git add -u debian/changelog &&
+ sudo -H -u jenkins git commit -m "New snapshot version \$NEW_VERSION"
+ ) &&
+ gbp buildpackage -nc --git-force-create --git-notify=false --git-ignore-branch --git-ignore-new --git-verbose --git-export-dir=../build-area -S -uc -us'""")
+}
+
+/*
+ * Run lintian checks
+ *
+ * @param changes Changes file to test against
+ * @param profile Lintian profile to use (default debian)
+ * @param image Image name to use for build (default debian:sid)
+ */
+def runLintian(changes, profile="debian", image="debian:sid") {
+ def common = new com.mirantis.mk.Common()
+ def img = docker.image(image)
+ workspace = common.getWorkspace()
+ sh("""docker run -e DEBIAN_FRONTEND=noninteractive -v ${workspace}:${workspace} -w ${workspace} --rm=true --privileged ${image} /bin/bash -c '
+ apt-get update && apt-get install -y lintian &&
+ lintian -Ii -E --pedantic --profile=${profile} ${changes}'""")
+}
diff --git a/src/com/mirantis/mk/docker.groovy b/src/com/mirantis/mk/Docker.groovy
similarity index 100%
rename from src/com/mirantis/mk/docker.groovy
rename to src/com/mirantis/mk/Docker.groovy
diff --git a/src/com/mirantis/mk/git.groovy b/src/com/mirantis/mk/Git.groovy
similarity index 100%
rename from src/com/mirantis/mk/git.groovy
rename to src/com/mirantis/mk/Git.groovy
diff --git a/src/com/mirantis/mk/Http.groovy b/src/com/mirantis/mk/Http.groovy
new file mode 100644
index 0000000..7ffc015
--- /dev/null
+++ b/src/com/mirantis/mk/Http.groovy
@@ -0,0 +1,174 @@
+package com.mirantis.mk
+/**
+ *
+ * HTTP functions
+ *
+ */
+
+/**
+ * Make generic HTTP call and return parsed JSON
+ *
+ * @param url URL to make the request against
+ * @param method HTTP method to use (default GET)
+ * @param data JSON data to POST or PUT
+ * @param headers Map of additional request headers
+ */
+@NonCPS
+def sendHttpRequest(url, method = 'GET', data = null, headers = [:]) {
+
+ def connection = new URL(url).openConnection()
+ if (method != 'GET') {
+ connection.setRequestMethod(method)
+ }
+
+ if (data) {
+ headers['Content-Type'] = 'application/json'
+ }
+
+ headers['User-Agent'] = 'jenkins-groovy'
+ headers['Accept'] = 'application/json'
+
+ for (header in headers) {
+ connection.setRequestProperty(header.key, header.value)
+ }
+
+ if (data) {
+ connection.setDoOutput(true)
+ if (data instanceof String) {
+ dataStr = data
+ } else {
+ dataStr = new groovy.json.JsonBuilder(data).toString()
+ }
+ def output = new OutputStreamWriter(connection.outputStream)
+ //infoMsg("[HTTP] Request URL: ${url}, method: ${method}, headers: ${headers}, content: ${dataStr}")
+ output.write(dataStr)
+ output.close()
+ }
+
+ if ( connection.responseCode == 200 ) {
+ response = connection.inputStream.text
+ try {
+ response_content = new groovy.json.JsonSlurperClassic().parseText(response)
+ } catch (groovy.json.JsonException e) {
+ response_content = response
+ }
+ //successMsg("[HTTP] Response: code ${connection.responseCode}")
+ return response_content
+ } else {
+ //errorMsg("[HTTP] Response: code ${connection.responseCode}")
+ throw new Exception(connection.responseCode + ": " + connection.inputStream.text)
+ }
+
+}
+
+/**
+ * Make HTTP GET request
+ *
+ * @param url URL which will requested
+ * @param data JSON data to PUT
+ */
+def sendHttpGetRequest(url, data = null, headers = [:]) {
+ return sendHttpRequest(url, 'GET', data, headers)
+}
+
+/**
+ * Make HTTP POST request
+ *
+ * @param url URL which will requested
+ * @param data JSON data to PUT
+ */
+def sendHttpPostRequest(url, data = null, headers = [:]) {
+ return sendHttpRequest(url, 'POST', data, headers)
+}
+
+/**
+ * Make HTTP PUT request
+ *
+ * @param url URL which will requested
+ * @param data JSON data to PUT
+ */
+def sendHttpPutRequest(url, data = null, headers = [:]) {
+ return sendHttpRequest(url, 'PUT', data, headers)
+}
+
+/**
+ * Make HTTP DELETE request
+ *
+ * @param url URL which will requested
+ * @param data JSON data to PUT
+ */
+def sendHttpDeleteRequest(url, data = null, headers = [:]) {
+ return sendHttpRequest(url, 'DELETE', data, headers)
+}
+
+/**
+ * Make generic call using Salt REST API and return parsed JSON
+ *
+ * @param master Salt connection object
+ * @param uri URI which will be appended to Salt server base URL
+ * @param method HTTP method to use (default GET)
+ * @param data JSON data to POST or PUT
+ * @param headers Map of additional request headers
+ */
+def restCall(master, uri, method = 'GET', data = null, headers = [:]) {
+ def connection = new URL("${master.url}${uri}").openConnection()
+ if (method != 'GET') {
+ connection.setRequestMethod(method)
+ }
+
+ connection.setRequestProperty('User-Agent', 'jenkins-groovy')
+ connection.setRequestProperty('Accept', 'application/json')
+ if (master.authToken) {
+ // XXX: removeme
+ connection.setRequestProperty('X-Auth-Token', master.authToken)
+ }
+
+ for (header in headers) {
+ connection.setRequestProperty(header.key, header.value)
+ }
+
+ if (data) {
+ connection.setDoOutput(true)
+ if (data instanceof String) {
+ dataStr = data
+ } else {
+ connection.setRequestProperty('Content-Type', 'application/json')
+ dataStr = new groovy.json.JsonBuilder(data).toString()
+ }
+ def out = new OutputStreamWriter(connection.outputStream)
+ out.write(dataStr)
+ out.close()
+ }
+
+ if ( connection.responseCode >= 200 && connection.responseCode < 300 ) {
+ res = connection.inputStream.text
+ try {
+ return new groovy.json.JsonSlurperClassic().parseText(res)
+ } catch (Exception e) {
+ return res
+ }
+ } else {
+ throw new Exception(connection.responseCode + ": " + connection.inputStream.text)
+ }
+}
+
+/**
+ * Make GET request using Salt REST API and return parsed JSON
+ *
+ * @param master Salt connection object
+ * @param uri URI which will be appended to Salt server base URL
+ */
+def restGet(master, uri, data = null) {
+ return restCall(master, uri, 'GET', data)
+}
+
+/**
+ * Make POST request using Salt REST API and return parsed JSON
+ *
+ * @param master Salt connection object
+ * @param uri URI which will be appended to Docker server base URL
+ * @param data JSON Data to PUT
+ */
+def restPost(master, uri, data = null) {
+ return restCall(master, uri, 'POST', data, ['Accept': '*/*'])
+}
diff --git a/src/com/mirantis/mk/openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
similarity index 100%
rename from src/com/mirantis/mk/openstack.groovy
rename to src/com/mirantis/mk/Openstack.groovy
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
new file mode 100644
index 0000000..a7637e4
--- /dev/null
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -0,0 +1,212 @@
+package com.mirantis.mk
+/**
+ * Orchestration functions
+ *
+*/
+
+def validateFoundationInfra(master) {
+ def salt = new com.mirantis.mk.Salt()
+ salt.runSaltProcessStep(master, 'I@salt:master', 'cmd.run', ['salt-key'])
+ salt.runSaltProcessStep(master, 'I@salt:minion', 'test.version')
+ salt.runSaltProcessStep(master, 'I@salt:master', 'cmd.run', ['reclass-salt --top'])
+ salt.runSaltProcessStep(master, 'I@reclass:storage', 'reclass.inventory')
+ salt.runSaltProcessStep(master, 'I@salt:minion', 'state.show_top')
+}
+
+
+def installFoundationInfra(master) {
+ def salt = new com.mirantis.mk.Salt()
+ salt.runSaltProcessStep(master, 'I@salt:master', 'state.sls', ['salt.master,reclass'])
+ salt.runSaltProcessStep(master, 'I@linux:system', 'saltutil.refresh_pillar')
+ salt.runSaltProcessStep(master, 'I@linux:system', 'saltutil.sync_all')
+ salt.runSaltProcessStep(master, 'I@linux:system', 'state.sls', ['linux,openssh,salt.minion,ntp'])
+}
+
+
+def installOpenstackMkInfra(master) {
+ def salt = new com.mirantis.mk.Salt()
+ // Install keepaliveds
+ //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keepalived'])
+ salt.runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
+ // Check the keepalived VIPs
+ salt.runSaltProcessStep(master, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
+ // Install glusterfs
+ salt.runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
+ //runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.setup'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(master, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(master, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
+ salt.runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
+ // Install rabbitmq
+ salt.runSaltProcessStep(master, 'I@rabbitmq:server', 'state.sls', ['rabbitmq'])
+ // Check the rabbitmq status
+ salt.runSaltProcessStep(master, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
+ // Install galera
+ salt.runSaltProcessStep(master, 'I@galera:master', 'state.sls', ['galera'])
+ salt.runSaltProcessStep(master, 'I@galera:slave', 'state.sls', ['galera'])
+ // Check galera status
+ salt.runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
+ salt.runSaltProcessStep(master, 'I@galera:slave', 'mysql.status')
+ // Install haproxy
+ salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
+ salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'service.status', ['haproxy'])
+ salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
+ // Install memcached
+ salt.runSaltProcessStep(master, 'I@memcached:server', 'state.sls', ['memcached'])
+}
+
+
+def installOpenstackMkControl(master) {
+ def salt = new com.mirantis.mk.Salt()
+ // setup keystone service
+ //runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keystone.server'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
+ // populate keystone services/tenants/roles/users
+ salt.runSaltProcessStep(master, 'I@keystone:client', 'state.sls', ['keystone.client'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; keystone service-list'])
+ // Install glance and ensure glusterfs clusters
+ //runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glance.server'])
+ salt.runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'])
+ salt.runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glusterfs.client'])
+ // Update fernet tokens before doing request on keystone server
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
+ // Check glance service
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'])
+ // Install and check nova service
+ //runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['nova'])
+ salt.runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'])
+ // Install and check cinder service
+ //runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['cinder'])
+ salt.runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'])
+ // Install neutron service
+ //runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['neutron'])
+ salt.runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'])
+ // Install heat service
+ //runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'], 1)
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['heat'])
+ salt.runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'])
+ // Install horizon dashboard
+ runSaltProcessStep(master, 'I@horizon:server', 'state.sls', ['horizon'])
+ runSaltProcessStep(master, 'I@nginx:server', 'state.sls', ['nginx'])
+}
+
+
+def installOpenstackMkNetwork(master) {
+ def salt = new com.mirantis.mk.Salt()
+ // Install opencontrail database services
+ //runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
+ salt.runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail.database'])
+ salt.runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'])
+ // Install opencontrail control services
+ //runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
+ salt.runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail'])
+ salt.runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'])
+ // Provision opencontrail control services
+ salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl01 --host_ip 172.16.10.101 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+ salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl02 --host_ip 172.16.10.102 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+ salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl03 --host_ip 172.16.10.103 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
+ // Test opencontrail
+ salt.runSaltProcessStep(master, 'I@opencontrail:control', 'cmd.run', ['contrail-status'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
+ salt.runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
+}
+
+
+def installOpenstackMkCompute(master) {
+ def salt = new com.mirantis.mk.Salt()
+ // Configure compute nodes
+ salt.runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
+ salt.runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
+ // Provision opencontrail virtual routers
+ salt.runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_vrouter.py --host_name cmp01 --host_ip 172.16.10.105 --api_server_ip 172.16.10.254 --oper add --admin_user admin --admin_password workshop --admin_tenant_name admin'])
+ salt.runSaltProcessStep(master, 'I@nova:compute', 'system.reboot')
+}
+
+
+def installOpenstackMcpInfra(master) {
+ def salt = new com.mirantis.mk.Salt()
+ // Comment nameserver
+ salt.runSaltProcessStep(master, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])
+ // Install glusterfs
+ salt.runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
+ // Install keepalived
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keepalived'])
+ salt.runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
+ // Check the keepalived VIPs
+ salt.runSaltProcessStep(master, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
+ // Setup glusterfs
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(master, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(master, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
+ salt.runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
+ salt.runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
+ // Install haproxy
+ salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
+ salt.runSaltProcessStep(master, 'I@haproxy:proxy', 'service.status', ['haproxy'])
+ // Install docker
+ salt.runSaltProcessStep(master, 'I@docker:host', 'state.sls', ['docker.host'])
+ salt.runSaltProcessStep(master, 'I@docker:host', 'cmd.run', ['docker ps'])
+ // Install bird
+ salt.runSaltProcessStep(master, 'I@bird:server', 'state.sls', ['bird'])
+ // Install etcd
+ salt.runSaltProcessStep(master, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
+ salt.runSaltProcessStep(master, 'I@etcd:server', 'cmd.run', ['etcdctl cluster-health'])
+}
+
+
+def installOpenstackMcpControl(master) {
+ def salt = new com.mirantis.mk.Salt()
+ // Install Kubernetes pool and Calico
+ salt.runSaltProcessStep(master, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
+ salt.runSaltProcessStep(master, 'I@kubernetes:pool', 'cmd.run', ['calicoctl node status'])
+
+ // Setup etcd server
+ salt.runSaltProcessStep(master, 'I@kubernetes:master', 'state.sls', ['etcd.server.setup'])
+
+ // Run k8s without master.setup
+ salt.runSaltProcessStep(master, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])
+
+ // Run k8s master setup
+ salt.runSaltProcessStep(master, 'ctl01*', 'state.sls', ['kubernetes.master.setup'])
+
+ // Revert comment nameserver
+ salt.runSaltProcessStep(master, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])
+
+ // Set route
+ salt.runSaltProcessStep(master, 'I@kubernetes:pool', 'cmd.run', ['ip r a 10.254.0.0/16 dev ens4'])
+
+ // Restart kubelet
+ salt.runSaltProcessStep(master, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
+}
+
+
+def installOpenstackMcpCompute(master) {
+ def salt = new com.mirantis.mk.Salt();
+ // Install opencontrail
+ salt.runSaltProcessStep(master, 'I@opencontrail:compute', 'state.sls', ['opencontrail'])
+ // Reboot compute nodes
+ salt.runSaltProcessStep(master, 'I@opencontrail:compute', 'system.reboot')
+}
+
+
+def installStacklightControl(master) {
+ def salt = new com.mirantis.mk.Salt();
+ salt.runSaltProcessStep(master, 'I@elasticsearch:server', 'state.sls', ['elasticsearch.server'])
+ salt.runSaltProcessStep(master, 'I@influxdb:server', 'state.sls', ['influxdb'])
+ salt.runSaltProcessStep(master, 'I@kibana:server', 'state.sls', ['kibana.server'])
+ salt.runSaltProcessStep(master, 'I@grafana:server', 'state.sls', ['grafana'])
+ salt.runSaltProcessStep(master, 'I@nagios:server', 'state.sls', ['nagios'])
+ salt.runSaltProcessStep(master, 'I@elasticsearch:client', 'state.sls', ['elasticsearch.client'])
+ salt.runSaltProcessStep(master, 'I@kibana:client', 'state.sls', ['kibana.client'])
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/python.groovy b/src/com/mirantis/mk/Python.groovy
similarity index 100%
rename from src/com/mirantis/mk/python.groovy
rename to src/com/mirantis/mk/Python.groovy
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
new file mode 100644
index 0000000..8422cb3
--- /dev/null
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -0,0 +1,233 @@
+package com.mirantis.mk
+
+/**
+ * Salt functions
+ *
+*/
+
+/**
+ * Salt connection and context parameters
+ *
+ * @param url Salt API server URL
+ * @param credentialsID ID of credentials store entry
+ */
+def connection(url, credentialsId = "salt") {
+ def common = new com.mirantis.mk.Common();
+ params = [
+ "url": url,
+ "credentialsId": credentialsId,
+ "authToken": null,
+ "creds": common.getCredentials(credentialsId)
+ ]
+ params["authToken"] = saltLogin(params)
+
+ return params
+}
+
+/**
+ * Login to Salt API, return auth token
+ *
+ * @param master Salt connection object
+ */
+def saltLogin(master) {
+ data = [
+ 'username': master.creds.username,
+ 'password': master.creds.password.toString(),
+ 'eauth': 'pam'
+ ]
+ authToken = restGet(master, '/login', data)['return'][0]['token']
+ return authToken
+}
+
+/**
+ * Run action using Salt API
+ *
+ * @param master Salt connection object
+ * @param client Client type
+ * @param target Target specification, eg. for compound matches by Pillar
+ * data: ['expression': 'I@openssh:server', 'type': 'compound'])
+ * @param function Function to execute (eg. "state.sls")
+ * @param batch
+ * @param args Additional arguments to function
+ * @param kwargs Additional key-value arguments to function
+ */
+@NonCPS
+def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null) {
+ def http = new com.mirantis.mk.http()
+
+ data = [
+ 'tgt': target.expression,
+ 'fun': function,
+ 'client': client,
+ 'expr_form': target.type,
+ ]
+
+ if (batch) {
+ data['batch'] = batch
+ }
+
+ if (args) {
+ data['arg'] = args
+ }
+
+ if (kwargs) {
+ data['kwarg'] = kwargs
+ }
+
+ headers = [
+ 'X-Auth-Token': "${master.authToken}"
+ ]
+
+ return http.sendHttpPostRequest("${master.url}/", data, headers)
+}
+
+def pillarGet(master, target, pillar) {
+ def out = runSaltCommand(master, 'local', target, 'pillar.get', null, [pillar.replace('.', ':')])
+ return out
+}
+
+def enforceState(master, target, state, output = false) {
+ def run_states
+ if (state instanceof String) {
+ run_states = state
+ } else {
+ run_states = state.join(',')
+ }
+
+ def out = runSaltCommand(master, 'local', target, 'state.sls', null, [run_states])
+ try {
+ checkResult(out)
+ } finally {
+ if (output == true) {
+ printResult(out)
+ }
+ }
+ return out
+}
+
+def cmdRun(master, target, cmd) {
+ def out = runSaltCommand(master, 'local', target, 'cmd.run', null, [cmd])
+ return out
+}
+
+def syncAll(master, target) {
+ return runSaltCommand(master, 'local', target, 'saltutil.sync_all')
+}
+
+def enforceHighstate(master, target, output = false) {
+ def out = runSaltCommand(master, 'local', target, 'state.highstate')
+ try {
+ checkResult(out)
+ } finally {
+ if (output == true) {
+ printResult(out)
+ }
+ }
+ return out
+}
+
+def generateNodeKey(master, target, host, keysize = 4096) {
+ args = [host]
+ kwargs = ['keysize': keysize]
+ return runSaltCommand(master, 'wheel', target, 'key.gen_accept', args, kwargs)
+}
+
+def generateNodeMetadata(master, target, host, classes, parameters) {
+ args = [host, '_generated']
+ kwargs = ['classes': classes, 'parameters': parameters]
+ return runSaltCommand(master, 'local', target, 'reclass.node_create', args, kwargs)
+}
+
+def orchestrateSystem(master, target, orchestrate) {
+ return runSaltCommand(master, 'runner', target, 'state.orchestrate', [orchestrate])
+}
+
+def runSaltProcessStep(master, tgt, fun, arg = [], batch = null) {
+ if (batch) {
+ result = runSaltCommand(master, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg)
+ }
+ else {
+ result = runSaltCommand(master, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg)
+ }
+ echo("${result}")
+}
+
+/**
+ * Check result for errors and throw exception if any found
+ *
+ * @param result Parsed response of Salt API
+ */
+def checkResult(result) {
+ for (entry in result['return']) {
+ if (!entry) {
+ throw new Exception("Salt API returned empty response: ${result}")
+ }
+ for (node in entry) {
+ for (resource in node.value) {
+ if (resource instanceof String || resource.value.result.toString().toBoolean() != true) {
+ throw new Exception("Salt state on node ${node.key} failed: ${node.value}")
+ }
+ }
+ }
+ }
+}
+
+/**
+ * Print Salt state run results in human-friendly form
+ *
+ * @param result Parsed response of Salt API
+ * @param onlyChanges If true (default), print only changed resources
+ * parsing
+ */
+def printSaltStateResult(result, onlyChanges = true) {
+ def out = [:]
+ for (entry in result['return']) {
+ for (node in entry) {
+ out[node.key] = [:]
+ for (resource in node.value) {
+ if (resource instanceof String) {
+ out[node.key] = node.value
+ } else if (resource.value.result.toString().toBoolean() == false || resource.value.changes || onlyChanges == false) {
+ out[node.key][resource.key] = resource.value
+ }
+ }
+ }
+ }
+
+ for (node in out) {
+ if (node.value) {
+ println "Node ${node.key} changes:"
+ print new groovy.json.JsonBuilder(node.value).toPrettyString()
+ } else {
+ println "No changes for node ${node.key}"
+ }
+ }
+}
+
+/**
+ * Print Salt state run results in human-friendly form
+ *
+ * @param result Parsed response of Salt API
+ * @param onlyChanges If true (default), print only changed resources
+ * parsing
+ */
+def printSaltCommandResult(result, onlyChanges = true) {
+ def out = [:]
+ for (entry in result['return']) {
+ for (node in entry) {
+ out[node.key] = [:]
+ for (resource in node.value) {
+ out[node.key] = node.value
+ }
+ }
+ }
+
+ for (node in out) {
+ if (node.value) {
+ println "Node ${node.key} changes:"
+ print new groovy.json.JsonBuilder(node.value).toPrettyString()
+ } else {
+ println "No changes for node ${node.key}"
+ }
+ }
+}
diff --git a/src/com/mirantis/mk/ssl.groovy b/src/com/mirantis/mk/Ssl.groovy
similarity index 100%
rename from src/com/mirantis/mk/ssl.groovy
rename to src/com/mirantis/mk/Ssl.groovy
diff --git a/src/com/mirantis/mk/Test.groovy b/src/com/mirantis/mk/Test.groovy
new file mode 100644
index 0000000..c212c61
--- /dev/null
+++ b/src/com/mirantis/mk/Test.groovy
@@ -0,0 +1,18 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Tests providing functions
+ *
+ */
+
+/**
+ * Run e2e conformance tests
+ *
+ * @param k8s_api Kubernetes api address
+ * @param image Docker image with tests
+ */
+def runConformanceTests(master, k8s_api, image) {
+ def salt = new com.mirantis.mk.Salt()
+ salt = runSaltProcessStep(master, 'ctl01*', 'cmd.run', ["docker run --rm --net=host -e API_SERVER=${k8s_api} ${image} >> e2e-conformance.log"])
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/common.groovy b/src/com/mirantis/mk/common.groovy
deleted file mode 100644
index f64b9db..0000000
--- a/src/com/mirantis/mk/common.groovy
+++ /dev/null
@@ -1,177 +0,0 @@
-package com.mirantis.mk
-
-/**
- *
- * Common functions
- *
- */
-
-/**
- * Generate current timestamp
- *
- * @param format Defaults to yyyyMMddHHmmss
- */
-def getDatetime(format="yyyyMMddHHmmss") {
- def now = new Date();
- return now.format(format, TimeZone.getTimeZone('UTC'));
-}
-
-/**
- * Abort build, wait for some time and ensure we will terminate
- */
-def abortBuild() {
- currentBuild.build().doStop()
- sleep(180)
- // just to be sure we will terminate
- throw new InterruptedException()
-}
-
-/**
- * Print informational message
- *
- * @param msg
- * @param color Colorful output or not
- */
-def infoMsg(msg, color = true) {
- printMsg(msg, "cyan")
-}
-
-/**
- * Print error message
- *
- * @param msg
- * @param color Colorful output or not
- */
-def errorMsg(msg, color = true) {
- printMsg(msg, "red")
-}
-
-/**
- * Print success message
- *
- * @param msg
- * @param color Colorful output or not
- */
-def successMsg(msg, color = true) {
- printMsg(msg, "green")
-}
-
-/**
- * Print warning message
- *
- * @param msg
- * @param color Colorful output or not
- */
-def warningMsg(msg, color = true) {
- printMsg(msg, "blue")
-}
-
-/**
- * Print message
- *
- * @param msg Message to be printed
- * @param level Level of message (default INFO)
- * @param color Color to use for output or false (default)
- */
-def printMsg(msg, color = false) {
- colors = [
- 'red' : '\u001B[31m',
- 'black' : '\u001B[30m',
- 'green' : '\u001B[32m',
- 'yellow': '\u001B[33m',
- 'blue' : '\u001B[34m',
- 'purple': '\u001B[35m',
- 'cyan' : '\u001B[36m',
- 'white' : '\u001B[37m',
- 'reset' : '\u001B[0m'
- ]
- if (color != false) {
- wrap([$class: 'AnsiColorBuildWrapper']) {
- print "${colors[color]}${msg}${colors.reset}"
- }
- } else {
- print "[${level}] ${msg}"
- }
-}
-
-/**
- * Traverse directory structure and return list of files
- *
- * @param path Path to search
- * @param type Type of files to search (groovy.io.FileType.FILES)
- */
-@NonCPS
-def getFiles(path, type=groovy.io.FileType.FILES) {
- files = []
- new File(path).eachFile(type) {
- files[] = it
- }
- return files
-}
-
-/**
- * Helper method to convert map into form of list of [key,value] to avoid
- * unserializable exceptions
- *
- * @param m Map
- */
-@NonCPS
-def entries(m) {
- m.collect {k, v -> [k, v]}
-}
-
-/**
- * Opposite of build-in parallel, run map of steps in serial
- *
- * @param steps Map of String<name>: CPSClosure2<step>
- */
-def serial(steps) {
- stepsArray = entries(steps)
- for (i=0; i < stepsArray.size; i++) {
- s = stepsArray[i]
- dummySteps = ["${s[0]}": s[1]]
- parallel dummySteps
- }
-}
-
-/**
- * Get password credentials from store
- *
- * @param id Credentials name
- */
-def getPasswordCredentials(id) {
- def creds = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
- com.cloudbees.plugins.credentials.common.StandardUsernamePasswordCredentials.class,
- jenkins.model.Jenkins.instance
- )
-
- for (Iterator<String> credsIter = creds.iterator(); credsIter.hasNext();) {
- c = credsIter.next();
- if ( c.id == id ) {
- return c;
- }
- }
-
- throw new Exception("Could not find credentials for ID ${id}")
-}
-
-/**
- * Get SSH credentials from store
- *
- * @param id Credentials name
- */
-def getSshCredentials(id) {
- def creds = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
- com.cloudbees.plugins.credentials.common.StandardUsernameCredentials.class,
- jenkins.model.Jenkins.instance
- )
-
- for (Iterator<String> credsIter = creds.iterator(); credsIter.hasNext();) {
- c = credsIter.next();
- if ( c.id == id ) {
- return c;
- }
- }
-
- throw new Exception("Could not find credentials for ID ${id}")
-}
diff --git a/src/com/mirantis/mk/http.groovy b/src/com/mirantis/mk/http.groovy
deleted file mode 100644
index c0bf70b..0000000
--- a/src/com/mirantis/mk/http.groovy
+++ /dev/null
@@ -1,102 +0,0 @@
-package com.mirantis.mk
-/**
- *
- * HTTP functions
- *
- */
-
-/**
- * Make generic HTTP call and return parsed JSON
- *
- * @param url URL to make the request against
- * @param method HTTP method to use (default GET)
- * @param data JSON data to POST or PUT
- * @param headers Map of additional request headers
- */
-@NonCPS
-def sendHttpRequest(url, method = 'GET', data = null, headers = [:]) {
-
- def connection = new URL(url).openConnection()
- if (method != 'GET') {
- connection.setRequestMethod(method)
- }
-
- if (data) {
- headers['Content-Type'] = 'application/json'
- }
-
- headers['User-Agent'] = 'jenkins-groovy'
- headers['Accept'] = 'application/json'
-
- for (header in headers) {
- connection.setRequestProperty(header.key, header.value)
- }
-
- if (data) {
- connection.setDoOutput(true)
- if (data instanceof String) {
- dataStr = data
- } else {
- dataStr = new groovy.json.JsonBuilder(data).toString()
- }
- def output = new OutputStreamWriter(connection.outputStream)
- //infoMsg("[HTTP] Request URL: ${url}, method: ${method}, headers: ${headers}, content: ${dataStr}")
- output.write(dataStr)
- output.close()
- }
-
- if ( connection.responseCode == 200 ) {
- response = connection.inputStream.text
- try {
- response_content = new groovy.json.JsonSlurperClassic().parseText(response)
- } catch (groovy.json.JsonException e) {
- response_content = response
- }
- //successMsg("[HTTP] Response: code ${connection.responseCode}")
- return response_content
- } else {
- //errorMsg("[HTTP] Response: code ${connection.responseCode}")
- throw new Exception(connection.responseCode + ": " + connection.inputStream.text)
- }
-
-}
-
-/**
- * Make HTTP GET request
- *
- * @param url URL which will requested
- * @param data JSON data to PUT
- */
-def sendHttpGetRequest(url, data = null, headers = [:]) {
- return sendHttpRequest(url, 'GET', data, headers)
-}
-
-/**
- * Make HTTP POST request
- *
- * @param url URL which will requested
- * @param data JSON data to PUT
- */
-def sendHttpPostRequest(url, data = null, headers = [:]) {
- return sendHttpRequest(url, 'POST', data, headers)
-}
-
-/**
- * Make HTTP PUT request
- *
- * @param url URL which will requested
- * @param data JSON data to PUT
- */
-def sendHttpPutRequest(url, data = null, headers = [:]) {
- return sendHttpRequest(url, 'PUT', data, headers)
-}
-
-/**
- * Make HTTP DELETE request
- *
- * @param url URL which will requested
- * @param data JSON data to PUT
- */
-def sendHttpDeleteRequest(url, data = null, headers = [:]) {
- return sendHttpRequest(url, 'DELETE', data, headers)
-}
diff --git a/src/com/mirantis/mk/salt.groovy b/src/com/mirantis/mk/salt.groovy
deleted file mode 100644
index 4b7c3b1..0000000
--- a/src/com/mirantis/mk/salt.groovy
+++ /dev/null
@@ -1,481 +0,0 @@
-package com.mirantis.mk
-
-/**
- *
- * SaltStack functions
- *
- */
-
-/**
- * Login to Salt API and return auth token
- *
- * @param url Salt API server URL
- * @param params Salt connection params
- */
-def getSaltToken(url, params) {
- def http = new com.mirantis.mk.http()
- data = [
- 'username': params.creds.username,
- 'password': params.creds.password.toString(),
- 'eauth': 'pam'
- ]
- authToken = http.sendHttpGetRequest("${url}/login", data, ['Accept': '*/*'])['return'][0]['token']
- return authToken
-}
-
-/**
- * Salt connection and context parameters
- *
- * @param url Salt API server URL
- * @param credentialsID ID of credentials store entry
- */
-def createSaltConnection(url, credentialsId) {
- def common = new com.mirantis.mk.common()
- params = [
- "url": url,
- "credentialsId": credentialsId,
- "authToken": null,
- "creds": common.getPasswordCredentials(credentialsId)
- ]
- params["authToken"] = getSaltToken(url, params)
-
- return params
-}
-
-/**
- * Run action using Salt API
- *
- * @param master Salt connection object
- * @param client Client type
- * @param target Target specification, eg. for compound matches by Pillar
- * data: ['expression': 'I@openssh:server', 'type': 'compound'])
- * @param function Function to execute (eg. "state.sls")
- * @param args Additional arguments to function
- * @param kwargs Additional key-value arguments to function
- */
-@NonCPS
-def runSaltCommand(master, client, target, function, batch = null, args = null, kwargs = null) {
- def http = new com.mirantis.mk.http()
-
- data = [
- 'tgt': target.expression,
- 'fun': function,
- 'client': client,
- 'expr_form': target.type,
- ]
-
- if (batch) {
- data['batch'] = batch
- }
-
- if (args) {
- data['arg'] = args
- }
-
- if (kwargs) {
- data['kwarg'] = kwargs
- }
-
- headers = [
- 'X-Auth-Token': "${master.authToken}"
- ]
-
- return http.sendHttpPostRequest("${master.url}/", data, headers)
-}
-
-def getSaltPillar(master, target, pillar) {
- def out = runSaltCommand(master, 'local', target, 'pillar.get', [pillar.replace('.', ':')])
- return out
-}
-
-def enforceSaltState(master, target, state, output = false) {
- def run_states
- if (state instanceof String) {
- run_states = state
- } else {
- run_states = state.join(',')
- }
-
- def out = runSaltCommand(master, 'local', target, 'state.sls', null, [run_states])
- try {
- checkSaltResult(out)
- } finally {
- if (output == true) {
- printSaltResult(out)
- }
- }
- return out
-}
-
-def runSaltCmd(master, target, cmd) {
- return runSaltCommand(master, 'local', target, 'cmd.run', null, [cmd])
-}
-
-def syncSaltAll(master, target) {
- return runSaltCommand(master, 'local', target, 'saltutil.sync_all')
-}
-
-def enforceSaltApply(master, target, output = false) {
- def out = runSaltCommand(master, 'local', target, 'state.highstate')
- try {
- checkSaltResult(out)
- } finally {
- if (output == true) {
- printSaltResult(out)
- }
- }
- return out
-}
-
-def generateSaltNodeKey(master, target, host, keysize = 4096) {
- args = [host]
- kwargs = ['keysize': keysize]
- return runSaltCommand(master, 'wheel', target, 'key.gen_accept', null, args, kwargs)
-}
-
-def generateSaltNodeMetadata(master, target, host, classes, parameters) {
- args = [host, '_generated']
- kwargs = ['classes': classes, 'parameters': parameters]
- return runSaltCommand(master, 'local', target, 'reclass.node_create', null, args, kwargs)
-}
-
-def orchestrateSaltSystem(master, target, orchestrate) {
- return runSaltCommand(master, 'runner', target, 'state.orchestrate', null, [orchestrate])
-}
-
-/**
- * Check result for errors and throw exception if any found
- *
- * @param result Parsed response of Salt API
- */
-def checkSaltResult(result) {
- for (entry in result['return']) {
- if (!entry) {
- throw new Exception("Salt API returned empty response: ${result}")
- }
- for (node in entry) {
- for (resource in node.value) {
- if (resource instanceof String || resource.value.result.toString().toBoolean() != true) {
- throw new Exception("Salt state on node ${node.key} failed: ${node.value}")
- }
- }
- }
- }
-}
-
-/**
- * Print Salt run results in human-friendly form
- *
- * @param result Parsed response of Salt API
- * @param onlyChanges If true (default), print only changed resources
- * @param raw Simply pretty print what we have, no additional
- * parsing
- */
-def printSaltResult(result, onlyChanges = true, raw = false) {
- if (raw == true) {
- print new groovy.json.JsonBuilder(result).toPrettyString()
- } else {
- def out = [:]
- for (entry in result['return']) {
- for (node in entry) {
- out[node.key] = [:]
- for (resource in node.value) {
- if (resource instanceof String) {
- out[node.key] = node.value
- } else if (resource.value.result.toString().toBoolean() == false || resource.value.changes || onlyChanges == false) {
- out[node.key][resource.key] = resource.value
- }
- }
- }
- }
-
- for (node in out) {
- if (node.value) {
- println "Node ${node.key} changes:"
- print new groovy.json.JsonBuilder(node.value).toPrettyString()
- } else {
- println "No changes for node ${node.key}"
- }
- }
- }
-}
-
-
-def runSaltProcessStep(master, tgt, fun, arg = [], batch = null) {
- if (batch) {
- result = runSaltCommand(master, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg)
- }
- else {
- result = runSaltCommand(master, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg)
- }
- echo("${result}")
-}
-
-
-def validateFoundationInfra(master) {
- runSaltProcessStep(master, 'I@salt:master', 'cmd.run', ['salt-key'])
- runSaltProcessStep(master, 'I@salt:minion', 'test.version')
- runSaltProcessStep(master, 'I@salt:master', 'cmd.run', ['reclass-salt --top'])
- runSaltProcessStep(master, 'I@reclass:storage', 'reclass.inventory')
- runSaltProcessStep(master, 'I@salt:minion', 'state.show_top')
-}
-
-
-def installFoundationInfra(master) {
- runSaltProcessStep(master, 'I@salt:master', 'state.sls', ['salt.master,reclass'])
- runSaltProcessStep(master, 'I@linux:system', 'saltutil.refresh_pillar')
- runSaltProcessStep(master, 'I@linux:system', 'saltutil.sync_all')
- runSaltProcessStep(master, 'I@linux:system', 'state.sls', ['linux,openssh,salt.minion,ntp'])
-}
-
-
-def installOpenstackMkInfra(master) {
- // Install keepaliveds
- //runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keepalived'])
- runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
- // Check the keepalived VIPs
- runSaltProcessStep(master, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
- // Install glusterfs
- runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
- //runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.setup'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
- runSaltProcessStep(master, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
- runSaltProcessStep(master, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
- runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
- runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
- // Install rabbitmq
- runSaltProcessStep(master, 'I@rabbitmq:server', 'state.sls', ['rabbitmq'])
- // Check the rabbitmq status
- runSaltProcessStep(master, 'I@rabbitmq:server', 'cmd.run', ['rabbitmqctl cluster_status'])
- // Install galera
- runSaltProcessStep(master, 'I@galera:master', 'state.sls', ['galera'])
- runSaltProcessStep(master, 'I@galera:slave', 'state.sls', ['galera'])
- // Check galera status
- runSaltProcessStep(master, 'I@galera:master', 'mysql.status')
- runSaltProcessStep(master, 'I@galera:slave', 'mysql.status')
- // Install haproxy
- runSaltProcessStep(master, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
- runSaltProcessStep(master, 'I@haproxy:proxy', 'service.status', ['haproxy'])
- runSaltProcessStep(master, 'I@haproxy:proxy', 'service.restart', ['rsyslog'])
- // Install memcached
- runSaltProcessStep(master, 'I@memcached:server', 'state.sls', ['memcached'])
-}
-
-
-def installOpenstackMkControl(master) {
- // setup keystone service
- //runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keystone.server'])
- runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
- // populate keystone services/tenants/roles/users
- runSaltProcessStep(master, 'I@keystone:client', 'state.sls', ['keystone.client'])
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; keystone service-list'])
- // Install glance and ensure glusterfs clusters
- //runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glance.server'])
- runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glance.server'])
- runSaltProcessStep(master, 'I@glance:server', 'state.sls', ['glusterfs.client'])
- // Update fernet tokens before doing request on keystone server
- runSaltProcessStep(master, 'I@keystone:server', 'state.sls', ['keystone.server'])
- // Check glance service
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; glance image-list'])
- // Install and check nova service
- //runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['nova'])
- runSaltProcessStep(master, 'I@nova:controller', 'state.sls', ['nova'])
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova service-list'])
- // Install and check cinder service
- //runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['cinder'])
- runSaltProcessStep(master, 'I@cinder:controller', 'state.sls', ['cinder'])
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; cinder list'])
- // Install neutron service
- //runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['neutron'])
- runSaltProcessStep(master, 'I@neutron:server', 'state.sls', ['neutron'])
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron agent-list'])
- // Install heat service
- //runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'], 1)
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['heat'])
- runSaltProcessStep(master, 'I@heat:server', 'state.sls', ['heat'])
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; heat resource-type-list'])
- // Install horizon dashboard
- runSaltProcessStep(master, 'I@horizon:server', 'state.sls', ['horizon'])
- runSaltProcessStep(master, 'I@nginx:server', 'state.sls', ['nginx'])
-}
-
-
-def installOpenstackMkNetwork(master) {
- // Install opencontrail database services
- //runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'], 1)
- runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail.database'])
- runSaltProcessStep(master, 'I@opencontrail:database', 'state.sls', ['opencontrail.database'])
- // Install opencontrail control services
- //runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'], 1)
- runSaltProcessStep(master, 'ntw01*', 'state.sls', ['opencontrail'])
- runSaltProcessStep(master, 'I@opencontrail:control', 'state.sls', ['opencontrail'])
- // Provision opencontrail control services
- runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl01 --host_ip 172.16.10.101 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
- runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl02 --host_ip 172.16.10.102 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
- runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_control.py --api_server_ip 172.16.10.254 --api_server_port 8082 --host_name ctl03 --host_ip 172.16.10.103 --router_asn 64512 --admin_password workshop --admin_user admin --admin_tenant_name admin --oper add'])
- // Test opencontrail
- runSaltProcessStep(master, 'I@opencontrail:control', 'cmd.run', ['contrail-status'])
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; neutron net-list'])
- runSaltProcessStep(master, 'I@keystone:server', 'cmd.run', ['. /root/keystonerc; nova net-list'])
-}
-
-
-def installOpenstackMkCompute(master) {
- // Configure compute nodes
- runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
- runSaltProcessStep(master, 'I@nova:compute', 'state.apply')
- // Provision opencontrail virtual routers
- runSaltProcessStep(master, 'I@opencontrail:control:id:1', 'cmd.run', ['/usr/share/contrail-utils/provision_vrouter.py --host_name cmp01 --host_ip 172.16.10.105 --api_server_ip 172.16.10.254 --oper add --admin_user admin --admin_password workshop --admin_tenant_name admin'])
- runSaltProcessStep(master, 'I@nova:compute', 'system.reboot')
-}
-
-
-def installOpenstackMcpInfra(master) {
- // Comment nameserver
- runSaltProcessStep(master, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])
- // Install glusterfs
- runSaltProcessStep(master, 'I@glusterfs:server', 'state.sls', ['glusterfs.server.service'])
- // Install keepalived
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['keepalived'])
- runSaltProcessStep(master, 'I@keepalived:cluster', 'state.sls', ['keepalived'])
- // Check the keepalived VIPs
- runSaltProcessStep(master, 'I@keepalived:cluster', 'cmd.run', ['ip a | grep 172.16.10.2'])
- // Setup glusterfs
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['glusterfs.server.setup'])
- runSaltProcessStep(master, 'ctl02*', 'state.sls', ['glusterfs.server.setup'])
- runSaltProcessStep(master, 'ctl03*', 'state.sls', ['glusterfs.server.setup'])
- runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster peer status'])
- runSaltProcessStep(master, 'I@glusterfs:server', 'cmd.run', ['gluster volume status'])
- // Install haproxy
- runSaltProcessStep(master, 'I@haproxy:proxy', 'state.sls', ['haproxy'])
- runSaltProcessStep(master, 'I@haproxy:proxy', 'service.status', ['haproxy'])
- // Install docker
- runSaltProcessStep(master, 'I@docker:host', 'state.sls', ['docker.host'])
- runSaltProcessStep(master, 'I@docker:host', 'cmd.run', ['docker ps'])
- // Install bird
- runSaltProcessStep(master, 'I@bird:server', 'state.sls', ['bird'])
- // Install etcd
- runSaltProcessStep(master, 'I@etcd:server', 'state.sls', ['etcd.server.service'])
- runSaltProcessStep(master, 'I@etcd:server', 'cmd.run', ['etcdctl cluster-health'])
-}
-
-
-def installOpenstackMcpControl(master) {
-
- // Install Kubernetes pool and Calico
- runSaltProcessStep(master, 'I@kubernetes:pool', 'state.sls', ['kubernetes.pool'])
- runSaltProcessStep(master, 'I@kubernetes:pool', 'cmd.run', ['calicoctl node status'])
-
- // Setup etcd server
- runSaltProcessStep(master, 'I@kubernetes:master', 'state.sls', ['etcd.server.setup'])
-
- // Run k8s without master.setup
- runSaltProcessStep(master, 'I@kubernetes:master', 'state.sls', ['kubernetes', 'exclude=kubernetes.master.setup'])
-
- // Run k8s master setup
- runSaltProcessStep(master, 'ctl01*', 'state.sls', ['kubernetes.master.setup'])
-
- // Revert comment nameserver
- runSaltProcessStep(master, 'I@kubernetes:master', 'cmd.run', ["sed -i 's/nameserver 10.254.0.10/#nameserver 10.254.0.10/g' /etc/resolv.conf"])
-
- // Set route
- runSaltProcessStep(master, 'I@kubernetes:pool', 'cmd.run', ['ip r a 10.254.0.0/16 dev ens4'])
-
- // Restart kubelet
- runSaltProcessStep(master, 'I@kubernetes:pool', 'service.restart', ['kubelet'])
-}
-
-
-def installOpenstackMcpCompute(master) {
- // Install opencontrail
- runSaltProcessStep(master, 'I@opencontrail:compute', 'state.sls', ['opencontrail'])
- // Reboot compute nodes
- runSaltProcessStep(master, 'I@opencontrail:compute', 'system.reboot')
-}
-
-
-def installStacklightControl(master) {
- runSaltProcessStep(master, 'I@elasticsearch:server', 'state.sls', ['elasticsearch.server'])
- runSaltProcessStep(master, 'I@influxdb:server', 'state.sls', ['influxdb'])
- runSaltProcessStep(master, 'I@kibana:server', 'state.sls', ['kibana.server'])
- runSaltProcessStep(master, 'I@grafana:server', 'state.sls', ['grafana'])
- runSaltProcessStep(master, 'I@nagios:server', 'state.sls', ['nagios'])
- runSaltProcessStep(master, 'I@elasticsearch:client', 'state.sls', ['elasticsearch.client'])
- runSaltProcessStep(master, 'I@kibana:client', 'state.sls', ['kibana.client'])
-}
-
-/**
- * Run e2e conformance tests
- *
- * @param k8s_api Kubernetes api address
- * @param image Docker image with tests
- */
-def runConformanceTests(master, k8s_api, image) {
- runSaltProcessStep(master, 'ctl01*', 'cmd.run', ["docker run --rm --net=host -e API_SERVER=${k8s_api} ${image} >> e2e-conformance.log"])
-}
-
-/**
- * Print Salt state run results in human-friendly form
- *
- * @param result Parsed response of Salt API
- * @param onlyChanges If true (default), print only changed resources
- * parsing
- */
-def printSaltStateResult(result, onlyChanges = true) {
- def out = [:]
- for (entry in result['return']) {
- for (node in entry) {
- out[node.key] = [:]
- for (resource in node.value) {
- if (resource instanceof String) {
- out[node.key] = node.value
- } else if (resource.value.result.toString().toBoolean() == false || resource.value.changes || onlyChanges == false) {
- out[node.key][resource.key] = resource.value
- }
- }
- }
- }
-
- for (node in out) {
- if (node.value) {
- println "Node ${node.key} changes:"
- print new groovy.json.JsonBuilder(node.value).toPrettyString()
- } else {
- println "No changes for node ${node.key}"
- }
- }
-}
-
-/**
- * Print Salt state run results in human-friendly form
- *
- * @param result Parsed response of Salt API
- * @param onlyChanges If true (default), print only changed resources
- * parsing
- */
-def printSaltCommandResult(result, onlyChanges = true) {
- def out = [:]
- for (entry in result['return']) {
- for (node in entry) {
- out[node.key] = [:]
- for (resource in node.value) {
- out[node.key] = node.value
- }
- }
- }
-
- for (node in out) {
- if (node.value) {
- println "Node ${node.key} changes:"
- print new groovy.json.JsonBuilder(node.value).toPrettyString()
- } else {
- println "No changes for node ${node.key}"
- }
- }
-}