Merge "Add python.runCmd() to get more details from shell commands"
diff --git a/src/com/mirantis/mk/GoogleCloudStorage.groovy b/src/com/mirantis/mk/GoogleCloudStorage.groovy
index 5bf5ad7..7206715 100644
--- a/src/com/mirantis/mk/GoogleCloudStorage.groovy
+++ b/src/com/mirantis/mk/GoogleCloudStorage.groovy
@@ -97,30 +97,46 @@
* @param dest Destination path in Google Storage, in format: gs://<path>
* @param acls ACLs for uploaded files
* @param entireTree Copy entire directory to bucket
+ * @param useDifferentGcloudSDKDir Allow to use different SDK config dirs/accounts in parallel
+ * @param gcloudSDKDir Actual path to SDK config dir, defaults to gcloud default: /home/<user>/.gcloud
+ * or if HOME is unset to /tmp/.gcloud
+ * @param revokeAccount Revoke account after actions
*
* Returns URLs list of uploaded files
*/
def uploadArtifactToGoogleStorageBucket(Map config) {
def gcloudDir = config.get('gcloudDir', '/tmp/gcloud')
+ def gcloudSDKDir = config.get('gcloudSDKDir', "${env.HOME ?: '/tmp'}/.gcloud")
def creds = config.get('creds')
def project = config.get('project')
def acls = config.get('acls', ['AllUsers:R'])
def sources = config.get('sources')
def dest = config.get('dest')
def entireTree = config.get('entireTree', false)
+ def useDifferentGcloudSDKDir = config.get('useDifferentGcloudSDKDir', true)
+ def revokeAccount = config.get('revokeAccount', true)
def fileURLs = []
if (!checkGcloudBinary(gcloudDir)) {
downloadGcloudUtil(gcloudDir)
}
- try {
- authGcloud(gcloudDir, creds, project)
- for(String src in sources) {
- def fileURL = cpFile(gcloudDir, src, dest, entireTree)
- setAcl(gcloudDir, fileURL, acls)
- fileURLs << fileURL
+ if (useDifferentGcloudSDKDir) {
+ gcloudSDKDir = "${gcloudSDKDir}_" + UUID.randomUUID().toString()
+ sh "mkdir -p ${gcloudSDKDir}"
+ }
+ withEnv(["CLOUDSDK_CONFIG=${gcloudSDKDir}"]) {
+ try {
+ authGcloud(gcloudDir, creds, project)
+ for(String src in sources) {
+ def fileURL = cpFile(gcloudDir, src, dest, entireTree)
+ setAcl(gcloudDir, fileURL, acls)
+ fileURLs << fileURL
+ }
+ } finally {
+ if (revokeAccount) {
+ revokeGcloud(gcloudDir)
+ sh "rm -rf ${gcloudSDKDir}"
+ }
}
- } finally {
- revokeGcloud(gcloudDir)
}
return fileURLs
}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/ReleaseWorkflow.groovy b/src/com/mirantis/mk/ReleaseWorkflow.groovy
index 7da212d..7fbf065 100644
--- a/src/com/mirantis/mk/ReleaseWorkflow.groovy
+++ b/src/com/mirantis/mk/ReleaseWorkflow.groovy
@@ -11,12 +11,13 @@
*/
def checkoutReleaseMetadataRepo(Map params = [:]) {
def git = new com.mirantis.mk.Git()
- Boolean cloneRepo = params.get('cloneRepo', true)
+
String gitCredentialsId = params.get('metadataCredentialsId', 'mcp-ci-gerrit')
String gitUrl = params.get('metadataGitRepoUrl', "ssh://${gitCredentialsId}@gerrit.mcp.mirantis.net:29418/mcp/release-metadata")
String gitBranch = params.get('metadataGitRepoBranch', 'master')
String gitRef = params.get('metadataGitRepoRef', '')
String repoDir = params.get('repoDir', 'release-metadata')
+ Boolean cloneRepo = params.get('cloneRepo', true)
if (cloneRepo) {
stage('Cleanup repo dir') {
dir(repoDir) {
@@ -37,24 +38,17 @@
* @param key metadata key
* @param params map with expected parameters:
* - toxDockerImage
- * - metadataCredentialsId
- * - metadataGitRepoUrl
- * - metadataGitRepoBranch
+ * - outputFormat
* - repoDir
*/
def getReleaseMetadataValue(String key, Map params = [:]) {
String result
// Get params
String toxDockerImage = params.get('toxDockerImage', 'docker-prod-virtual.docker.mirantis.net/mirantis/external/tox')
- String gitCredentialsId = params.get('metadataCredentialsId', 'mcp-ci-gerrit')
- String gitUrl = params.get('metadataGitRepoUrl', "ssh://${gitCredentialsId}@gerrit.mcp.mirantis.net:29418/mcp/release-metadata")
- String gitBranch = params.get('metadataGitRepoBranch', 'master')
- String repoDir = params.get('repoDir', 'release-metadata')
String outputFormat = params.get('outputFormat', 'json')
- Boolean cloneRepo = params.get('cloneRepo', true)
+ String repoDir = params.get('repoDir', 'release-metadata')
// Libs
- def git = new com.mirantis.mk.Git()
def common = new com.mirantis.mk.Common()
String opts = ''
@@ -79,42 +73,47 @@
*
* @param key metadata key
* @param value metadata value
- * @param params string map with credentialsID, metadataRepoUrl, metadataGerritBranch and crTopic
+ * @param params map with expected parameters:
+ * - metadataCredentialsId
+ * - metadataGitRepoUrl
+ * - metadataGitRepoBranch
+ * - repoDir
+ * - comment
+ * - crTopic
+ * - crAuthorName
+ * - crAuthorEmail
*/
def updateReleaseMetadata(String key, String value, Map params) {
- credentialsID = params['credentialsID'] ?: "mcp-ci-gerrit"
- metadataRepoUrl = params['metadataRepoUrl'] ?: "ssh://mcp-ci-gerrit@gerrit.mcp.mirantis.net:29418/mcp/release-metadata"
- metadataGerritBranch = params['metadataGerritBranch'] ?: "master"
- comment = params['comment'] ?: ""
- crTopic = params['crTopic'] ?: ""
- Boolean cloneRepo = params.get('cloneRepo', true)
+ String gitCredentialsId = params.get('metadataCredentialsId', 'mcp-ci-gerrit')
+ String metadataRepoUrl = params.get('metadataGitRepoUrl', "ssh://${gitCredentialsId}@gerrit.mcp.mirantis.net:29418/mcp/release-metadata")
+ String metadataGerritBranch = params.get('metadataGitRepoBranch', 'master')
+ String repoDir = params.get('repoDir', 'release-metadata')
+ String comment = params.get('comment', '')
+ String crTopic = params.get('crTopic', '')
+ String changeAuthorName = params.get('crAuthorName', 'MCP-CI')
+ String changeAuthorEmail = params.get('crAuthorEmail', 'mcp-ci-jenkins@ci.mcp.mirantis.net')
+
def common = new com.mirantis.mk.Common()
def python = new com.mirantis.mk.Python()
def gerrit = new com.mirantis.mk.Gerrit()
- def git = new com.mirantis.mk.Git()
- def changeAuthorName = "MCP-CI"
- def changeAuthorEmail = "mcp-ci-jenkins@ci.mcp.mirantis.net"
- def cred = common.getCredentials(credentialsID, 'key')
+ def git = new com.mirantis.mk.Git()
+
+ def cred = common.getCredentials(gitCredentialsId, 'key')
String gerritUser = cred.username
- def gerritHost = metadataRepoUrl.tokenize('@')[-1].tokenize(':')[0]
- def metadataProject = metadataRepoUrl.tokenize('/')[-2..-1].join('/')
- def gerritPort = metadataRepoUrl.tokenize(':')[-1].tokenize('/')[0]
- def workspace = common.getWorkspace()
- def venvDir = "${workspace}/gitreview-venv"
- def repoDir = params.get('repoDir', "${venvDir}/repo")
- def metadataDir = "${repoDir}/metadata"
- def ChangeId
- def commitMessage
- def gitRemote
+ String gerritHost = metadataRepoUrl.tokenize('@')[-1].tokenize(':')[0]
+ String metadataProject = metadataRepoUrl.tokenize('/')[-2..-1].join('/')
+ String gerritPort = metadataRepoUrl.tokenize(':')[-1].tokenize('/')[0]
+ String workspace = common.getWorkspace()
+ String venvDir = "${workspace}/gitreview-venv"
+ String metadataDir = "${repoDir}/metadata"
+ String ChangeId
+ String commitMessage
+ String gitRemote
stage("Installing virtualenv") {
python.setupVirtualenv(venvDir, 'python3', ['git-review', 'PyYaml'])
}
- checkoutReleaseMetadataRepo(['metadataCredentialsId': credentialsID,
- 'metadataGitRepoBranch': metadataGerritBranch,
- 'metadataGitRepoUrl': metadataRepoUrl,
- 'repoDir': repoDir,
- 'cloneRepo': cloneRepo])
+ checkoutReleaseMetadataRepo(params)
dir(repoDir) {
gitRemote = sh(
script:
@@ -126,7 +125,7 @@
stage('Creating CR') {
def gerritAuth = ['PORT': gerritPort, 'USER': gerritUser, 'HOST': gerritHost]
def changeParams = ['owner': gerritUser, 'status': 'open', 'project': metadataProject, 'branch': metadataGerritBranch, 'topic': crTopic]
- def gerritChange = gerrit.findGerritChange(credentialsID, gerritAuth, changeParams)
+ def gerritChange = gerrit.findGerritChange(gitCredentialsId, gerritAuth, changeParams)
git.changeGitBranch(repoDir, metadataGerritBranch)
if (gerritChange) {
def jsonChange = readJSON text: gerritChange
@@ -134,7 +133,7 @@
ChangeId = 'Change-Id: '
ChangeId += jsonChange['id']
//get existent change from gerrit
- gerrit.getGerritChangeByNum(credentialsID, venvDir, repoDir, gitRemote, changeNum)
+ gerrit.getGerritChangeByNum(gitCredentialsId, venvDir, repoDir, gitRemote, changeNum)
} else {
ChangeId = ''
git.createGitBranch(repoDir, crTopic)
@@ -153,6 +152,6 @@
git.commitGitChanges(repoDir, commitMessage, changeAuthorEmail, changeAuthorName, false)
}
//post change
- gerrit.postGerritReview(credentialsID, venvDir, repoDir, changeAuthorName, changeAuthorEmail, gitRemote, crTopic, metadataGerritBranch)
+ gerrit.postGerritReview(gitCredentialsId, venvDir, repoDir, changeAuthorName, changeAuthorEmail, gitRemote, crTopic, metadataGerritBranch)
}
}
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 7347205..d972252 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -534,15 +534,16 @@
* Restart and wait for salt-minions on target nodes.
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
* @param target unique identification of a minion or group of salt minions
- * @param wait timeout for the salt command if minions do not return (default 5)
- * @param maxRetries finite number of iterations to check status of a command (default 10)
+ * @param wait timeout for the salt command if minions do not return (default 10)
+ * @param maxRetries finite number of iterations to check status of a command (default 15)
+ * @param async Run salt minion restart and do not wait for response
* @return output of salt command
*/
-def restartSaltMinion(saltId, target, wait = 5, maxRetries = 10) {
+def restartSaltMinion(saltId, target, wait = 10, maxRetries = 15, async = true) {
def common = new com.mirantis.mk.Common()
common.infoMsg("Restarting salt-minion on ${target} and waiting for they are reachable.")
- runSaltProcessStep(saltId, target, 'cmd.shell', ['salt-call service.restart salt-minion'], null, true, 60)
- checkTargetMinionsReady(['saltId': saltId, 'target_reachable': target, timeout: wait, retries: maxRetries])
+ runSaltProcessStep(saltId, target, 'cmd.shell', ['salt-call service.restart salt-minion'], null, true, 60, null, async)
+ checkTargetMinionsReady(['saltId': saltId, 'target': target, timeout: wait, retries: maxRetries])
common.infoMsg("All ${target} minions are alive...")
}
@@ -919,12 +920,13 @@
* @param tgt Salt process step target
* @param fun Salt process step function
* @param arg process step arguments (optional, default [])
- * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
+ * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch). Can't be used with async
* @param output print output (optional, default true)
* @param timeout Additional argument salt api timeout
+ * @param async Run the salt command but don't wait for a reply. Can't be used with batch
* @return output of salt command
*/
-def runSaltProcessStep(saltId, tgt, fun, arg = [], batch = null, output = true, timeout = -1, kwargs = null) {
+def runSaltProcessStep(saltId, tgt, fun, arg = [], batch = null, output = true, timeout = -1, kwargs = null, async = false) {
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def out
@@ -933,6 +935,8 @@
if (batch == true) {
out = runSaltCommand(saltId, 'local_batch', ['expression': tgt, 'type': 'compound'], fun, String.valueOf(batch), arg, kwargs, timeout)
+ } else if (async == true) {
+ out = runSaltCommand(saltId, 'local_async', ['expression': tgt, 'type': 'compound'], fun, batch, arg, kwargs, timeout)
} else {
out = runSaltCommand(saltId, 'local', ['expression': tgt, 'type': 'compound'], fun, batch, arg, kwargs, timeout)
}