Merge "Fix units of disk utilization check for galera recovery pipe"
diff --git a/src/com/mirantis/mk/Artifactory.groovy b/src/com/mirantis/mk/Artifactory.groovy
index d1eb218..73ed80f 100644
--- a/src/com/mirantis/mk/Artifactory.groovy
+++ b/src/com/mirantis/mk/Artifactory.groovy
@@ -394,3 +394,66 @@
)
}
}
+
+/**
+ * Get Helm repo for Artifactory
+ *
+ * @param art Artifactory connection object
+ * @param repoName Chart repository name
+ */
+def getArtifactoryProjectByName(art, repoName){
+ return restGet(art, "/repositories/${repoName}")
+}
+
+/**
+ * Get repo by packageType for Artifactory
+ *
+ * @param art Artifactory connection object
+ * @param packageType Repository package type
+ */
+def getArtifactoryProjectByPackageType(art, repoName){
+ return restGet(art, "/repositories?${packageType}")
+}
+
+/**
+ * Create Helm repo for Artifactory
+ *
+ * @param art Artifactory connection object
+ * @param repoName Chart repository name
+ * @param data Transmitted data
+ */
+def createArtifactoryChartRepo(art, repoName){
+ return restPut(art, "/repositories/${repoName}", '{"rclass": "local","handleSnapshots": false,"packageType": "helm"}')
+}
+
+/**
+ * Delete Helm repo for Artifactory
+ *
+ * @param art Artifactory connection object
+ * @param repoName Chart repository name
+ */
+def deleteArtifactoryChartRepo(art, repoName){
+ return restDelete(art, "/repositories/${repoName}")
+}
+
+/**
+ * Create Helm repo for Artifactory
+ *
+ * @param art Artifactory connection object
+ * @param repoName Repository Chart name
+ * @param chartName Chart name
+ */
+def publishArtifactoryHelmChart(art, repoName, chartName){
+ return restPut(art, "/repositories/${repoName}", "${chartName}")
+}
+
+/**
+ * Create Helm repo for Artifactory
+ *
+ * @param art Artifactory connection object
+ * @param repoName Repository Chart name
+ * @param chartName Chart name
+ */
+def deleteArtifactoryHelmChart(art, repoName, chartName){
+ return restDelete(art, "/repositories/${repoName}", "${chartName}")
+}
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index b5760da..9a350c2 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -1006,3 +1006,16 @@
Date parseDate(String date, String format) {
return Date.parse(format, date)
}
+
+/**
+ * Generate Random Hash string
+ * @param n Hash length
+ * @param pool Pool to use for hash generation
+*/
+def generateRandomHashString(int n, ArrayList pool = []) {
+ if (!pool) {
+ pool = ['a'..'z','A'..'Z',0..9,'_','+','='].flatten()
+ }
+ Random rand = new Random(System.currentTimeMillis())
+ return (1..n).collect { pool[rand.nextInt(pool.size())] }.join()
+}
diff --git a/src/com/mirantis/mk/GoogleCloudStorage.groovy b/src/com/mirantis/mk/GoogleCloudStorage.groovy
new file mode 100644
index 0000000..5bf5ad7
--- /dev/null
+++ b/src/com/mirantis/mk/GoogleCloudStorage.groovy
@@ -0,0 +1,126 @@
+package com.mirantis.mk
+
+/**
+ Work with Google Cloud Storage
+**/
+
+/** Exists or not gcloud binary file
+ *
+ * @param gcloudBinDir Path to check
+*/
+def checkGcloudBinary(String gcloudBinDir) {
+ def status = sh(script: "${gcloudBinDir}/google-cloud-sdk/bin/gcloud info > /dev/null", returnStatus: true)
+ return "${status}" == "0"
+}
+
+/** Download gcloud archive with binarties
+ *
+ * @param gcloudBinDir Path to save binaries
+ * @param url Specific URL to use to download
+*/
+def downloadGcloudUtil(String gcloudBinDir, String url="") {
+ if (!url) {
+ url="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-253.0.0-linux-x86_64.tar.gz"
+ }
+ dir(gcloudBinDir) {
+ def archiveName='google-cloud-sdk'
+ sh """
+ wget -O ${archiveName}.tar.gz ${url}
+ tar -xf ${archiveName}.tar.gz
+ """
+ }
+}
+
+/** Auth gcloud util with provided creds
+ *
+ * @param gcloudBinDir Path to save binaries
+ * @param creds Creds name to use, saved as secret file
+ * @param projectName Project name to use
+*/
+def authGcloud(String gcloudBinDir, String creds, String projectName) {
+ ws {
+ withCredentials([
+ file(credentialsId: creds,
+ variable: 'key_file')
+ ]) {
+ sh "${gcloudBinDir}/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file $key_file --project ${projectName}"
+ }
+ }
+}
+
+/** Revoke gcloud auth account
+ *
+ * @param gcloudBinDir Path to save binaries
+*/
+def revokeGcloud(String gcloudBinDir) {
+ sh "${gcloudBinDir}/google-cloud-sdk/bin/gcloud auth revoke"
+}
+
+/** Copy file to Google Storage Bucket
+ *
+ * @param gcloudBinDir Path to save binaries
+ * @param src Source file
+ * @param dst Destination path in storage
+ * @param entireTree Copy entire directory tree
+*/
+def cpFile(String gcloudBinDir, String src, String dst, Boolean entireTree=false) {
+ def fileURL = ''
+ if (entireTree) {
+ sh "${gcloudBinDir}/google-cloud-sdk/bin/gsutil cp -r ${src} ${dst}"
+ return dst
+ } else {
+ def fileBaseName = sh(script:"basename ${src}", returnStdout: true).trim()
+ sh "${gcloudBinDir}/google-cloud-sdk/bin/gsutil cp ${src} ${dst}/${fileBaseName}"
+ return "${dst}/${fileBaseName}"
+ }
+}
+
+/** Set ACL on files in bucket
+ *
+ * @param gcloudBinDir Path to save binaries
+ * @param path Path to file in bucket
+ * @param acls ACLs to be set for file
+*/
+def setAcl(String gcloudBinDir, String path, ArrayList acls) {
+ for(String acl in acls) {
+ sh "${gcloudBinDir}/google-cloud-sdk/bin/gsutil acl ch -u ${acl} ${path}"
+ }
+}
+
+/** Upload files to Google Cloud Storage Bucket
+ *
+ * @param config LinkedHashMap with next parameters:
+ * @param gcloudBinDir Path to save binaries
+ * @param creds Creds name to use, saved as secret file
+ * @param projectName Project name to use
+ * @param sources List of file to upload
+ * @param dest Destination path in Google Storage, in format: gs://<path>
+ * @param acls ACLs for uploaded files
+ * @param entireTree Copy entire directory to bucket
+ *
+ * Returns URLs list of uploaded files
+*/
+def uploadArtifactToGoogleStorageBucket(Map config) {
+ def gcloudDir = config.get('gcloudDir', '/tmp/gcloud')
+ def creds = config.get('creds')
+ def project = config.get('project')
+ def acls = config.get('acls', ['AllUsers:R'])
+ def sources = config.get('sources')
+ def dest = config.get('dest')
+ def entireTree = config.get('entireTree', false)
+ def fileURLs = []
+ if (!checkGcloudBinary(gcloudDir)) {
+ downloadGcloudUtil(gcloudDir)
+ }
+ try {
+ authGcloud(gcloudDir, creds, project)
+ for(String src in sources) {
+ def fileURL = cpFile(gcloudDir, src, dest, entireTree)
+ setAcl(gcloudDir, fileURL, acls)
+ fileURLs << fileURL
+ }
+ } finally {
+ revokeGcloud(gcloudDir)
+ }
+ return fileURLs
+}
\ No newline at end of file
diff --git a/src/com/mirantis/mk/ReleaseWorkflow.groovy b/src/com/mirantis/mk/ReleaseWorkflow.groovy
index a4d1add..b5046c1 100644
--- a/src/com/mirantis/mk/ReleaseWorkflow.groovy
+++ b/src/com/mirantis/mk/ReleaseWorkflow.groovy
@@ -12,12 +12,13 @@
* @param params string map with credentialsID, metadataRepoUrl, metadataGerritBranch and crTopic
*/
-def updateReleaseMetadata(key, value, String[] params) {
+def updateReleaseMetadata(String key, String value, Map params) {
credentialsID = params['credentialsID'] ?: "mcp-ci-gerrit"
- metadataRepoUrl = params['metadataRepoUrl'] ?: "ssh://mcp-ci-gerrit@gerrit.mcp.mirantis.net:29418/mcp/release-matadata"
+ metadataRepoUrl = params['metadataRepoUrl'] ?: "ssh://mcp-ci-gerrit@gerrit.mcp.mirantis.net:29418/mcp/release-metadata"
metadataGerritBranch = params['metadataGerritBranch'] ?: "master"
comment = params['comment'] ?: ""
crTopic = params['crTopic'] ?: ""
+ def common = new com.mirantis.mk.Common()
def python = new com.mirantis.mk.Python()
def gerrit = new com.mirantis.mk.Gerrit()
def git = new com.mirantis.mk.Git()
@@ -69,7 +70,7 @@
ChangeId = ''
git.createGitBranch(repoDir, crTopic)
}
- cmdText = "python ${repoDir}/utils/app.py --path ${metadataDir} update --key ${key} --value ${value}"
+ cmdText = "python '${repoDir}/utils/app.py' --path '${metadataDir}' update --key '${key}' --value '${value}'"
python.runVirtualenvCommand(venvDir, cmdText)
commitMessage =
"""${comment}
@@ -85,4 +86,4 @@
//post change
gerrit.postGerritReview(credentialsID, venvDir, repoDir, changeAuthorName, changeAuthorEmail, gitRemote, crTopic, metadataGerritBranch)
}
-}
\ No newline at end of file
+}
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 21bb4c9..176d02c 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -458,32 +458,45 @@
* You can call this function when salt-master already contains salt keys of the target_nodes
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
* @param target Should always be salt-master
- * @param target_nodes unique identification of a minion or group of salt minions
+ * @param targetNodes unique identification of a minion or group of salt minions
* @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
- * @param wait timeout for the salt command if minions do not return (default 10)
+ * @param cmdTimeout timeout for the salt command if minions do not return (default 10)
* @param maxRetries finite number of iterations to check status of a command (default 200)
* @return output of salt command
*/
-def minionsReachable(saltId, target, target_nodes, batch=null, wait = 10, maxRetries = 200) {
+
+def minionsReachable(saltId, target, targetNodes, batch=null, cmdTimeout = 10, maxRetries = 200) {
def common = new com.mirantis.mk.Common()
- def cmd = "salt -t${wait} -C '${target_nodes}' test.ping"
- common.infoMsg("Checking if all ${target_nodes} minions are reachable")
- def count = 0
- while(count < maxRetries) {
+ def cmd = "salt -t${cmdTimeout} -C '${targetNodes}' test.ping"
+ common.infoMsg("Checking if all ${targetNodes} minions are reachable")
+ def retriesCount = 0
+ while(retriesCount < maxRetries) {
Calendar timeout = Calendar.getInstance();
- timeout.add(Calendar.SECOND, wait);
- def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, wait)
+ timeout.add(Calendar.SECOND, cmdTimeout);
+ def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, cmdTimeout)
Calendar current = Calendar.getInstance();
if (current.getTime().before(timeout.getTime())) {
- printSaltCommandResult(out)
- return out
+ common.infoMsg("Successful response received from all targeted nodes.")
+ printSaltCommandResult(out)
+ return out
}
- common.infoMsg("Not all of the targeted '${target_nodes}' minions returned yet. Waiting ...")
- count++
+ def outYaml = readYaml text: getReturnValues(out)
+ def successfulNodes = []
+ def failedNodes = []
+ for (node in outYaml.keySet()) {
+ if (outYaml[node] == true || outYaml[node].toString().toLowerCase() == 'true') {
+ successfulNodes.add(node)
+ } else {
+ failedNodes.add(node)
+ }
+ }
+ common.infoMsg("Not all of the targeted minions returned yet. Successful response from ${successfulNodes}. Still waiting for ${failedNodes}.")
+ retriesCount++
sleep(time: 500, unit: 'MILLISECONDS')
}
}
+
/**
* You can call this function when need to check that all minions are available, free and ready for command execution
* @param config LinkedHashMap config parameter, which contains next: