Merge "Do not generate octavia certs on gateway node"
diff --git a/src/com/mirantis/mcp/Common.groovy b/src/com/mirantis/mcp/Common.groovy
index b501b04..7a29432 100644
--- a/src/com/mirantis/mcp/Common.groovy
+++ b/src/com/mirantis/mcp/Common.groovy
@@ -1,5 +1,8 @@
package com.mirantis.mcp
+import java.util.zip.GZIPInputStream
+import java.util.zip.GZIPOutputStream
+
@Grab(group='org.yaml', module='snakeyaml', version='1.17')
import org.yaml.snakeyaml.Yaml
@@ -200,3 +203,19 @@
} //else
}
+
+def zipBase64(String s){
+ def targetStream = new ByteArrayOutputStream()
+ def zipStream = new GZIPOutputStream(targetStream)
+ zipStream.write(s.getBytes('UTF-8'))
+ zipStream.close()
+ def zippedBytes = targetStream.toByteArray()
+ targetStream.close()
+ return zippedBytes.encodeBase64()
+}
+
+def unzipBase64(String compressed){
+ def inflaterStream = new GZIPInputStream(new ByteArrayInputStream(compressed.decodeBase64()))
+ def uncompressedStr = inflaterStream.getText('UTF-8')
+ return uncompressedStr
+}
diff --git a/src/com/mirantis/mk/Gerrit.groovy b/src/com/mirantis/mk/Gerrit.groovy
index 7087118..7c19875 100644
--- a/src/com/mirantis/mk/Gerrit.groovy
+++ b/src/com/mirantis/mk/Gerrit.groovy
@@ -280,3 +280,34 @@
ssh.ensureKnownHosts(gerritHost)
ssh.agentSh(String.format("ssh -p 29418 %s@%s gerrit review %s,%s -m \"'%s'\" --code-review 0", gerritName, gerritHost, gerritChangeNumber, gerritPatchSetNumber, message))
}
+
+/**
+ * Return map of dependent patches info for current patch set
+ * based on commit message hints: Depends-On: https://gerrit_address/_CHANGE_NUMBER_
+ * @param changeInfo Map Info about current patch set, such as:
+ * gerritName Gerrit user name (usually GERRIT_NAME property)
+ * gerritHost Gerrit host (usually GERRIT_HOST property)
+ * gerritChangeNumber Gerrit change number (usually GERRIT_CHANGE_NUMBER property)
+ * credentialsId Jenkins credentials id for gerrit
+ * @return map of dependent patches info
+ */
+LinkedHashMap getDependentPatches(LinkedHashMap changeInfo) {
+ def dependentPatches = [:]
+ def currentChange = getGerritChange(changeInfo.gerritName, changeInfo.gerritHost, changeInfo.gerritChangeNumber, changeInfo.credentialsId, true)
+ def dependentCommits = currentChange.commitMessage.tokenize('\n').findAll { it ==~ /Depends-On: \b[^ ]+\b(\/)?/ }
+ if (dependentCommits) {
+ dependentCommits.each { commit ->
+ def patchLink = commit.tokenize(' ')[1]
+ def changeNumber = patchLink.tokenize('/')[-1].trim()
+ def dependentCommit = getGerritChange(changeInfo.gerritName, changeInfo.gerritHost, changeNumber, changeInfo.credentialsId, true)
+ if (dependentCommit.status == "NEW") {
+ dependentPatches[dependentCommit.project] = [
+ 'number': dependentCommit.number,
+ 'ref': dependentCommit.currentPatchSet.ref,
+ 'branch': dependentCommit.branch,
+ ]
+ }
+ }
+ }
+ return dependentPatches
+}
diff --git a/src/com/mirantis/mk/JenkinsUtils.groovy b/src/com/mirantis/mk/JenkinsUtils.groovy
index d092153..8b35aca 100644
--- a/src/com/mirantis/mk/JenkinsUtils.groovy
+++ b/src/com/mirantis/mk/JenkinsUtils.groovy
@@ -105,3 +105,71 @@
}
return params
}
+
+/**
+ * Get list of causes actions for given build
+ *
+ * @param build Job build object (like, currentBuild.rawBuild)
+ * @return list of causes actions for given build
+ */
+@NonCPS
+def getBuildCauseActions(build) {
+ def causeAction = build.actions.find { it -> it instanceof hudson.model.CauseAction }
+ if(causeAction) {
+ return causeAction.causes
+ } else {
+ return []
+ }
+}
+
+/**
+ * Get list of builds, triggered by Gerrit with given build
+ * @param build Job build object (like, currentBuild.rawBuild)
+ * @return list of builds with names and numbers
+ */
+@NonCPS
+def getGerritBuildContext(build) {
+ def causes = getBuildCauseActions(build)
+ if (causes) {
+ def gerritTriggerCause = causes.find { cause ->
+ cause instanceof com.sonyericsson.hudson.plugins.gerrit.trigger.hudsontrigger.GerritCause
+ }
+ return gerritTriggerCause.context.getOtherBuilds()
+ } else {
+ return []
+ }
+}
+
+/**
+ * Wait for other jobs
+ * @param config config parameter:
+ * builds - List of job build objects, which should be checked
+ * checkBuilds - List of job names or regexps, which should be used to check provided builds list
+ * regexp - Wheither to use regexp or simple string matching
+ */
+def waitForOtherBuilds(LinkedHashMap config){
+ def common = new com.mirantis.mk.Common()
+ def builds = config.get('builds')
+ def checkBuilds = config.get('checkBuilds')
+ def regexp = config.get('regexp', false)
+ def waitForBuilds = builds.findAll { build ->
+ def jobName = build.fullDisplayName.tokenize(' ')[0]
+ if (regexp) {
+ checkBuilds.find { jobName ==~ it }
+ } else {
+ jobName in checkBuilds
+ }
+ }
+ if (waitForBuilds) {
+ def waiting = true
+ common.infoMsg("Waiting for next jobs: ${waitForBuilds}")
+ while(waiting) {
+ waiting = false
+ waitForBuilds.each { job ->
+ if (job.inProgress) {
+ waiting = true
+ }
+ }
+ }
+ }
+}
diff --git a/src/com/mirantis/mk/Mirror.groovy b/src/com/mirantis/mk/Mirror.groovy
new file mode 100644
index 0000000..a13f1b7
--- /dev/null
+++ b/src/com/mirantis/mk/Mirror.groovy
@@ -0,0 +1,34 @@
+package com.mirantis.mk
+
+/**
+ *
+ * Functions to work with mirror.mirantis.com
+ *
+ */
+
+/**
+ * Returns latest meta for latest available snapshot
+ *
+ * @param host mirror host
+ * @param version version of components (nightly|testing|stable|release-x)
+ * @param component component name (salt-formulas)
+ * @param distribution ubuntu distribution to get repo for (xenial by default)
+ */
+def getLatestSnapshotMeta(host, version, component, distribution='xenial') {
+ common = new com.mirantis.mk.Common()
+
+ def repoUrl = "http://${host}/${version}/${component}/${distribution}.target.txt"
+ def res
+ def snapshotName
+ def meta = [:]
+
+ res = common.shCmdStatus("curl ${repoUrl}")
+ // Return multiple lines where first one is the latest snapshot
+ // ../../.snapshots/nightly-salt-formulas-xenial-2018-12-10-204157
+ snapshotName = res['stdout'].split("\n")[0].tokenize('/').last().trim()
+
+ meta['repoUrl'] = "http://${host}/.snapshots/${snapshotName}"
+ meta['repoName'] = snapshotName
+
+ return meta
+}
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 773a1ee..75443a3 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -277,7 +277,10 @@
// Install sphinx server
salt.enforceStateWithTest(master, "I@sphinx:server ${extra_tgt}", 'sphinx')
- salt.enforceStateWithTest(master, "I@nginx:server ${extra_tgt}", 'salt.minion')
+ // Running minion states in a batch to avoid races related to certificates which are placed on glusterfs
+ // Details on races: https://mirantis.jira.com/browse/PROD-25796
+ // TODO: Run in parallel when glusterfs for certificates is dropped in cookiecutter
+ salt.enforceStateWithTest(master, "I@nginx:server ${extra_tgt}", 'salt.minion', '', true, true, 1)
salt.enforceStateWithTest(master, "I@nginx:server ${extra_tgt}", 'nginx')
// setup keystone service
@@ -353,7 +356,7 @@
salt.enforceStateWithTest(master, "I@heat:server:role:primary ${extra_tgt}", 'heat', "I@heat:server ${extra_tgt}")
salt.enforceStateWithTest(master, "I@heat:server ${extra_tgt}", 'heat')
if (salt.testTarget(master, "I@keystone:server and I@heat:server ${extra_tgt}")) {
- common.retry(3,5){
+ common.retry(10,5){
salt.cmdRun(master, "I@keystone:server ${extra_tgt}", '. /root/keystonercv3; openstack orchestration resource type list')
}
}
diff --git a/src/com/mirantis/mk/Python.groovy b/src/com/mirantis/mk/Python.groovy
index f135cc0..6183f51 100644
--- a/src/com/mirantis/mk/Python.groovy
+++ b/src/com/mirantis/mk/Python.groovy
@@ -247,7 +247,8 @@
requirements = [
'cookiecutter',
'jinja2==2.8.1',
- 'PyYAML==3.12'
+ 'PyYAML==3.12',
+ 'python-gnupg==0.4.3'
]
setupVirtualenv(path, 'python2', requirements)
}
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index ca209be..3d46bb3 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -159,9 +159,27 @@
* @param saltArgs additional salt args eq. ["runas=aptly"]
* @return output of salt command
*/
+def enforceStateWithExclude(Map params) {
+ //Set defaults
+ defaults = ["excludedStates": "", "output": true, "failOnError": true, "batch": null, "optional": false,
+ "read_timeout": -1, "retries": -1, "queue": true, "saltArgs": []]
+ params = defaults + params
+ params.saltArgs << "exclude=${params.excludedStates}"
+ params.remove('excludedStates')
+ return enforceState(params)
+}
+
+
def enforceStateWithExclude(saltId, target, state, excludedStates = "", output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs=[]) {
- saltArgs << "exclude=${excludedStates}"
- return enforceState(saltId, target, state, output, failOnError, batch, optional, read_timeout, retries, queue, saltArgs)
+// Deprecated, convert state to use Map as input parameter
+ def common = new com.mirantis.mk.Common()
+ common.infoMsg("This method is deprecated. Convert you method call to use Map as input parameter")
+ // Convert to Map
+ params = ['saltId': saltId, 'target': target, 'state': state, 'excludedStates': excludedStates, 'output': output,
+ 'failOnError': failOnError, 'batch': batch, 'optional': optional, 'read_timeout': read_timeout,
+ 'retries': retries, 'queue': queue, 'saltArgs': saltArgs]
+ // Call new method with Map as parameter
+ return enforceStateWithExclude(params)
}
/**
@@ -180,23 +198,40 @@
* @param saltArgs additional salt args eq. ["runas=aptly"]
* @return output of salt command
*/
-def enforceStateWithTest(saltId, target, state, testTargetMatcher = "", output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs=[]) {
+def enforceStateWithTest(Map params) {
def common = new com.mirantis.mk.Common()
- if (!testTargetMatcher) {
- testTargetMatcher = target
+ //Set defaults
+ defaults = ["testTargetMatcher": "", "output": true, "failOnError": true, "batch": null, "optional": false,
+ "read_timeout": -1, "retries": -1, "queue": true, "saltArgs":[]]
+ params = defaults + params
+ if (!params.testTargetMatcher) {
+ params.testTargetMatcher = params.target
}
- if (testTarget(saltId, testTargetMatcher)) {
- return enforceState(saltId, target, state, output, failOnError, batch, false, read_timeout, retries, queue, saltArgs)
+ if (testTarget(params.saltId, params.testTargetMatcher)) {
+ return enforceState(params)
} else {
- if (!optional) {
- common.infoMsg("No Minions matched the target matcher: ${testTargetMatcher}, and 'optional' param was set to false. - This may signify missing pillar definition!!")
+ if (!params.optional) {
+ common.infoMsg("No Minions matched the target matcher: ${params.testTargetMatcher}, and 'optional' param was set to false. - This may signify missing pillar definition!!")
// throw new Exception("No Minions matched the target matcher: ${testTargetMatcher}.") TODO: Change the infoMsg to Error once the methods are changed to Use named params and optional param will be set globally
} else {
- common.infoMsg("No Minions matched the target matcher: ${testTargetMatcher}, but 'optional' param was set to true - Pipeline continues. ")
+ common.infoMsg("No Minions matched the target matcher: ${params.testTargetMatcher}, but 'optional' param was set to true - Pipeline continues. ")
}
}
}
+
+def enforceStateWithTest(saltId, target, state, testTargetMatcher = "", output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs=[]) {
+// Deprecated, convert state to use Map as input parameter
+ def common = new com.mirantis.mk.Common()
+ common.infoMsg("This method is deprecated. Convert you method call to use Map as input parameter")
+ // Convert to Map
+ params = ['saltId': saltId, 'target': target, 'state': state, 'testTargetMatcher': testTargetMatcher, 'output': output,
+ 'failOnError': failOnError, 'batch': batch, 'optional': optional, 'read_timeout': read_timeout,
+ 'retries': retries, 'queue': queue, 'saltArgs': saltArgs]
+ // Call new method with Map as parameter
+ return enforceStateWithTest(params)
+}
+
/* Enforces state on given saltId and target
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
* @param target State enforcing target
@@ -212,45 +247,61 @@
* @param minionRestartWaitTimeout specifies timeout that we should wait after minion restart.
* @return output of salt command
*/
-def enforceState(saltId, target, state, output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs = [], minionRestartWaitTimeout=10) {
+def enforceState(Map params) {
def common = new com.mirantis.mk.Common()
+ //Set defaults
+ defaults = ["output": true, "failOnError": true, "batch": null, "optional": false,
+ "read_timeout": -1, "retries": -1, "queue": true, "saltArgs": [], "minionRestartWaitTimeout": 10]
+ params = defaults + params
// add state to salt args
- if (state instanceof String) {
- saltArgs << state
+ if (params.state instanceof String) {
+ params.saltArgs << params.state
} else {
- saltArgs << state.join(',')
+ params.saltArgs << params.state.join(',')
}
- common.infoMsg("Running state ${state} on ${target}")
+ common.infoMsg("Running state ${params.state} on ${params.target}")
def out
def kwargs = [:]
- if (queue && batch == null) {
+ if (params.queue && params.batch == null) {
kwargs["queue"] = true
}
- if (optional == false || testTarget(saltId, target)){
- if (retries > 0){
+ if (params.optional == false || params.testTarget(params.saltId, params.target)){
+ if (params.retries > 0){
def retriesCounter = 0
- retry(retries){
+ retry(params.retries){
retriesCounter++
// we have to reverse order in saltArgs because salt state have to be first
- out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, saltArgs.reverse(), kwargs, -1, read_timeout)
+ out = runSaltCommand(params.saltId, 'local', ['expression': params.target, 'type': 'compound'], 'state.sls', params.batch, params.saltArgs.reverse(), kwargs, -1, params.read_timeout)
// failOnError should be passed as true because we need to throw exception for retry block handler
- checkResult(out, true, output, true, retriesCounter < retries) //disable ask on error for every interation except last one
+ checkResult(out, true, params.output, true, retriesCounter < params.retries) //disable ask on error for every interation except last one
}
} else {
// we have to reverse order in saltArgs because salt state have to be first
- out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'state.sls', batch, saltArgs.reverse(), kwargs, -1, read_timeout)
- checkResult(out, failOnError, output)
+ out = runSaltCommand(params.saltId, 'local', ['expression': params.target, 'type': 'compound'], 'state.sls', params.batch, params.saltArgs.reverse(), kwargs, -1, params.read_timeout)
+ checkResult(out, params.failOnError, params.output)
}
- waitForMinion(out, minionRestartWaitTimeout)
+ waitForMinion(out, params.minionRestartWaitTimeout)
return out
} else {
common.infoMsg("No Minions matched the target given, but 'optional' param was set to true - Pipeline continues. ")
}
}
+def enforceState(saltId, target, state, output = true, failOnError = true, batch = null, optional = false, read_timeout=-1, retries=-1, queue=true, saltArgs = [], minionRestartWaitTimeout=10) {
+// Deprecated, convert state to use Map as input parameter
+ def common = new com.mirantis.mk.Common()
+ common.infoMsg("This method is deprecated. Convert you method call to use Map as input parameter")
+ // Convert to Map
+ params = ['saltId': saltId, 'target': target, 'state': state, 'output': output,
+ 'failOnError': failOnError, 'batch': batch, 'optional': optional, 'read_timeout': read_timeout,
+ 'retries': retries, 'queue': queue, 'saltArgs': saltArgs, 'minionRestartWaitTimeout': minionRestartWaitTimeout]
+ // Call new method with Map as parameter
+ return enforceState(params)
+}
+
/**
* Run command on salt minion (salt cmd.run wrapper)
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 156034b..16e469c 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -83,8 +83,8 @@
mcp_extra:
source: "${extraRepoSource}"
mcp_saltformulas:
- source: "deb http://apt.mcp.mirantis.net/xenial ${distribRevision} salt salt-latest"
- repo_key: "http://apt.mcp.mirantis.net/public.gpg"
+ source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial xenial main"
+ repo_key: "http://mirror.mirantis.com/${distribRevision}/salt-formulas/xenial/archive-salt-formulas.key"
ubuntu:
source: "deb [arch=amd64] http://mirror.mirantis.com/${distribRevision}/ubuntu xenial main restricted universe"
ubuntu-upd:
diff --git a/src/com/mirantis/mk/Test.groovy b/src/com/mirantis/mk/Test.groovy
index 483a922..8c663e5 100644
--- a/src/com/mirantis/mk/Test.groovy
+++ b/src/com/mirantis/mk/Test.groovy
@@ -409,7 +409,9 @@
*/
def install_docker(master, target) {
def salt = new com.mirantis.mk.Salt()
- salt.runSaltProcessStep(master, target, 'pkg.install', ["docker.io"])
+ def dockerPackagesPillar = salt.getPillar(master, target, 'docker:host:pkgs')
+ def dockerPackages = salt.getReturnValues(dockerPackagesPillar) ?: ['docker.io']
+ salt.runSaltProcessStep(master, target, 'pkg.install', dockerPackages)
}