Merge "Separating salt ca minions from highstate run to protect from race condition"
diff --git a/aptly-add-packages.groovy b/aptly-add-packages.groovy
new file mode 100644
index 0000000..2d117fc
--- /dev/null
+++ b/aptly-add-packages.groovy
@@ -0,0 +1,67 @@
+/**
+ *
+ * Aptly add packages pipeline
+ *
+ * Expected parameters:
+ * APTLY_API_URL - URL of Aptly API, example: http://10.1.0.12:8080
+ * APTLY_REPO - Name of aptly repo to put packages
+ * CLEANUP_REPO - Option to cleanup repo contents after publish
+ * PROMOTE - Option to promote the publish
+ * PROMOTE_COMPONENT - Component to promote (only used when PROMOTE = True)
+ * PROMOTE_SOURCE - Source regex to promote from (only used when PROMOTE = True)
+ * PROMOTE_TARGET - Target regex to promote to (only used when PROMOTE = True)
+ * PROMOTE_STORAGES - Storages to promote (only used when PROMOTE = True)
+ *
+ */
+
+// Load shared libs
+aptly = new com.mirantis.mk.Aptly()
+common = new com.mirantis.mk.Common()
+
+timeout(time: 12, unit: 'HOURS') {
+ node("docker&&hardware") {
+ try {
+ def aptlyServer = ["url": APTLY_API_URL]
+ def workspace = common.getWorkspace()
+ def actualTime = (System.currentTimeMillis()/1000).toInteger()
+ def snapshotName = APTLY_REPO + "-" + actualTime
+
+ lock("aptly-api") {
+ stage("upload") {
+ def inputFile = input message: 'Upload file', parameters: [file(name: 'packages.tar.gz')]
+ new hudson.FilePath(new File("$workspace/packages.tar.gz")).copyFrom(inputFile)
+ inputFile.delete()
+
+ sh "mkdir ${workspace}/packages;tar -xvzf ${workspace}/packages.tar.gz --directory ${workspace}/packages"
+
+ def packages = sh(script: "ls -1a ${workspace}/packages | tail -n +3", returnStdout: true)
+ packages = packages.tokenize("\n")
+ for(pkg in packages){
+ aptly.uploadPackage("${workspace}/packages/${pkg}", APTLY_API_URL, APTLY_REPO)
+ }
+ }
+
+ stage("publish") {
+ aptly.snapshotCreateByAPI(aptlyServer, APTLY_REPO, snapshotName)
+ aptly.publish(APTLY_API_URL)
+
+ if(PROMOTE.toBoolean()){
+ aptly.promotePublish(APTLY_API_URL, PROMOTE_SOURCE, PROMOTE_TARGET, false, PROMOTE_COMPONENT, null, false, '-d --timeout 600', false, PROMOTE_STORAGES)
+ }
+
+ if(CLEANUP_REPO.toBoolean()){
+ def packageList = aptly.listPackagesFromRepoByAPI(aptlyServer, APTLY_REPO)
+ aptly.deletePackagesFromRepoByAPI(aptlyServer, APTLY_REPO, packageList)
+ }
+ }
+ }
+ sh "rm -rf ${workspace}/*"
+ }
+ catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ sh "rm -rf ${workspace}/*"
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+ }
+}
\ No newline at end of file
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 7281677..66e4bc7 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -201,6 +201,13 @@
envParams.put('cluster_name', STACK_CLUSTER_NAME)
}
+ if (common.validInputParam('STACK_COMPUTE_COUNT')) {
+ if (STACK_COMPUTE_COUNT.toInteger() > 0){
+ common.infoMsg("Setting cluster_node_count to ${STACK_COMPUTE_COUNT}")
+ envParams.put('cluster_node_count', STACK_COMPUTE_COUNT)
+ }
+ }
+
openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, venv)
}
diff --git a/cloud-update.groovy b/cloud-update.groovy
index 9a33c36..8c8b8b6 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -352,77 +352,27 @@
}
}
-def getCfgNodeProvider(pepperEnv, master_name) {
+def getNodeProvider(pepperEnv, nodeName, type='') {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
- if (!CFG_NODE_PROVIDER?.trim()) {
- kvms = salt.getMinions(pepperEnv, 'I@salt:control')
- for (kvm in kvms) {
- try {
- vms = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, kvm, 'virt.list_active_vms', [], null, true))
- if (vms.toString().contains(master_name)) {
+ def kvms = salt.getMinions(pepperEnv, 'I@salt:control')
+ for (kvm in kvms) {
+ try {
+ vms = salt.getReturnValues(salt.runSaltProcessStep(pepperEnv, kvm, 'virt.list_domains', [], null, true))
+ if (vms.toString().contains(nodeName)) {
+ if (type == 'master' && !CFG_NODE_PROVIDER?.trim()) {
CFG_NODE_PROVIDER = kvm
+ } else {
+ return kvm
//break
}
- } catch (Exception er) {
- common.infoMsg("${master_name} not present on ${kvm}")
}
- }
- }
-}
-
-/*
-def rollbackSaltMaster(pepperEnv, target, path='/var/lib/libvirt/images') {
- def salt = new com.mirantis.mk.Salt()
- def common = new com.mirantis.mk.Common()
- try {
- input message: "PART1 - Are you sure to rollback ${target}? To rollback click on PROCEED. To skip rollback PART1 click on ABORT."
- } catch (Exception er) {
- common.infoMsg("skipping rollback of ${target}")
- return
- }
- def domain = salt.getDomainName(pepperEnv)
- def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
- getCfgNodeProvider(pepperEnv, master)
- try {
- try {
- salt.getReturnValues(salt.cmdRun(pepperEnv, CFG_NODE_PROVIDER, "ls -la ${path}/${master}.${domain}.xml"))
- common.errorMsg('Pipeline is about to disconnect from salt-api. You will have to rerun the pipeline with ROLLBACK_CFG checked and skip PART1 to finish rollback.')
- salt.cmdRun(pepperEnv, CFG_NODE_PROVIDER, "virsh destroy ${master}.${domain}; virsh define ${path}/${master}.${domain}.xml; virsh start ${master}.${domain} ")
} catch (Exception er) {
- common.errorMsg(er)
- input message: "Rollback for ${target} failed. Rollback manually."
+ common.infoMsg("${nodeName} not present on ${kvm}")
}
- } catch (Exception er) {
- common.errorMsg(er)
- input message: "Rollback for ${target} failed. Rollback manually."
}
}
-def finishSaltMasterRollback(pepperEnv, target, path='/var/lib/libvirt/images') {
- def salt = new com.mirantis.mk.Salt()
- def common = new com.mirantis.mk.Common()
- def virsh = new com.mirantis.mk.Virsh()
- try {
- input message: "PART2 - Are you sure to finalize ${target} rollback? Click on PROCEED. To skip rollback click on ABORT."
- } catch (Exception er) {
- common.infoMsg("skipping finalize rollback of ${target}")
- return
- }
- salt.minionsReachable(pepperEnv, 'I@salt:master', '*')
- def domain = salt.getDomainName(pepperEnv)
- def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
- getCfgNodeProvider(pepperEnv, master)
- try {
- virsh.liveSnapshotAbsent(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME, path)
- // purge and setup previous repos
- salt.enforceState(pepperEnv, target, 'linux.system.repo')
- } catch (Exception e) {
- common.errorMsg(e)
- input message: "Check what failed after ${target} rollback. Do you want to PROCEED?"
- }
-}*/
-
def services(pepperEnv, probe, target, action='stop') {
def services = ["keepalived","haproxy","nginx","nova-api","cinder","glance","heat","neutron","apache2","rabbitmq-server"]
if (action == 'stop') {
@@ -543,14 +493,10 @@
def domain = salt.getDomainName(pepperEnv)
def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
common.warningMsg(target_hosts)
- //def nodeCount = 1
for (t in target_hosts) {
def target = salt.stripDomainName(t)
- def nodeCount = target[4]
- common.warningMsg(nodeCount)
- def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+ def nodeProvider = getNodeProvider(pepperEnv, t)
virsh.liveSnapshotPresent(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
- //nodeCount++
}
}
@@ -559,18 +505,16 @@
def virsh = new com.mirantis.mk.Virsh()
def domain = salt.getDomainName(pepperEnv)
def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
- def nodeCount = 1
for (t in target_hosts) {
if (tgt == 'I@salt:master') {
def master = salt.getReturnValues(salt.getPillar(pepperEnv, t, 'linux:network:hostname'))
- getCfgNodeProvider(pepperEnv, master)
+ getNodeProvider(pepperEnv, master, 'master')
virsh.liveSnapshotMerge(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME)
} else {
def target = salt.stripDomainName(t)
- def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+ def nodeProvider = getNodeProvider(pepperEnv, t)
virsh.liveSnapshotMerge(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
}
- nodeCount++
}
salt.minionsReachable(pepperEnv, 'I@salt:master', tgt)
}
@@ -584,20 +528,16 @@
def domain = salt.getDomainName(pepperEnv)
def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
// first destroy all vms
- def nodeCount = 1
for (t in target_hosts) {
def target = salt.stripDomainName(t)
- def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+ def nodeProvider = getNodeProvider(pepperEnv, t)
salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.destroy', ["${target}.${domain}"], null, true)
- nodeCount++
}
- nodeCount = 1
// rollback vms
for (t in target_hosts) {
def target = salt.stripDomainName(t)
- def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+ def nodeProvider = getNodeProvider(pepperEnv, t)
virsh.liveSnapshotRollback(pepperEnv, nodeProvider, target, SNAPSHOT_NAME)
- nodeCount++
}
try {
salt.minionsReachable(pepperEnv, 'I@salt:master', tgt)
@@ -620,10 +560,9 @@
def domain = salt.getDomainName(pepperEnv)
def target_hosts = salt.getMinionsSorted(pepperEnv, "${tgt}")
// first destroy all vms
- def nodeCount = 1
for (t in target_hosts) {
def target = salt.stripDomainName(t)
- def nodeProvider = salt.getNodeProvider(pepperEnv, "${generalTarget}0${nodeCount}")
+ def nodeProvider = getNodeProvider(pepperEnv, t)
salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.destroy', ["${target}.${domain}"], null, true)
//salt.runSaltProcessStep(pepperEnv, "${nodeProvider}*", 'virt.undefine', ["${target}.${domain}"], null, true)
try {
@@ -631,7 +570,6 @@
} catch (Exception e) {
common.warningMsg('does not match any accepted, unaccepted or rejected keys. They were probably already removed. We should continue to run')
}
- nodeCount++
}
}
@@ -880,7 +818,7 @@
def type = 'cfg'
if (salt.testTarget(pepperEnv, target)) {
def master = salt.getReturnValues(salt.getPillar(pepperEnv, target, 'linux:network:hostname'))
- getCfgNodeProvider(pepperEnv, master)
+ getNodeProvider(pepperEnv, master, 'master')
if (!ROLLBACK_BY_REDEPLOY.toBoolean()) {
virsh.liveSnapshotPresent(pepperEnv, CFG_NODE_PROVIDER, master, SNAPSHOT_NAME)
} else {
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 21134a2..2f91865 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -107,10 +107,17 @@
salt.runSaltProcessStep(pepperEnv, "${test_upgrade_node}*", 'saltutil.sync_all', [])
}
+ stateRun(pepperEnv, "${test_upgrade_node}*", ['linux.network.proxy'])
+ try {
+ salt.runSaltProcessStep(pepperEnv, "${test_upgrade_node}*", 'state.sls', ["salt.minion.base"], null, true, 60)
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+
stateRun(pepperEnv, "${test_upgrade_node}*", ['linux', 'openssh'])
try {
- salt.runSaltProcessStep(master, "${test_upgrade_node}*", 'state.sls', ["salt.minion"], null, true, 60)
+ salt.runSaltProcessStep(pepperEnv, "${test_upgrade_node}*", 'state.sls', ["salt.minion"], null, true, 60)
} catch (Exception e) {
common.warningMsg(e)
}
@@ -263,6 +270,13 @@
// salt '*' saltutil.sync_all
salt.runSaltProcessStep(pepperEnv, upgrade_general_target, 'saltutil.sync_all', [])
+ stateRun(pepperEnv, upgrade_general_target, ['linux.network.proxy'])
+ try {
+ salt.runSaltProcessStep(pepperEnv, upgrade_general_target, 'state.sls', ["salt.minion.base"], null, true, 60)
+ } catch (Exception e) {
+ common.warningMsg(e)
+ }
+
if (OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == false) {
try {
@@ -302,7 +316,7 @@
common.warningMsg(e)
}
try {
- salt.runSaltProcessStep(master, upgrade_general_target, 'state.sls', ["salt.minion"], null, true, 60)
+ salt.runSaltProcessStep(pepperEnv, upgrade_general_target, 'state.sls', ["salt.minion"], null, true, 60)
} catch (Exception e) {
common.warningMsg(e)
}
diff --git a/release-mcp-version.groovy b/release-mcp-version.groovy
index 95e8bac..8af3fbe 100644
--- a/release-mcp-version.groovy
+++ b/release-mcp-version.groovy
@@ -16,6 +16,10 @@
* DOCKER_IMAGES
* GIT_CREDENTIALS
* GIT_REPO_LIST
+ * EMAIL_NOTIFY
+ * NOTIFY_RECIPIENTS
+ * NOTIFY_TEXT
+ *
*/
common = new com.mirantis.mk.Common()
@@ -51,31 +55,14 @@
]
}
-def gitRepoAddTag(repoURL, repoName, tag, credentials, ref = "HEAD"){
- git.checkoutGitRepository(repoName, repoURL, "master", credentials)
- dir(repoName) {
- def checkTag = sh(script: "git tag -l ${tag}", returnStdout: true)
- if(checkTag == ""){
- sh "git tag -a ${tag} ${ref} -m \"Release of mcp version ${tag}\""
- }else{
- def currentTagRef = sh(script: "git rev-list -n 1 ${tag}", returnStdout: true)
- if(currentTagRef.equals(ref)){
- common.infoMsg("Tag is already on the right ref")
- return
- }
- else{
- sshagent([credentials]) {
- sh "git push --delete origin ${tag}"
- }
- sh "git tag --delete ${tag}"
- sh "git tag -a ${tag} ${ref} -m \"Release of mcp version ${tag}\""
- }
- }
- sshagent([credentials]) {
- sh "git push origin ${tag}"
- }
- }
+def triggerGitTagJob(gitRepoList, gitCredentials, tag) {
+ build job: "tag-git-repos-stable", parameters: [
+ [$class: 'StringParameterValue', name: 'GIT_REPO_LIST', value: gitRepoList],
+ [$class: 'StringParameterValue', name: 'GIT_CREDENTIALS', value: gitCredentials],
+ [$class: 'StringParameterValue', name: 'TAG', value: tag]
+ ]
}
+
timeout(time: 12, unit: 'HOURS') {
node() {
try {
@@ -100,18 +87,13 @@
if(RELEASE_GIT.toBoolean())
{
common.infoMsg("Promoting Git repositories")
- def repos = GIT_REPO_LIST.tokenize('\n')
- def repoUrl, repoName, repoCommit, repoArray
- for (repo in repos){
- if(repo.trim().indexOf(' ') == -1){
- throw new IllegalArgumentException("Wrong format of repository and commit input")
- }
- repoArray = repo.trim().tokenize(' ')
- repoName = repoArray[0]
- repoUrl = repoArray[1]
- repoCommit = repoArray[2]
- gitRepoAddTag(repoUrl, repoName, TARGET_REVISION, GIT_CREDENTIALS, repoCommit)
- }
+ triggerGitTagJob(GIT_REPO_LIST, GIT_CREDENTIALS, TARGET_REVISION)
+
+ }
+ if (EMAIL_NOTIFY.toBoolean()) {
+ emailext(to: NOTIFY_RECIPIENTS,
+ body: NOTIFY_TEXT,
+ subject: "MCP Promotion has been done")
}
}
} catch (Throwable e) {
diff --git a/tag-git-repos.groovy b/tag-git-repos.groovy
new file mode 100644
index 0000000..373e029
--- /dev/null
+++ b/tag-git-repos.groovy
@@ -0,0 +1,62 @@
+/**
+ *
+ * Tag Git repositories
+ *
+ * Expected parameters:
+ * GIT_REPO_LIST
+ * GIT_CREDENTIALS
+ * TAG
+ *
+ */
+
+common = new com.mirantis.mk.Common()
+git = new com.mirantis.mk.Git()
+
+def gitRepoAddTag(repoURL, repoName, tag, credentials, ref = "HEAD"){
+ git.checkoutGitRepository(repoName, repoURL, "master", credentials)
+ dir(repoName) {
+ def checkTag = sh(script: "git tag -l ${tag}", returnStdout: true)
+ if(checkTag == ""){
+ sh "git tag -a ${tag} ${ref} -m \"Release of mcp version ${tag}\""
+ }else{
+ def currentTagRef = sh(script: "git rev-list -n 1 ${tag}", returnStdout: true)
+ if(currentTagRef.equals(ref)){
+ common.infoMsg("Tag is already on the right ref")
+ return
+ }
+ else{
+ sshagent([credentials]) {
+ sh "git push --delete origin ${tag}"
+ }
+ sh "git tag --delete ${tag}"
+ sh "git tag -a ${tag} ${ref} -m \"Release of mcp version ${tag}\""
+ }
+ }
+ sshagent([credentials]) {
+ sh "git push origin ${tag}"
+ }
+ }
+}
+
+timeout(time: 12, unit: 'HOURS') {
+ node() {
+ try {
+ def repos = GIT_REPO_LIST.tokenize('\n')
+ def repoUrl, repoName, repoCommit, repoArray
+ for (repo in repos){
+ if(repo.trim().indexOf(' ') == -1){
+ throw new IllegalArgumentException("Wrong format of repository and commit input")
+ }
+ repoArray = repo.trim().tokenize(' ')
+ repoName = repoArray[0]
+ repoUrl = repoArray[1]
+ repoCommit = repoArray[2]
+ gitRepoAddTag(repoUrl, repoName, TAG, GIT_CREDENTIALS, repoCommit)
+ }
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+ }
+}
\ No newline at end of file