Merge "Revert "Add restoreGluster function""
diff --git a/src/com/mirantis/mcp/SlackNotification.groovy b/src/com/mirantis/mcp/SlackNotification.groovy
new file mode 100644
index 0000000..d78d63a
--- /dev/null
+++ b/src/com/mirantis/mcp/SlackNotification.groovy
@@ -0,0 +1,76 @@
+package com.mirantis.mcp
+
+/**
+ * Send POST request to url with some params
+ * @param urlString url
+ * @param paramString JSON string
+ */
+def sendPostRequest(String urlString, String paramString){
+ def url = new URL(urlString)
+ def conn = url.openConnection()
+ conn.setDoOutput(true)
+ def writer = new OutputStreamWriter(conn.getOutputStream())
+ writer.write(paramString)
+ writer.flush()
+ String line
+ def reader = new BufferedReader(new InputStreamReader(conn.getInputStream()))
+ while ((line = reader.readLine()) != null) {
+ println line
+ }
+ writer.close()
+ reader.close()
+}
+
+/**
+ * Send slack notification
+ * @param buildStatusParam message type (success, warning, error)
+ * @param jobName job name param, if empty env.JOB_NAME will be used
+ * @param buildNumber build number param, if empty env.BUILD_NUM will be used
+ * @param buildUrl build url param, if empty env.BUILD_URL will be used
+ * @param channel param, default is '#mk-ci'
+ * @param credentialsId slack hook url credential, default is 'SLACK_WEBHOOK_URL'
+ */
+def jobResultNotification(String buildStatusParam, String channel = "#mk-ci",
+ String jobName=null,
+ Number buildNumber=null, String buildUrl=null,
+ String credentialsId="SLACK_WEBHOOK_URL") {
+ def jobNameParam = jobName != null && jobName != "" ? jobName : env.JOB_NAME
+ def buildUrlParam = buildUrl != null && buildUrl != "" ? buildUrl : env.BUILD_URL
+ def buildNumberParam = buildNumber != null && buildNumber != "" ? buildNumber : env.BUILD_NUMBER
+
+
+ def common = new com.mirantis.mk.Common()
+ cred = common.getCredentialsById(credentialsId)
+ hook_url_parsed = cred.getSecret().toString()
+ if (buildStatusParam.toLowerCase().equals("success")) {
+ colorCode = "#00FF00"
+ colorName = "green"
+ } else if (buildStatusParam.toLowerCase().equals("unstable")) {
+ colorCode = "#FFFF00"
+ colorName = "yellow"
+ } else if (buildStatusParam.toLowerCase().equals("failure")) {
+ colorCode = "#FF0000"
+ colorName = "red"
+ }
+
+ queryString = 'payload={' +
+ "'text':'${buildStatusParam.toUpperCase()}: Job <${buildUrlParam}|${jobNameParam} [${buildNumberParam}]>', " +
+ "'color':'${colorCode}'," +
+ "'pretext': '', " +
+ '"icon_url": "https://cfr.slack-edge.com/ae7f/img/services/jenkins-ci_192.png",' +
+ "'channel': '${channel}', " +
+ '}'
+ sendPostRequest(hook_url_parsed, queryString)
+
+}
+
+/*
+node {
+ jobResultNotification(
+ "success",
+ "#test_reclass_notify",
+ "test-reclass-system",
+ 44,
+ "https://ci.mcp.mirantis.net/",)
+}
+*/
diff --git a/src/com/mirantis/mk/Aptly.groovy b/src/com/mirantis/mk/Aptly.groovy
index 4771084..94f085f 100644
--- a/src/com/mirantis/mk/Aptly.groovy
+++ b/src/com/mirantis/mk/Aptly.groovy
@@ -144,7 +144,8 @@
if (findFiles(glob: "${prefix}*")) {
archiveArtifacts artifacts: "${prefix}*"
} else {
- throw new Exception("Aptly dump publishes for a prefix ${prefix}* failed. No dump files found!")
+ def common = new com.mirantis.mk.Common()
+ common.warningMsg("Aptly dump publishes for a prefix ${prefix}* failed. No dump files found! This can be OK in case of your creating new publish")
}
}
@@ -205,6 +206,32 @@
}
/**
+ * Returns list of the packages from specified Aptly repo by REST API
+ *
+ * @param server URI of the server insluding port and protocol
+ * @param repo Local repo name
+ **/
+def listPackagesFromRepoByAPI(server, repo){
+ http = new com.mirantis.mk.Http()
+ def packageList = http.restGet(server, "/api/repos/${repo}/packages")
+ return packageList
+}
+
+/**
+ * Deletes packages from specified Aptly repo by REST API
+ *
+ * @param server URI of the server insluding port and protocol
+ * @param repo Local repo name
+ * @param packageRefs Package list specified by packageRefs
+ **/
+def deletePackagesFromRepoByAPI(server, repo, packageRefs){
+ http = new com.mirantis.mk.Http()
+ def data = [:]
+ data['PackageRefs'] = packageRefs
+ http.restDelete(server, "/api/repos/${repo}/packages", data)
+}
+
+/**
* Returns list of the packages matched to pattern and
* belonged to particular snapshot by REST API
*
@@ -247,7 +274,7 @@
data['PackageRefs'] = packageRefs
http.restPost(server, '/api/snapshots', data)
} else {
- http.restPost(server + "/api/repos/${repo}/snapshots", data)
+ http.restPost(server, "/api/repos/${repo}/snapshots", data)
}
}
diff --git a/src/com/mirantis/mk/Aws.groovy b/src/com/mirantis/mk/Aws.groovy
index 6a9c8ae..5c07193 100644
--- a/src/com/mirantis/mk/Aws.groovy
+++ b/src/com/mirantis/mk/Aws.groovy
@@ -123,14 +123,14 @@
// check for desired state
if (stack_info['StackStatus'] == state) {
- common.successMsg("Stack ${stack_name} in in state ${state}")
+ common.successMsg("Stack ${stack_name} is in state ${state}")
common.prettyPrint(stack_info)
break
}
// check for failed state
if (state_failed.contains(stack_info['StackStatus'])) {
- throw new Exception("Stack ${stack_name} in in failed state")
+ throw new Exception("Stack ${stack_name} is in failed state")
}
// print stack resources
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index 4097b3c..9a82fbe 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -513,3 +513,29 @@
currentBuild.result = "FAILURE"
throw new Exception("Failed after $times retries")
}
+
+
+/**
+ * Wait for user input with timeout
+ *
+ * @param timeoutInSeconds Timeout
+ * @param options Options for input widget
+ */
+def waitForInputThenPass(timeoutInSeconds, options=[message: 'Ready to go?']) {
+ def userInput = true
+ try {
+ timeout(time: timeoutInSeconds, unit: 'SECONDS') {
+ userInput = input options
+ }
+ } catch(err) { // timeout reached or input false
+ def user = err.getCauses()[0].getUser()
+ if('SYSTEM' == user.toString()) { // SYSTEM means timeout.
+ println("Timeout, proceeding")
+ } else {
+ userInput = false
+ println("Aborted by: [${user}]")
+ throw err
+ }
+ }
+ return userInput
+}
diff --git a/src/com/mirantis/mk/Debian.groovy b/src/com/mirantis/mk/Debian.groovy
index ff05ce3..4d7b1ee 100644
--- a/src/com/mirantis/mk/Debian.groovy
+++ b/src/com/mirantis/mk/Debian.groovy
@@ -29,7 +29,7 @@
def imageArray = image.split(":")
def os = imageArray[0]
def dist = imageArray[1]
- def img = dockerLib.getImage("tcpcloud/debian-build-${os}-${dist}:latest", image)
+ def img = dockerLib.getImage("mirantis/debian-build-${os}-${dist}:latest", image)
def workspace = common.getWorkspace()
def debug = env.getEnvironment().containsKey("DEBUG") && env["DEBUG"].toBoolean() ? "true" : ""
@@ -38,6 +38,7 @@
export DEBUG="${debug}" &&
export LD_LIBRARY_PATH=\${LD_LIBRARY_PATH:+"\$LD_LIBRARY_PATH:"}/usr/lib/libeatmydata &&
export LD_PRELOAD=\${LD_PRELOAD:+"\$LD_PRELOAD "}libeatmydata.so &&
+ export DEB_BUILD_OPTIONS=nocheck &&
[[ -z "${extraRepoUrl}" && "${extraRepoUrl}" != "null" ]] || echo "${extraRepoUrl}" | tr ";" "\\\\n" >/etc/apt/sources.list.d/extra.list &&
[[ -z "${extraRepoKeyUrl}" && "${extraRepoKeyUrl}" != "null" ]] || (
which curl || (apt-get update && apt-get install -y curl) &&
@@ -91,7 +92,7 @@
def imageArray = image.split(":")
def os = imageArray[0]
def dist = imageArray[1]
- def img = dockerLib.getImage("tcpcloud/debian-build-${os}-${dist}:latest", image)
+ def img = dockerLib.getImage("mirantis/debian-build-${os}-${dist}:latest", image)
def workspace = common.getWorkspace()
img.inside("-u root:root" ) {
@@ -122,7 +123,7 @@
def imageArray = image.split(":")
def os = imageArray[0]
def dist = imageArray[1]
- def img = dockerLib.getImage("tcpcloud/debian-build-${os}-${dist}:latest", image)
+ def img = dockerLib.getImage("mirantis/debian-build-${os}-${dist}:latest", image)
img.inside("-u root:root") {
@@ -154,7 +155,15 @@
else
NEW_VERSION=\$VERSION+\$TIMESTAMP.`git rev-parse --short HEAD`$revisionPostfix
fi &&
- sudo -H -E -u jenkins gbp dch --auto --multimaint-merge --ignore-branch --new-version=\$NEW_VERSION --distribution `lsb_release -c -s` --force-distribution &&
+ sudo -H -E -u jenkins gbp dch \
+ --auto \
+ --git-author \
+ --id-length=7 \
+ --git-log='--reverse' \
+ --ignore-branch \
+ --new-version=\$NEW_VERSION \
+ --distribution `lsb_release -c -s` \
+ --force-distribution &&
sudo -H -E -u jenkins git add -u debian/changelog &&
sudo -H -E -u jenkins git commit -m "New snapshot version \$NEW_VERSION"
) &&
@@ -177,7 +186,7 @@
def imageArray = image.split(":")
def os = imageArray[0]
def dist = imageArray[1]
- def img = dockerLib.getImage("tcpcloud/debian-build-${os}-${dist}:latest", image)
+ def img = dockerLib.getImage("mirantis/debian-build-${os}-${dist}:latest", image)
img.inside("-u root:root") {
sh("""cd ${workspace} && apt-get update && apt-get install -y lintian &&
lintian -Ii -E --pedantic --profile=${profile} ${changes}""")
diff --git a/src/com/mirantis/mk/Docker.groovy b/src/com/mirantis/mk/Docker.groovy
index 7b57e5b..796cc3c 100644
--- a/src/com/mirantis/mk/Docker.groovy
+++ b/src/com/mirantis/mk/Docker.groovy
@@ -35,7 +35,7 @@
/**
* Build step to build docker image.
*
- * @param dockerHubImg Name of image on dockerhub (ie: tcpcloud/salt-models-testing)
+ * @param dockerHubImg Name of image on dockerhub (ie: mirantis/salt-models-testing)
* @param defaultImg Image to use if dockerHubImg is not found
* @return img Docker image
*/
diff --git a/src/com/mirantis/mk/Http.groovy b/src/com/mirantis/mk/Http.groovy
index e14a523..987a998 100644
--- a/src/com/mirantis/mk/Http.groovy
+++ b/src/com/mirantis/mk/Http.groovy
@@ -220,3 +220,138 @@
System.getProperties().remove("http.nonProxyHosts")
System.getProperties().remove("https.nonProxyHosts")
}
+
+/**
+ * Make HTTP request to the specified URL
+ *
+ * @param url URL to do HTTP request to
+ * @param method HTTP method to execute (`GET` by default)
+ * @param credsId Jenkins credentials ID to use for authorization
+ * @param pushData Data to send
+ * @param pushType MIME-type of pushData
+ * @param params Custom HTTP-headers
+ *
+ * @return Array containing return code and text
+ *
+ * @exception org.jenkinsci.plugins.scriptsecurity.sandbox.RejectedAccessException
+ */
+def methodCall(String url, String method = 'GET', String credsId = '',
+ String pushData = '', String pushType = '', Map params = [:]) {
+
+ // Connection object
+ def httpReq = new URI(url).normalize().toURL().openConnection()
+ httpReq.setRequestMethod(method)
+
+ // Timeouts
+ httpReq.setConnectTimeout(10*1000) // milliseconds
+ httpReq.setReadTimeout(600*1000) // milliseconds
+
+ // Add authentication data
+ if (credsId) {
+ String authHeader = ''
+ def cred = com.cloudbees.plugins.credentials.CredentialsProvider.lookupCredentials(
+ com.cloudbees.plugins.credentials.impl.BaseStandardCredentials.class,
+ jenkins.model.Jenkins.instance).findAll {it.id == credsId}[0]
+ if (cred.class == com.cloudbees.plugins.credentials.impl.UsernamePasswordCredentialsImpl.class) {
+ authHeader = 'Basic ' + "${cred.getUsername()}:${cred.getPassword()}".getBytes('UTF-8').encodeBase64().toString()
+ } else if (cred.class == org.jenkinsci.plugins.plaincredentials.impl.StringCredentialsImpl.class) {
+ authHeader = 'Bearer ' + cred.getSecret()
+ //params << ['X-JFrog-Art-Api': cred.getSecret()]
+ }
+ params << ['Authorization': authHeader]
+ }
+
+ // Add custom headers if any
+ for (param in params) {
+ httpReq.setRequestProperty(param.key, param.value.toString())
+ }
+
+ // Do request
+ try {
+ if (pushData) {
+ httpReq.setRequestProperty('Content-Type', pushType ?: 'application/x-www-form-urlencoded')
+ if (method == 'GET') { // override incorrect method if pushData is passed
+ httpReq.setRequestMethod('POST')
+ }
+ httpReq.setDoOutput(true)
+ httpReq.getOutputStream().write(pushData.getBytes('UTF-8'))
+ } else {
+ httpReq.connect()
+ }
+ } catch (org.jenkinsci.plugins.scriptsecurity.sandbox.RejectedAccessException e) {
+ throw e // show sandbox errors
+ } catch (Exception e) {
+ echo "Cauhgt '${e.class.name}' error: ${e.getMessage()}"
+ return [ -1, e.getMessage() ]
+ }
+
+ // Handle return data
+ int respCode = httpReq.getResponseCode()
+ String respText = ''
+ if (respCode >= 400) {
+ respText = httpReq.getErrorStream().getText() ?: httpReq.getResponseMessage()
+ } else {
+ respText = httpReq.getInputStream().getText()
+ }
+
+ // Return result as a tuple of response code and text
+ return [respCode, respText]
+}
+
+/**
+ * Make HTTP GET request to the specified URL
+ *
+ * @param url URL to do HTTP request to
+ * @param credsId Jenkins credentials ID to use for authorization
+ * @param params Custom HTTP-headers
+ *
+ * @return Array containing return code and text
+ */
+def doGet(String url, String credsId = '', Map params = [:]) {
+ return methodCall(url, 'GET', credsId, null, null, params)
+}
+
+/**
+ * Make HTTP POST request to the specified URL
+ *
+ * @param url URL to do HTTP request to
+ * @param credsId Jenkins credentials ID to use for authorization
+ * @param pushData Data to send
+ * @param pushType MIME-type of pushData
+ * @param params Custom HTTP-headers
+ *
+ * @return Array containing return code and text
+ */
+def doPost(String url, String credsId = '',
+ String pushData = '', String pushType = '', Map params = [:]) {
+ return methodCall(url, 'POST', credsId, pushData, pushType, params)
+}
+
+/**
+ * Make HTTP PUT request to the specified URL
+ *
+ * @param url URL to do HTTP request to
+ * @param credsId Jenkins credentials ID to use for authorization
+ * @param pushData Data to send
+ * @param pushType MIME-type of pushData
+ * @param params Custom HTTP-headers
+ *
+ * @return Array containing return code and text
+ */
+def doPut(String url, String credsId = '',
+ String pushData = '', String pushType = '', Map params = [:]) {
+ return methodCall(url, 'PUT', credsId, pushData, pushType, params)
+}
+
+/**
+ * Make HTTP DELETE request to the specified URL
+ *
+ * @param url URL to do HTTP request to
+ * @param credsId Jenkins credentials ID to use for authorization
+ * @param params Custom HTTP-headers
+ *
+ * @return Array containing return code and text
+ */
+def doDelete(String url, String credsId = '', Map params = [:]) {
+ return methodCall(url, 'DELETE', credsId, null, null, params)
+}
diff --git a/src/com/mirantis/mk/Openstack.groovy b/src/com/mirantis/mk/Openstack.groovy
index 85f014f..eedfbd8 100644
--- a/src/com/mirantis/mk/Openstack.groovy
+++ b/src/com/mirantis/mk/Openstack.groovy
@@ -47,6 +47,13 @@
def openstack_latest_packages = [
//XXX: hack to fix https://bugs.launchpad.net/ubuntu/+source/python-pip/+bug/1635463
'cliff==2.8',
+ // NOTE(vsaienko): cmd2 is dependency for cliff, since we don't using upper-contstraints
+ // we have to pin cmd2 < 0.9.0 as later versions are not compatible with python2.
+ // TODO(vsaienko): use upper-constraints here, as in requirements we set only lowest library
+ // versions.
+ 'cmd2<0.9.0;python_version=="2.7"',
+ 'cmd2>=0.9.1;python_version=="3.4"',
+ 'cmd2>=0.9.1;python_version=="3.5"',
'python-openstackclient',
'python-heatclient',
'docutils'
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 664871f..33c0734 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -16,6 +16,7 @@
def installFoundationInfra(master, staticMgmtNet=false) {
def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
// NOTE(vsaienko) Apply reclass first, it may update cluster model
// apply linux and salt.master salt.minion states afterwards to make sure
@@ -29,48 +30,128 @@
salt.enforceState(master, 'I@salt:master', ['salt.minion'], true, false, null, false, 60, 2)
salt.enforceState(master, 'I@salt:master', ['salt.minion'])
salt.fullRefresh(master, "*")
-
- salt.enforceState(master, '*', ['linux.system'])
+ salt.enforceState(master, '*', ['linux.network.proxy'], true, false, null, false, 60, 2)
+ try {
+ salt.enforceState(master, '*', ['salt.minion.base'], true, false, null, false, 60, 2)
+ sleep(5)
+ } catch (Throwable e) {
+ common.warningMsg('Salt state salt.minion.base is not present in the Salt-formula yet.')
+ }
+ common.retry(2,5){
+ salt.enforceState(master, '*', ['linux.system'])
+ }
if (staticMgmtNet) {
salt.runSaltProcessStep(master, '*', 'cmd.shell', ["salt-call state.sls linux.network; salt-call service.restart salt-minion"], null, true, 60)
}
+ salt.enforceState(master, 'I@linux:network:interface', ['linux.network.interface'])
+ sleep(5)
salt.enforceState(master, 'I@linux:system', ['linux', 'openssh', 'ntp', 'rsyslog'])
salt.enforceState(master, '*', ['salt.minion'], true, false, null, false, 60, 2)
sleep(5)
+
+ salt.fullRefresh(master, "*")
salt.runSaltProcessStep(master, '*', 'mine.update', [], null, true)
salt.enforceState(master, '*', ['linux.network.host'])
+
+ // Install and configure iptables
+ if (salt.testTarget(master, 'I@iptables:service')) {
+ salt.enforceState(master, 'I@iptables:service', 'iptables')
+ }
+
+ // Install and configure logrotate
+ if (salt.testTarget(master, 'I@logrotate:server')) {
+ salt.enforceState(master, 'I@logrotate:server', 'logrotate')
+ }
}
def installFoundationInfraOnTarget(master, target, staticMgmtNet=false) {
def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
salt.enforceState(master, 'I@salt:master', ['reclass'], true, false, null, false, 120, 2)
salt.fullRefresh(master, target)
- salt.enforceState(master, target, ['linux.system'])
+ salt.enforceState(master, target, ['linux.network.proxy'], true, false, null, false, 60, 2)
+ try {
+ salt.enforceState(master, target, ['salt.minion.base'], true, false, null, false, 60, 2)
+ sleep(5)
+ } catch (Throwable e) {
+ common.warningMsg('Salt state salt.minion.base is not present in the Salt-formula yet.')
+ }
+ common.retry(2,5){
+ salt.enforceState(master, target, ['linux.system'])
+ }
if (staticMgmtNet) {
salt.runSaltProcessStep(master, target, 'cmd.shell', ["salt-call state.sls linux.network; salt-call service.restart salt-minion"], null, true, 60)
}
salt.enforceState(master, target, ['salt.minion'], true, false, null, false, 60, 2)
salt.enforceState(master, target, ['salt.minion'])
-
+ salt.enforceState(master, target, ['linux.network.interface'])
+ sleep(5)
salt.enforceState(master, target, ['linux', 'openssh', 'ntp', 'rsyslog'])
sleep(5)
+
+ salt.fullRefresh(master, target)
salt.runSaltProcessStep(master, target, 'mine.update', [], null, true)
salt.enforceState(master, target, ['linux.network.host'])
}
def installInfraKvm(master) {
+ def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
- salt.fullRefresh(master, 'I@linux:system')
+ def infra_compound = 'I@salt:control'
+ def minions = []
+ def wait_timeout = 10
+ def retries = wait_timeout * 30
+ salt.fullRefresh(master, 'I@linux:system')
salt.enforceState(master, 'I@salt:control', ['salt.minion'], true, false, null, false, 60, 2)
salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp', 'rsyslog'])
salt.enforceState(master, 'I@salt:control', 'libvirt')
salt.enforceState(master, 'I@salt:control', 'salt.control')
- sleep(600)
+ common.infoMsg("Building minions list...")
+ if (salt.testTarget(master, infra_compound)) {
+ // Gathering minions
+ for ( infra_node in salt.getMinionsSorted(master, infra_compound) ) {
+ def pillar = salt.getPillar(master, infra_node, 'salt:control:cluster')
+ if ( !pillar['return'].isEmpty() ) {
+ for ( cluster in pillar['return'][0].values() ) {
+ def engine = cluster.values()[0]['engine']
+ def domain = cluster.values()[0]['domain']
+ def node = cluster.values()[0]['node']
+ if ( engine == "virt" ) {
+ def nodes = node.values()
+ if ( !nodes.isEmpty() ) {
+ for ( vm in nodes ) {
+ if ( vm['name'] != null ) {
+ def vm_fqdn = vm['name'] + '.' + domain
+ if ( !minions.contains(vm_fqdn) ) {
+ minions.add(vm_fqdn)
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ def minions_compound = minions.join(' or ')
+
+ common.infoMsg("Waiting for next minions to register within ${wait_timeout} minutes: " + minions_compound)
+ timeout(time: wait_timeout, unit: 'MINUTES') {
+ salt.minionsPresentFromList(master, 'I@salt:master', minions, true, null, true, retries, 1)
+ }
+
+ common.infoMsg('Waiting for minions to respond')
+ timeout(time: wait_timeout, unit: 'MINUTES') {
+ salt.minionsReachable(master, 'I@salt:master', minions_compound)
+ }
+
+ common.infoMsg("All minions are up.")
salt.fullRefresh(master, '* and not kvm*')
+
}
def installInfra(master) {
@@ -265,11 +346,6 @@
}
}
- // Create neutron resources
- if (salt.testTarget(master, 'I@neutron:client')) {
- salt.enforceState(master, 'I@neutron:client', 'neutron.client')
- }
-
// Install heat service
if (salt.testTarget(master, 'I@heat:server')) {
// run on first node first
@@ -409,6 +485,20 @@
def installOpenstackNetwork(master, physical = "false") {
def salt = new com.mirantis.mk.Salt()
+ //run full neutron state on neutron.gateway - this will install
+ //neutron agents in addition to neutron server. Once neutron agents
+ //are up neutron resources can be created without hitting the situation when neutron resources are created
+ //prior to neutron agents which results in creating ports in non-usable state
+ if (salt.testTarget(master, 'I@neutron:gateway')) {
+ salt.enforceState(master, 'I@neutron:gateway', 'neutron')
+ }
+
+ // Create neutron resources - this step was moved here to ensure that
+ //neutron resources are created after neutron agens are up. In this case neutron ports will be in
+ //usable state. More information: https://bugs.launchpad.net/neutron/+bug/1399249
+ if (salt.testTarget(master, 'I@neutron:client')) {
+ salt.enforceState(master, 'I@neutron:client', 'neutron.client')
+ }
salt.enforceHighstate(master, 'I@neutron:gateway')
@@ -429,9 +519,25 @@
if (salt.testTarget(master, compute_compound)) {
// In case if infrastructure nodes are used as nova computes too
def gluster_compound = 'I@glusterfs:server'
- // Enforce highstate asynchronous only on compute nodes which are not glusterfs servers
- retry(2) {
- salt.enforceHighstateWithExclude(master, compute_compound + ' and not ' + gluster_compound, 'opencontrail.client')
+ def salt_ca_compound = 'I@salt:minion:ca:salt_master_ca'
+ // Enforce highstate asynchronous only on compute nodes which are not glusterfs and not salt ca servers
+ def hightstateTarget = "${compute_compound} and not ${gluster_compound} and not ${salt_ca_compound}"
+ if (salt.testTarget(master, hightstateTarget)) {
+ retry(2) {
+ salt.enforceHighstateWithExclude(master, hightstateTarget, 'opencontrail.client')
+ }
+ }
+ // Iterate through salt ca servers and check if they have compute role
+ // TODO: switch to batch once salt 2017.7+ would be used
+ for ( target in salt.getMinionsSorted(master, salt_ca_compound) ) {
+ for ( cmp_target in salt.getMinionsSorted(master, compute_compound) ) {
+ if ( target == cmp_target ) {
+ // Enforce highstate one by one on salt ca servers which are compute nodes
+ retry(2) {
+ salt.enforceHighstateWithExclude(master, target, 'opencontrail.client')
+ }
+ }
+ }
}
// Iterate through glusterfs servers and check if they have compute role
// TODO: switch to batch once salt 2017.7+ would be used
@@ -514,6 +620,9 @@
salt.enforceState(master, 'I@kubernetes:master and *01*', 'etcd.server.setup')
}
+ // Run k8s master at *01* to simplify namespaces creation
+ salt.enforceStateWithExclude(master, 'I@kubernetes:master and *01*', "kubernetes.master", "kubernetes.master.setup")
+
// Run k8s without master.setup
salt.enforceStateWithExclude(master, 'I@kubernetes:master', "kubernetes", "kubernetes.master.setup")
@@ -569,14 +678,69 @@
def installCicd(master) {
def salt = new com.mirantis.mk.Salt()
- salt.fullRefresh(master, 'I@jenkins:client or I@gerrit:client')
+ def common = new com.mirantis.mk.Common()
+ def gerrit_compound = 'I@gerrit:client and ci*'
+ def jenkins_compound = 'I@jenkins:client and ci*'
+
+ salt.fullRefresh(master, gerrit_compound)
+ salt.fullRefresh(master, jenkins_compound)
if (salt.testTarget(master, 'I@aptly:publisher')) {
salt.enforceState(master, 'I@aptly:publisher', 'aptly.publisher',true, null, false, -1, 2)
}
salt.enforceState(master, 'I@docker:swarm:role:master and I@jenkins:client', 'docker.client', true, true, null, false, -1, 2)
- sleep(500)
+
+ // API timeout in minutes
+ def wait_timeout = 10
+
+ // Gerrit
+ def gerrit_master_url = salt.getPillar(master, gerrit_compound, '_param:gerrit_master_url')
+
+ if(!gerrit_master_url['return'].isEmpty()) {
+ gerrit_master_url = gerrit_master_url['return'][0].values()[0]
+ } else {
+ gerrit_master_url = ''
+ }
+
+ if (gerrit_master_url != '') {
+ common.infoMsg('Gerrit master url "' + gerrit_master_url + '" retrieved at _param:gerrit_master_url')
+ } else {
+
+ common.infoMsg('Gerrit master url could not be retrieved at _param:gerrit_master_url. Falling back to gerrit pillar')
+
+ def gerrit_host
+ def gerrit_http_port
+ def gerrit_http_scheme
+
+ def host_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:host')
+ gerrit_host = salt.getReturnValues(host_pillar)
+
+ def port_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:http_port')
+ gerrit_http_port = salt.getReturnValues(port_pillar)
+
+ def scheme_pillar = salt.getPillar(master, gerrit_compound, 'gerrit:client:server:protocol')
+ gerrit_http_scheme = salt.getReturnValues(scheme_pillar)
+
+ gerrit_master_url = gerrit_http_scheme + '://' + gerrit_host + ':' + gerrit_http_port
+
+ }
+
+ timeout(wait_timeout) {
+ common.infoMsg('Waiting for Gerrit to come up..')
+ def check_gerrit_cmd = 'while true; do curl -sI -m 3 -o /dev/null -w' + " '" + '%{http_code}' + "' " + gerrit_master_url + '/ | grep 200 && break || sleep 1; done'
+ salt.cmdRun(master, gerrit_compound, 'timeout ' + (wait_timeout*60+3) + ' /bin/sh -c -- ' + '"' + check_gerrit_cmd + '"')
+ }
+
+ // Jenkins
+ def jenkins_master_url_pillar = salt.getPillar(master, jenkins_compound, '_param:jenkins_master_url')
+ jenkins_master_url = salt.getReturnValues(jenkins_master_url_pillar)
+
+ timeout(wait_timeout) {
+ common.infoMsg('Waiting for Jenkins to come up..')
+ def check_jenkins_cmd = 'while true; do curl -sI -m 3 -o /dev/null -w' + " '" + '%{http_code}' + "' " + jenkins_master_url + '/whoAmI/ | grep 200 && break || sleep 1; done'
+ salt.cmdRun(master, jenkins_compound, 'timeout ' + (wait_timeout*60+3) + ' /bin/sh -c -- ' + '"' + check_jenkins_cmd + '"')
+ }
if (salt.testTarget(master, 'I@aptly:server')) {
salt.enforceState(master, 'I@aptly:server', 'aptly', true, true, null, false, -1, 2)
@@ -614,6 +778,8 @@
def installStacklight(master) {
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
+ def retries_wait = 20
+ def retries = 15
// Install core services for K8S environments:
// HAProxy, Nginx and lusterFS clients
@@ -631,9 +797,15 @@
}
}
- // Launch containers
- salt.enforceState(master, 'I@docker:swarm:role:master and I@prometheus:server', 'docker.client')
- salt.runSaltProcessStep(master, 'I@docker:swarm and I@prometheus:server', 'dockerng.ps')
+ // Install MongoDB for Alerta
+ if (salt.testTarget(master, 'I@mongodb:server')) {
+ salt.enforceState(master, 'I@mongodb:server', 'mongodb.server')
+
+ // Initialize mongodb replica set
+ common.retry(5,20){
+ salt.enforceState(master, 'I@mongodb:server', 'mongodb.cluster')
+ }
+ }
//Install Telegraf
salt.enforceState(master, 'I@telegraf:agent or I@telegraf:remote_agent', 'telegraf')
@@ -648,8 +820,34 @@
salt.enforceState(master, 'I@elasticsearch:server', 'elasticsearch.server')
salt.enforceState(master, '*01* and I@kibana:server', 'kibana.server')
salt.enforceState(master, 'I@kibana:server', 'kibana.server')
- salt.enforceState(master, 'I@elasticsearch:client', 'elasticsearch.client')
- salt.enforceState(master, 'I@kibana:client', 'kibana.client')
+
+ // Check ES health cluster status
+ def pillar = salt.getPillar(master, 'I@elasticsearch:client', 'elasticsearch:client:server:host')
+ def elasticsearch_vip
+ if(!pillar['return'].isEmpty()) {
+ elasticsearch_vip = pillar['return'][0].values()[0]
+ } else {
+ common.errorMsg('[ERROR] Elasticsearch VIP address could not be retrieved')
+ }
+ pillar = salt.getPillar(master, 'I@elasticsearch:client', 'elasticsearch:client:server:port')
+ def elasticsearch_port
+ if(!pillar['return'].isEmpty()) {
+ elasticsearch_port = pillar['return'][0].values()[0]
+ } else {
+ common.errorMsg('[ERROR] Elasticsearch VIP port could not be retrieved')
+ }
+ common.retry(retries,retries_wait) {
+ common.infoMsg('Waiting for Elasticsearch to become green..')
+ salt.cmdRun(master, 'I@elasticsearch:client', "curl -sf ${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
+ }
+
+ common.retry(retries,retries_wait) {
+ salt.enforceState(master, 'I@elasticsearch:client', 'elasticsearch.client', true, true, 1)
+ }
+
+ common.retry(retries,retries_wait) {
+ salt.enforceState(master, 'I@kibana:client', 'kibana.client', true, true, 1)
+ }
//Install InfluxDB
if (salt.testTarget(master, 'I@influxdb:server')) {
@@ -657,11 +855,6 @@
salt.enforceState(master, 'I@influxdb:server', 'influxdb')
}
- //Install Prometheus LTS
- if (salt.testTarget(master, 'I@prometheus:relay')) {
- salt.enforceState(master, 'I@prometheus:relay', 'prometheus')
- }
-
// Install service for the log collection
if (salt.testTarget(master, 'I@fluentd:agent')) {
salt.enforceState(master, 'I@fluentd:agent', 'fluentd')
@@ -699,13 +892,22 @@
salt.enforceState(master, 'I@docker:swarm and I@prometheus:server', 'heka.remote_collector', true, false)
}
+ // Launch containers
+ salt.enforceState(master, 'I@docker:swarm:role:master and I@prometheus:server', 'docker.client')
+ salt.runSaltProcessStep(master, 'I@docker:swarm and I@prometheus:server', 'dockerng.ps')
+
+ //Install Prometheus LTS
+ if (salt.testTarget(master, 'I@prometheus:relay')) {
+ salt.enforceState(master, 'I@prometheus:relay', 'prometheus')
+ }
+
// Install sphinx server
if (salt.testTarget(master, 'I@sphinx:server')) {
salt.enforceState(master, 'I@sphinx:server', 'sphinx')
}
//Configure Grafana
- def pillar = salt.getPillar(master, 'ctl01*', '_param:stacklight_monitor_address')
+ pillar = salt.getPillar(master, 'ctl01*', '_param:stacklight_monitor_address')
common.prettyPrint(pillar)
def stacklight_vip
@@ -965,15 +1167,17 @@
// connect Ceph to the env
if (salt.testTarget(master, 'I@ceph:common and I@glance:server')) {
salt.enforceState(master, 'I@ceph:common and I@glance:server', ['ceph.common', 'ceph.setup.keyring', 'glance'])
- salt.runSaltProcessStep(master, 'I@ceph:common and I@glance:server', 'service.restart', ['glance-api', 'glance-glare', 'glance-registry'])
+ salt.runSaltProcessStep(master, 'I@ceph:common and I@glance:server', 'service.restart', ['glance-api'])
}
if (salt.testTarget(master, 'I@ceph:common and I@cinder:controller')) {
salt.enforceState(master, 'I@ceph:common and I@cinder:controller', ['ceph.common', 'ceph.setup.keyring', 'cinder'])
+ salt.runSaltProcessStep(master, 'I@ceph:common and I@cinder:controller', 'service.restart', ['cinder-volume'])
}
if (salt.testTarget(master, 'I@ceph:common and I@nova:compute')) {
salt.enforceState(master, 'I@ceph:common and I@nova:compute', ['ceph.common', 'ceph.setup.keyring'])
salt.runSaltProcessStep(master, 'I@ceph:common and I@nova:compute', 'saltutil.sync_grains')
salt.enforceState(master, 'I@ceph:common and I@nova:compute', ['nova'])
+ salt.runSaltProcessStep(master, 'I@ceph:common and I@nova:compute', 'service.restart', ['nova-compute'])
}
}
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 5973184..84659f6 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -261,34 +261,43 @@
* @param answers how many minions should return (optional, default 1)
* @return output of salt command
*/
-def minionPresent(saltId, target, minion_name, waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
+def minionPresent(saltId, target, minion_name, waitUntilPresent = true, batch=null, output = true, maxRetries = 180, answers = 1) {
minion_name = minion_name.replace("*", "")
def common = new com.mirantis.mk.Common()
+ common.infoMsg("Looking for minion: " + minion_name)
def cmd = 'salt-key | grep ' + minion_name
if (waitUntilPresent){
def count = 0
while(count < maxRetries) {
+ try {
+ def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
+ if (output) {
+ printSaltCommandResult(out)
+ }
+ def valueMap = out["return"][0]
+ def result = valueMap.get(valueMap.keySet()[0])
+ def resultsArray = result.tokenize("\n")
+ def size = resultsArray.size()
+ if (size >= answers) {
+ return out
+ }
+ count++
+ sleep(time: 1000, unit: 'MILLISECONDS')
+ common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
+ } catch (Exception er) {
+ common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
+ }
+ }
+ } else {
+ try {
def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
if (output) {
printSaltCommandResult(out)
}
- def valueMap = out["return"][0]
- def result = valueMap.get(valueMap.keySet()[0])
- def resultsArray = result.tokenize("\n")
- def size = resultsArray.size()
- if (size >= answers) {
- return out
- }
- count++
- sleep(time: 500, unit: 'MILLISECONDS')
- common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
+ return out
+ } catch (Exception er) {
+ common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
}
- } else {
- def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
- if (output) {
- printSaltCommandResult(out)
- }
- return out
}
// otherwise throw exception
common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")
@@ -296,8 +305,8 @@
}
/**
- * Checks if salt minion is in a list of salt master's accepted keys
- * @usage minionPresent(saltId, 'I@salt:master', 'I@salt:minion', true, null, true, 200, 3)
+ * Checks if salt minions are in a list of salt master's accepted keys by matching compound
+ * @usage minionsPresent(saltId, 'I@salt:master', 'I@salt:minion', true, null, true, 200, 3)
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
* @param target Performs tests on this target node
* @param target_minions all targeted minions to test (for ex. I@salt:minion)
@@ -309,10 +318,31 @@
* @return output of salt command
*/
def minionsPresent(saltId, target = 'I@salt:master', target_minions = '', waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
- def target_hosts = getMinionsSorted(pepperEnv, target_minions)
+ def target_hosts = getMinionsSorted(saltId, target_minions)
for (t in target_hosts) {
- def tgt = salt.stripDomainName(t)
- salt.minionPresent(pepperEnv, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
+ def tgt = stripDomainName(t)
+ minionPresent(saltId, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
+ }
+}
+
+/**
+ * Checks if salt minions are in a list of salt master's accepted keys by matching a list
+ * @usage minionsPresentFromList(saltId, 'I@salt:master', ["cfg01.example.com", "bmk01.example.com"], true, null, true, 200, 3)
+ * @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
+ * @param target Performs tests on this target node
+ * @param target_minions list to test (for ex. ["cfg01.example.com", "bmk01.example.com"])
+ * @param waitUntilPresent return after the minion becomes present (default true)
+ * @param batch salt batch parameter integer or string with percents (optional, default null - disable batch)
+ * @param output print salt command (default true)
+ * @param maxRetries finite number of iterations to check status of a command (default 200)
+ * @param answers how many minions should return (optional, default 1)
+ * @return output of salt command
+ */
+def minionsPresentFromList(saltId, target = 'I@salt:master', target_minions = [], waitUntilPresent = true, batch=null, output = true, maxRetries = 200, answers = 1) {
+ def common = new com.mirantis.mk.Common()
+ for (tgt in target_minions) {
+ common.infoMsg("Checking if minion " + tgt + " is present")
+ minionPresent(saltId, target, tgt, waitUntilPresent, batch, output, maxRetries, answers)
}
}
@@ -476,7 +506,7 @@
* @return output of salt command
*/
def enforceHighstate(saltId, target, output = false, failOnError = true, batch = null, saltArgs = []) {
- def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'state.highstate', batch)
+ def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'state.highstate', batch, saltArgs)
def common = new com.mirantis.mk.Common()
common.infoMsg("Running state highstate on ${target}")
@@ -575,12 +605,22 @@
* @param name Name of the VM (for ex. ctl01)
* @return Salt minion ID of KVM node hosting 'name' VM
*/
-def getNodeProvider(saltId, name) {
- def kvm = getKvmMinionId(saltId)
- return getReturnValues(getPillar(saltId, "${kvm}", "salt:control:cluster:internal:node:${name}:provider"))
+def getNodeProvider(saltId, nodeName) {
+ def salt = new com.mirantis.mk.Salt()
+ def common = new com.mirantis.mk.Common()
+ def kvms = salt.getMinions(saltId, 'I@salt:control')
+ for (kvm in kvms) {
+ try {
+ vms = salt.getReturnValues(salt.runSaltProcessStep(saltId, kvm, 'virt.list_domains', [], null, true))
+ if (vms.toString().contains(nodeName)) {
+ return kvm
+ }
+ } catch (Exception er) {
+ common.infoMsg("${nodeName} not present on ${kvm}")
+ }
+ }
}
-
/**
* Test if there are any minions to target
* @param saltId Salt Connection object or pepperEnv (the command will be sent using the selected method)
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 77959db..f0921c4 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -24,7 +24,7 @@
def saltOpts = "--retcode-passthrough --force-color"
def common = new com.mirantis.mk.Common()
def workspace = common.getWorkspace()
- def img = docker.image("tcpcloud/salt-models-testing:latest")
+ def img = docker.image("mirantis/salt:saltstack-ubuntu-xenial-salt-2017.7")
img.pull()
if (!extraFormulas || extraFormulas == "") {
@@ -41,8 +41,8 @@
"DEBUG=1", "MASTER_HOSTNAME=${masterName}", "CLUSTER_NAME=${clusterName}", "MINION_ID=${masterName}",
"RECLASS_VERSION=${reclassVersion}", "RECLASS_IGNORE_CLASS_NOTFOUND=${ignoreClassNotfound}", "APT_REPOSITORY=${aptRepoUrl}",
"APT_REPOSITORY_GPG=${aptRepoGPG}"]){
-
- sh("""cp -r ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
+ sh("git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts")
+ sh("""rsync -ah ${testDir}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;""")
sh("""for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
@@ -51,15 +51,14 @@
done""")
sh("""timeout ${testTimeout} bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && source_local_envs && configure_salt_master && configure_salt_minion && install_salt_formula_pkg'
bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && saltservice_restart'""")
-
sh("timeout ${testTimeout} bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && source_local_envs && saltmaster_init'")
- if (!legacyTestingMode) {
+ if (!legacyTestingMode.toBoolean()) {
sh("bash -c 'source /srv/salt/scripts/bootstrap.sh; cd /srv/salt/scripts && verify_salt_minions'")
}
}
- if (legacyTestingMode) {
+ if (legacyTestingMode.toBoolean()) {
common.infoMsg("Running legacy mode test for master hostname ${masterName}")
def nodes = sh script: "find /srv/salt/reclass/nodes -name '*.yml' | grep -v 'cfg*.yml'", returnStdout: true
for (minion in nodes.tokenize()) {
diff --git a/src/com/mirantis/mk/Virsh.groovy b/src/com/mirantis/mk/Virsh.groovy
index a866fa3..d972d91 100644
--- a/src/com/mirantis/mk/Virsh.groovy
+++ b/src/com/mirantis/mk/Virsh.groovy
@@ -54,17 +54,17 @@
try {
salt.cmdRun(master, "${nodeProvider}*", "virsh snapshot-delete ${target}.${domain} --metadata ${snapshotName}")
} catch (Exception e) {
- common.warningMsg('Snapshot ${snapshotName} for ${target}.${domain} does not exist or failed to be removed')
+ common.warningMsg("Snapshot ${snapshotName} for ${target}.${domain} does not exist or failed to be removed")
}
try {
salt.runSaltProcessStep(master, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.${snapshotName}.qcow2"], null, true)
} catch (Exception e) {
- common.warningMsg('Snapshot ${snapshotName} qcow2 file for ${target}.${domain} does not exist or failed to be removed')
+ common.warningMsg("Snapshot ${snapshotName} qcow2 file for ${target}.${domain} does not exist or failed to be removed")
}
try {
salt.runSaltProcessStep(master, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.xml"], null, true)
} catch (Exception e) {
- common.warningMsg('Dumpxml file for ${target}.${domain} does not exist or failed to be removed')
+ common.warningMsg("Dumpxml file for ${target}.${domain} does not exist or failed to be removed")
}
}
@@ -87,7 +87,7 @@
liveSnapshotAbsent(master, nodeProvider, target, snapshotName, path)
salt.runSaltProcessStep(master, "${nodeProvider}*", 'virt.start', ["${target}.${domain}"], null, true)
} catch (Exception er) {
- common.infoMsg('No rollback for ${target}.${domain} was executed. Dumpxml file not present.')
+ common.infoMsg("No rollback for ${target}.${domain} was executed. Dumpxml file not present.")
}
}
@@ -109,17 +109,17 @@
try {
salt.cmdRun(master, "${nodeProvider}*", "virsh snapshot-delete ${target}.${domain} --metadata ${snapshotName}")
} catch (Exception e) {
- common.warningMsg('Snapshot ${snapshotName} for ${target}.${domain} does not exist or failed to be removed')
+ common.warningMsg("Snapshot ${snapshotName} for ${target}.${domain} does not exist or failed to be removed")
}
try {
salt.runSaltProcessStep(master, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.${snapshotName}.qcow2"], null, true)
} catch (Exception e) {
- common.warningMsg('Snapshot ${snapshotName} qcow2 file for ${target}.${domain} does not exist or failed to be removed')
+ common.warningMsg("Snapshot ${snapshotName} qcow2 file for ${target}.${domain} does not exist or failed to be removed")
}
try {
salt.runSaltProcessStep(master, "${nodeProvider}*", 'file.remove', ["${path}/${target}.${domain}.xml"], null, true)
} catch (Exception e) {
- common.warningMsg('Dumpxml file for ${target}.${domain} does not exist or failed to be removed')
+ common.warningMsg("Dumpxml file for ${target}.${domain} does not exist or failed to be removed")
}
} catch (Exception e) {
common.errorMsg("The live snapshoted VM ${target}.${domain} failed to be merged, trying to fix it")