Merge "installInfraKvm fix for slow responding environments."
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 2923682..92d317b 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -94,37 +94,35 @@
def installInfraKvm(master) {
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
- salt.fullRefresh(master, 'I@linux:system')
- def infra_conpund = 'I@salt:control'
+ def infra_compound = 'I@salt:control'
def minions = []
def wait_timeout = 10
def retries = wait_timeout * 30
+ salt.fullRefresh(master, 'I@linux:system')
salt.enforceState(master, 'I@salt:control', ['salt.minion'], true, false, null, false, 60, 2)
salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp', 'rsyslog'])
salt.enforceState(master, 'I@salt:control', 'libvirt')
salt.enforceState(master, 'I@salt:control', 'salt.control')
- timeout(wait_timeout) {
- common.infoMsg("Waiting for minions to come up...")
- if (salt.testTarget(master, infra_conpund)) {
- // Gathering minions
- for ( infra_node in salt.getMinionsSorted(master, infra_conpund) ) {
- def pillar = salt.getPillar(master, infra_node, 'salt:control:cluster')
- if ( !pillar['return'].isEmpty() ) {
- for ( cluster in pillar['return'][0].values() ) {
- def engine = cluster.values()[0]['engine']
- def domain = cluster.values()[0]['domain']
- def node = cluster.values()[0]['node']
- if ( engine == "virt" ) {
- def nodes = node.values()
- if ( !nodes.isEmpty() ) {
- for ( vm in nodes ) {
- if ( vm['name'] != null ) {
- def vm_fqdn = vm['name'] + '.' + domain
- if ( !minions.contains(vm_fqdn) ) {
- minions.add(vm_fqdn)
- }
+ common.infoMsg("Building minions list...")
+ if (salt.testTarget(master, infra_compound)) {
+ // Gathering minions
+ for ( infra_node in salt.getMinionsSorted(master, infra_compound) ) {
+ def pillar = salt.getPillar(master, infra_node, 'salt:control:cluster')
+ if ( !pillar['return'].isEmpty() ) {
+ for ( cluster in pillar['return'][0].values() ) {
+ def engine = cluster.values()[0]['engine']
+ def domain = cluster.values()[0]['domain']
+ def node = cluster.values()[0]['node']
+ if ( engine == "virt" ) {
+ def nodes = node.values()
+ if ( !nodes.isEmpty() ) {
+ for ( vm in nodes ) {
+ if ( vm['name'] != null ) {
+ def vm_fqdn = vm['name'] + '.' + domain
+ if ( !minions.contains(vm_fqdn) ) {
+ minions.add(vm_fqdn)
}
}
}
@@ -133,13 +131,18 @@
}
}
}
+ }
- def minions_compound = minions.join(' or ')
- common.infoMsg('Waiting for next minions to register: ' + minions_compound,)
+ def minions_compound = minions.join(' or ')
+
+ common.infoMsg("Waiting for next minions to register within ${wait_timeout} minutes: " + minions_compound)
+ timeout(time: wait_timeout, unit: 'MINUTES') {
salt.minionsPresentFromList(master, 'I@salt:master', minions, true, null, true, retries, 1)
- common.infoMsg('Waiting for minions to respond')
- salt.minionsReachable(master, 'I@salt:master', minions_compound )
+ }
+ common.infoMsg('Waiting for minions to respond')
+ timeout(time: wait_timeout, unit: 'MINUTES') {
+ salt.minionsReachable(master, 'I@salt:master', minions_compound)
}
common.infoMsg("All minions are up.")
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 0f81287..84659f6 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -269,27 +269,35 @@
if (waitUntilPresent){
def count = 0
while(count < maxRetries) {
+ try {
+ def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
+ if (output) {
+ printSaltCommandResult(out)
+ }
+ def valueMap = out["return"][0]
+ def result = valueMap.get(valueMap.keySet()[0])
+ def resultsArray = result.tokenize("\n")
+ def size = resultsArray.size()
+ if (size >= answers) {
+ return out
+ }
+ count++
+ sleep(time: 1000, unit: 'MILLISECONDS')
+ common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
+ } catch (Exception er) {
+ common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
+ }
+ }
+ } else {
+ try {
def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
if (output) {
printSaltCommandResult(out)
}
- def valueMap = out["return"][0]
- def result = valueMap.get(valueMap.keySet()[0])
- def resultsArray = result.tokenize("\n")
- def size = resultsArray.size()
- if (size >= answers) {
- return out
- }
- count++
- sleep(time: 1000, unit: 'MILLISECONDS')
- common.infoMsg("Waiting for ${cmd} on ${target} to be in correct state")
+ return out
+ } catch (Exception er) {
+ common.infoMsg('[WARNING]: runSaltCommand command read timeout within 5 seconds. You have very slow or broken environment')
}
- } else {
- def out = runSaltCommand(saltId, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', batch, [cmd], null, 5)
- if (output) {
- printSaltCommandResult(out)
- }
- return out
}
// otherwise throw exception
common.errorMsg("Status of command ${cmd} on ${target} failed, please check it.")