installInfraKvm fix for slow responding environments.
Issue:
- Minions are registered to salt msater however read timeout
occurs during saltcmdrun check which causes whole job to fail.
This happens only on slow responding environments.
Fix:
- Add try catch loop to process read timeouts as noncritical fault
Addons:
- Function logic has been changed to get "build minions list loop"
out of timeout section.
- Timeout call now has expilit unit set: "MINUTES"
Change-Id: Ia3f6ea027594e8427778d877d36f48d9b466ae80
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 68fb0dd..3a371ef 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -94,37 +94,35 @@
def installInfraKvm(master) {
def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
- salt.fullRefresh(master, 'I@linux:system')
- def infra_conpund = 'I@salt:control'
+ def infra_compound = 'I@salt:control'
def minions = []
def wait_timeout = 10
def retries = wait_timeout * 30
+ salt.fullRefresh(master, 'I@linux:system')
salt.enforceState(master, 'I@salt:control', ['salt.minion'], true, false, null, false, 60, 2)
salt.enforceState(master, 'I@salt:control', ['linux.system', 'linux.network', 'ntp', 'rsyslog'])
salt.enforceState(master, 'I@salt:control', 'libvirt')
salt.enforceState(master, 'I@salt:control', 'salt.control')
- timeout(wait_timeout) {
- common.infoMsg("Waiting for minions to come up...")
- if (salt.testTarget(master, infra_conpund)) {
- // Gathering minions
- for ( infra_node in salt.getMinionsSorted(master, infra_conpund) ) {
- def pillar = salt.getPillar(master, infra_node, 'salt:control:cluster')
- if ( !pillar['return'].isEmpty() ) {
- for ( cluster in pillar['return'][0].values() ) {
- def engine = cluster.values()[0]['engine']
- def domain = cluster.values()[0]['domain']
- def node = cluster.values()[0]['node']
- if ( engine == "virt" ) {
- def nodes = node.values()
- if ( !nodes.isEmpty() ) {
- for ( vm in nodes ) {
- if ( vm['name'] != null ) {
- def vm_fqdn = vm['name'] + '.' + domain
- if ( !minions.contains(vm_fqdn) ) {
- minions.add(vm_fqdn)
- }
+ common.infoMsg("Building minions list...")
+ if (salt.testTarget(master, infra_compound)) {
+ // Gathering minions
+ for ( infra_node in salt.getMinionsSorted(master, infra_compound) ) {
+ def pillar = salt.getPillar(master, infra_node, 'salt:control:cluster')
+ if ( !pillar['return'].isEmpty() ) {
+ for ( cluster in pillar['return'][0].values() ) {
+ def engine = cluster.values()[0]['engine']
+ def domain = cluster.values()[0]['domain']
+ def node = cluster.values()[0]['node']
+ if ( engine == "virt" ) {
+ def nodes = node.values()
+ if ( !nodes.isEmpty() ) {
+ for ( vm in nodes ) {
+ if ( vm['name'] != null ) {
+ def vm_fqdn = vm['name'] + '.' + domain
+ if ( !minions.contains(vm_fqdn) ) {
+ minions.add(vm_fqdn)
}
}
}
@@ -133,13 +131,18 @@
}
}
}
+ }
- def minions_compound = minions.join(' or ')
- common.infoMsg('Waiting for next minions to register: ' + minions_compound,)
+ def minions_compound = minions.join(' or ')
+
+ common.infoMsg("Waiting for next minions to register within ${wait_timeout} minutes: " + minions_compound)
+ timeout(time: wait_timeout, unit: 'MINUTES') {
salt.minionsPresentFromList(master, 'I@salt:master', minions, true, null, true, retries, 1)
- common.infoMsg('Waiting for minions to respond')
- salt.minionsReachable(master, 'I@salt:master', minions_compound )
+ }
+ common.infoMsg('Waiting for minions to respond')
+ timeout(time: wait_timeout, unit: 'MINUTES') {
+ salt.minionsReachable(master, 'I@salt:master', minions_compound)
}
common.infoMsg("All minions are up.")