getting rid of sleeps
Change-Id: I3a387d6e3976befa17b8208d2061ddc3fca1057a
diff --git a/opencontrail-upgrade.groovy b/opencontrail-upgrade.groovy
index 02c2bc1..a01affe 100644
--- a/opencontrail-upgrade.groovy
+++ b/opencontrail-upgrade.groovy
@@ -34,15 +34,20 @@
def CMP_PKGS = 'contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms'
def KERNEL_MODULE_RELOAD = 'service supervisor-vrouter stop;ifdown vhost0;rmmod vrouter;modprobe vrouter;ifup vhost0;service supervisor-vrouter start;'
-
def void runCommonCommands(target, command, args, check, salt, saltMaster, common) {
out = salt.runSaltCommand(saltMaster, 'local', ['expression': target, 'type': 'compound'], command, null, args, null)
salt.printSaltCommandResult(out)
- sleep(60)
- out = salt.runSaltCommand(saltMaster, 'local', ['expression': target, 'type': 'compound'], command, null, check, null)
- salt.printSaltCommandResult(out)
- input message: "Please check the output of \'${check}\' and continue if it is correct."
+ // wait until $check is in correct state
+ if ( check == "nodetool status" ) {
+ salt.commandStatus(saltMaster, target, check, 'Status=Up')
+ } else if ( check == "contrail-status" ) {
+ salt.commandStatus(saltMaster, target, "${check} | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
+ }
+
+ //out = salt.runSaltCommand(saltMaster, 'local', ['expression': target, 'type': 'compound'], command, null, check, null)
+ //salt.printSaltCommandResult(out)
+ //input message: "Please check the output of \'${check}\' and continue if it is correct."
}
timestamps {
@@ -142,21 +147,21 @@
args = 'apt install contrail-database -y;'
check = 'nodetool status'
- // ntw01
+ // nal01
runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
- // ntw02
+ // nal02
runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
- // ntw03
+ // nal03
runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
check = 'contrail-status'
- // ntw01
+ // nal01
runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
- // ntw02
+ // nal02
runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
- // ntw03
+ // nal03
runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
try {
@@ -221,7 +226,9 @@
}
salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
- sleep(10)
+
+ //sleep(10)
+ salt.commandStatus(saltMaster, targetLiveSubset, "${check} | grep -v == | grep -v active", null, false)
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
salt.printSaltCommandResult(out)
@@ -257,7 +264,8 @@
}
salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
- sleep(10)
+ //sleep(10)
+ salt.commandStatus(saltMaster, targetLiveAll, "${check} | grep -v == | grep -v active", null, false)
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
salt.printSaltCommandResult(out)
@@ -349,21 +357,21 @@
args = 'apt install contrail-database -y --force-yes;'
check = 'nodetool status'
- // ntw01
+ // nal01
runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
- // ntw02
+ // nal02
runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
- // ntw03
+ // nal03
runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
check = 'contrail-status'
- // ntw01
+ // nal01
runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, saltMaster, common)
- // ntw02
+ // nal02
runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, saltMaster, common)
- // ntw03
+ // nal03
runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, saltMaster, common)
try {
@@ -428,7 +436,8 @@
}
salt.runSaltProcessStep(saltMaster, targetLiveSubset, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
- sleep(10)
+ //sleep(10)
+ salt.commandStatus(saltMaster, targetLiveSubset, "${check} | grep -v == | grep -v active", null, false)
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveSubset, 'type': 'compound'], command, null, check, null)
salt.printSaltCommandResult(out)
@@ -465,7 +474,9 @@
}
salt.runSaltProcessStep(saltMaster, targetLiveAll, 'cmd.shell', ["${KERNEL_MODULE_RELOAD}"], null, true)
- sleep(10)
+
+ //sleep(10)
+ salt.commandStatus(saltMaster, targetLiveAll, "${check} | grep -v == | grep -v active", null, false)
out = salt.runSaltCommand(saltMaster, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, check, null)
salt.printSaltCommandResult(out)
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index 1dfc4cb..7bf646c 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -81,7 +81,8 @@
// salt 'kvm02*' state.sls salt.control
salt.enforceState(saltMaster, "${upgNodeProvider}", 'salt.control')
- sleep(70)
+ // wait until upg node is registered in salt-key
+ salt.minionPresent(saltMaster, 'I@salt:master', 'upg01')
// salt '*' saltutil.refresh_pillar
salt.runSaltProcessStep(saltMaster, 'upg*', 'saltutil.refresh_pillar', [], null, true)
@@ -288,7 +289,13 @@
// salt 'kvm*' state.sls salt.control
salt.enforceState(saltMaster, 'I@salt:control', 'salt.control')
- sleep(70)
+ // wait until ctl and prx nodes are registered in salt-key
+ salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
+
// salt '*' saltutil.refresh_pillar
salt.runSaltProcessStep(saltMaster, '*', 'saltutil.refresh_pillar', [], null, true)
@@ -410,7 +417,10 @@
salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
- sleep(5)
+
+ // wait until mysql service on galera master is up
+ salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
+
salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
//
@@ -559,7 +569,10 @@
salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
- sleep(5)
+
+ // wait until mysql service on galera master is up
+ salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
+
salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
//
@@ -572,7 +585,12 @@
// salt 'cmp*' cmd.run 'service nova-compute restart'
salt.runSaltProcessStep(saltMaster, 'cmp*', 'service.restart', ['nova-compute'], null, true)
- sleep(70)
+ // wait until ctl and prx nodes are registered in salt-key
+ salt.minionPresent(saltMaster, 'I@salt:master', 'ctl01')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'ctl02')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'ctl03')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'prx01')
+ salt.minionPresent(saltMaster, 'I@salt:master', 'prx02')
salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-conductor'], null, true)
salt.runSaltProcessStep(saltMaster, 'ctl*', 'service.restart', ['nova-scheduler'], null, true)
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index af2016e..4a9c89e 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -57,23 +57,26 @@
salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'service.start', ['supervisor-database'], null, true)
- sleep(30)
+ // wait until supervisor-database service is up
+ salt.commandStatus(saltMaster, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
// performs restore
salt.cmdRun(saltMaster, 'I@cassandra:backup:client', "su root -c 'salt-call state.sls cassandra'")
salt.runSaltProcessStep(saltMaster, 'I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and not I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
- salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control and not I@cassandra:backup:client', 'system.reboot', null, null, true, 5)
- sleep(60)
+ // wait until supervisor-database service is up
+ salt.commandStatus(saltMaster, 'I@cassandra:backup:client', 'service supervisor-database status', 'running')
+ salt.commandStatus(saltMaster, 'I@opencontrail:control and not I@cassandra:backup:client', 'service supervisor-database status', 'running')
+ sleep(5)
+
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.restart', ['supervisor-database'], null, true)
- sleep(50)
+ // wait until contrail-status is up
+ salt.commandStatus(saltMaster, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
+
salt.cmdRun(saltMaster, 'I@opencontrail:control', "nodetool status")
salt.cmdRun(saltMaster, 'I@opencontrail:control', "contrail-status")
}
}
}
-
-
-
diff --git a/restore-zookeeper.groovy b/restore-zookeeper.groovy
index 6b4822a..3e5da6c 100644
--- a/restore-zookeeper.groovy
+++ b/restore-zookeeper.groovy
@@ -42,7 +42,10 @@
} catch (Exception er) {
common.warningMsg('Zookeeper service already stopped')
}
- sleep(5)
+ //sleep(5)
+ // wait until zookeeper service is down
+ salt.commandStatus(saltMaster, 'I@opencontrail:control', 'service zookeeper status', 'stop')
+
try {
salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir -p /root/zookeeper/zookeeper.bak")
} catch (Exception er) {
@@ -73,7 +76,9 @@
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-config'], null, true)
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-control'], null, true)
- sleep(50)
+ // wait until contrail-status is up
+ salt.commandStatus(saltMaster, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
+
salt.cmdRun(saltMaster, 'I@opencontrail:control', "ls /var/lib/zookeeper/version-2")
try {
salt.cmdRun(saltMaster, 'I@opencontrail:control', "echo stat | nc localhost 2181")
diff --git a/xtrabackup-restore-mysql-db.groovy b/xtrabackup-restore-mysql-db.groovy
index c363ce2..345f280 100644
--- a/xtrabackup-restore-mysql-db.groovy
+++ b/xtrabackup-restore-mysql-db.groovy
@@ -70,7 +70,10 @@
salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
salt.cmdRun(saltMaster, 'I@xtrabackup:client', "su root -c 'salt-call state.sls xtrabackup'")
salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'service.start', ['mysql'], null, true)
- sleep(5)
+
+ // wait until mysql service on galera master is up
+ salt.commandStatus(saltMaster, 'I@galera:master', 'service mysql status', 'running')
+
salt.runSaltProcessStep(saltMaster, 'I@galera:slave', 'service.start', ['mysql'], null, true)
sleep(15)
salt.cmdRun(saltMaster, 'I@galera:master', "su root -c 'salt-call mysql.status | grep -A1 wsrep_cluster_size'")