restore_cassandra: add create backup stage
Changes:
- create backup script called before attempting to
do a restore;
- health_check module used for contrail services check
before and after restore; health check after restore is
retried 5 times with timeout to allow cassandra a
chance to recover;
- wait timeout for healthcheck after restore
increased for OC4.
PROD-22711
Change-Id: I9bfb33a3e6b23ed519d4f4ada25b448bbca2d723
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index c20c3a0..b585e7e 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -19,6 +19,19 @@
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
+ stage('Opencontrail controllers health check') {
+ try {
+ salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'opencontrail.upgrade.verify', true, true)
+ } catch (Exception er) {
+ common.errorMsg("Opencontrail controllers health check stage found issues with services. Please take a look at the logs above.")
+ throw er
+ }
+ }
+
+ stage('Backup') {
+ salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', 'bash /usr/local/bin/cassandra-backup-runner-call.sh')
+ }
+
stage('Restore') {
// get opencontrail version
def _pillar = salt.getPillar(pepperEnv, "I@opencontrail:control", '_param:opencontrail_version')
@@ -55,11 +68,9 @@
// the lovely wait-60-seconds mantra before restarting supervisor-database service
sleep(60)
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "doctrail controller systemctl restart contrail-database")
- // another mantra
+ // another mantra, wait till all services are up
sleep(60)
- salt.cmdRun(pepperEnv, 'I@opencontrail:control', "doctrail controller contrail-status")
- }
- else {
+ } else {
try {
salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.stop', ['neutron-server'], null, true)
} catch (Exception er) {
@@ -125,5 +136,12 @@
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "contrail-status")
}
}
+
+ stage('Opencontrail controllers health check') {
+ common.retry(3, 20){
+ salt.cmdRun(pepperEnv, 'I@opencontrail:control', "doctrail controller contrail-status")
+ salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'opencontrail.upgrade.verify', true, true)
+ }
+ }
}
}