restore dbs pipeline fixes
Change-Id: Id749c53c059737fa95d799d774f3fa94b402f084
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index 66e0a41..900b3d8 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -33,27 +33,31 @@
common.warningMsg('Supervisor-database service already stopped')
}
try {
- salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir /root/cassandra/cassandra.bak")
+ salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir -p /root/cassandra/cassandra.bak")
} catch (Exception er) {
common.warningMsg('Directory already exists')
}
- // add check if empty dir ?
try {
salt.cmdRun(saltMaster, 'I@opencontrail:control', "mv /var/lib/cassandra/* /root/cassandra/cassandra.bak")
} catch (Exception er) {
common.warningMsg('Files were already moved')
}
+ try {
+ salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm -rf /var/lib/cassandra/*")
+ } catch (Exception er) {
+ common.warningMsg('Directory already empty')
+ }
_pillar = salt.getPillar(saltMaster, "ntw01*", 'cassandra:backup:backup_dir')
backup_dir = _pillar['return'][0].values()[0]
if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/cassandra' }
print(backup_dir)
- salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
+ salt.runSaltProcessStep(saltMaster, 'ntw01*', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
salt.runSaltProcessStep(saltMaster, 'ntw01*', 'service.start', ['supervisor-database'], null, true)
- sleep(10)
+ sleep(30)
// performs restore
salt.cmdRun(saltMaster, 'ntw01*', "su root -c 'salt-call state.sls cassandra'")
@@ -61,10 +65,10 @@
salt.runSaltProcessStep(saltMaster, 'ntw02*', 'system.reboot', null, null, true, 5)
salt.runSaltProcessStep(saltMaster, 'ntw03*', 'system.reboot', null, null, true, 5)
- sleep(15)
+ sleep(60)
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.restart', ['supervisor-database'], null, true)
- sleep(10)
+ sleep(50)
salt.cmdRun(saltMaster, 'I@opencontrail:control', "nodetool status")
salt.cmdRun(saltMaster, 'I@opencontrail:control', "contrail-status")
}
diff --git a/restore-zookeeper.groovy b/restore-zookeeper.groovy
index 711601e..6b4822a 100644
--- a/restore-zookeeper.groovy
+++ b/restore-zookeeper.groovy
@@ -42,24 +42,29 @@
} catch (Exception er) {
common.warningMsg('Zookeeper service already stopped')
}
+ sleep(5)
try {
- salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir /root/zookeeper/zookeeper.bak")
+ salt.cmdRun(saltMaster, 'I@opencontrail:control', "mkdir -p /root/zookeeper/zookeeper.bak")
} catch (Exception er) {
common.warningMsg('Directory already exists')
}
- // add check if empty dir ?
try {
salt.cmdRun(saltMaster, 'I@opencontrail:control', "mv /var/lib/zookeeper/version-2/* /root/zookeeper/zookeeper.bak")
} catch (Exception er) {
common.warningMsg('Files were already moved')
}
+ try {
+ salt.cmdRun(saltMaster, 'I@opencontrail:control', "rm -rf /var/lib/zookeeper/version-2/*")
+ } catch (Exception er) {
+ common.warningMsg('Directory already empty')
+ }
_pillar = salt.getPillar(saltMaster, "I@opencontrail:control", 'zookeeper:backup:backup_dir')
backup_dir = _pillar['return'][0].values()[0]
if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/zookeeper' }
print(backup_dir)
- salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
+ salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
// performs restore
salt.cmdRun(saltMaster, 'I@opencontrail:control', "su root -c 'salt-call state.sls zookeeper'")
@@ -68,9 +73,13 @@
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-config'], null, true)
salt.runSaltProcessStep(saltMaster, 'I@opencontrail:control', 'service.start', ['supervisor-control'], null, true)
- sleep(15)
+ sleep(50)
salt.cmdRun(saltMaster, 'I@opencontrail:control', "ls /var/lib/zookeeper/version-2")
- salt.cmdRun(saltMaster, 'I@opencontrail:control', "echo stat | nc localhost 2181 | grep leader")
+ try {
+ salt.cmdRun(saltMaster, 'I@opencontrail:control', "echo stat | nc localhost 2181")
+ } catch (Exception er) {
+ common.warningMsg('Check which node is zookeeper leader')
+ }
salt.cmdRun(saltMaster, 'I@opencontrail:control', "contrail-status")
}
}
diff --git a/xtrabackup-restore-mysql-db.groovy b/xtrabackup-restore-mysql-db.groovy
index 6c8b379..c363ce2 100644
--- a/xtrabackup-restore-mysql-db.groovy
+++ b/xtrabackup-restore-mysql-db.groovy
@@ -43,21 +43,21 @@
common.warningMsg('Files are not present')
}
try {
- salt.cmdRun(saltMaster, 'I@galera:master', "mkdir /root/mysql/mysql.bak")
+ salt.cmdRun(saltMaster, 'I@galera:master', "mkdir -p /root/mysql/mysql.bak")
} catch (Exception er) {
common.warningMsg('Directory already exists')
}
try {
- salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /root/mysql/mysql.bak/*")
- } catch (Exception er) {
- common.warningMsg('Directory already empty')
- }
- try {
salt.cmdRun(saltMaster, 'I@galera:master', "mv /var/lib/mysql/* /root/mysql/mysql.bak")
} catch (Exception er) {
common.warningMsg('Files were already moved')
}
try {
+ salt.cmdRun(saltMaster, 'I@galera:master', "rm -rf /var/lib/mysql/*")
+ } catch (Exception er) {
+ common.warningMsg('Directory already empty')
+ }
+ try {
salt.runSaltProcessStep(saltMaster, 'I@galera:master', 'file.remove', ["/var/lib/mysql/.galera_bootstrap"], null, true)
} catch (Exception er) {
common.warningMsg('File is not present')