Several fixes for backupninja pipelines
* Do not fail if maas is not enabled
* Add backup server to known_hosts, so scp will not fail
* Additional checks for pillar data
PROD-30107
PROD-30108
Change-Id: Id5043717220de80c25f9d80761c1ddfa0910b083
diff --git a/backupninja-backup-pipeline.groovy b/backupninja-backup-pipeline.groovy
index 80467d4..e707814 100644
--- a/backupninja-backup-pipeline.groovy
+++ b/backupninja-backup-pipeline.groovy
@@ -8,10 +8,27 @@
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
+ stage('Verify pillar for backups') {
+ try {
+ def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
+ if (masterPillar['return'].isEmpty()) {
+ throw new Exception('Problem with salt-master pillar.')
+ }
+ def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
+ if (minionPillar['return'].isEmpty()) {
+ throw new Exception('Problem with salt-minion pillar.')
+ }
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master.html')
+ return
+ }
+ }
stage('Check backup location') {
- try{
- backupNode = salt.getMinions(pepperEnv, "I@backupninja:client")[0]
- salt.minionsReachable(pepperEnv, "I@salt:master", backupNode)
+ try {
+ backupNode = salt.getMinions(pepperEnv, "I@backupninja:client")[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", backupNode)
}
catch (Exception e) {
common.errorMsg(e.getMessage())
@@ -19,9 +36,9 @@
currentBuild.result = "FAILURE"
return
}
- try{
- backupServer = salt.getMinions(pepperEnv, "I@backupninja:server")[0]
- salt.minionsReachable(pepperEnv, "I@salt:master", backupServer)
+ try {
+ backupServer = salt.getMinions(pepperEnv, "I@backupninja:server")[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", backupServer)
}
catch (Exception e) {
common.errorMsg(e.getMessage())
@@ -30,30 +47,34 @@
return
}
}
- stage ('Prepare for backup') {
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:client', 'state': 'backupninja'])
+ stage('Prepare for backup') {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:client', 'state': 'backupninja'])
+ def backupMasterSource = salt.getReturnValues(salt.getPillar(pepperEnv, backupNode, 'salt:master:initial_data:source'))
+ def backupMinionSource = salt.getReturnValues(salt.getPillar(pepperEnv, backupNode, 'salt:minion:initial_data:source'))
+ [backupServer, backupMasterSource, backupMinionSource].unique().each {
+ salt.cmdRun(pepperEnv, backupNode, "ssh-keygen -F ${it} || ssh-keyscan -H ${it} >> /root/.ssh/known_hosts")
+ }
}
stage('Backup') {
def output = salt.getReturnValues(salt.cmdRun(pepperEnv, backupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
def outputPattern = java.util.regex.Pattern.compile("\\d+")
def outputMatcher = outputPattern.matcher(output)
- if (outputMatcher.find()) {
- try{
- result = outputMatcher.getAt([0,1,2,3])
- }
- catch (Exception e){
+ if (outputMatcher.find()) {
+ try {
+ result = outputMatcher.getAt([0, 1, 2, 3])
+ }
+ catch (Exception e) {
common.errorMsg(e.getMessage())
common.errorMsg("Parsing failed.")
currentBuild.result = "FAILURE"
return
- }
+ }
}
- if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)){
- common.successMsg("Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] +" warnings.")
- }
- else {
- common.errorMsg("Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] +" warnings.")
+ if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)) {
+ common.successMsg("Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
+ } else {
+ common.errorMsg("Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
currentBuild.result = "FAILURE"
return
}
diff --git a/backupninja-restore-pipeline.groovy b/backupninja-restore-pipeline.groovy
index b38cd6a..e412c96 100644
--- a/backupninja-restore-pipeline.groovy
+++ b/backupninja-restore-pipeline.groovy
@@ -2,6 +2,7 @@
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
+def maasNodes
timeout(time: 12, unit: 'HOURS') {
node() {
@@ -11,7 +12,7 @@
stage('Salt-Master restore') {
common.infoMsg('Verify pillar for salt-master backups')
try {
- def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
+ def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
if(masterPillar['return'].isEmpty()) {
throw new Exception('Problem with salt-master pillar.')
}
@@ -25,6 +26,7 @@
common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master/salt-master-restore.html')
return
}
+ maasNodes = salt.getMinions(pepperEnv, 'I@maas:server')
common.infoMsg('Performing restore')
salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.master.restore'])
salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.minion.restore'])
@@ -36,20 +38,22 @@
common.infoMsg('Salt-master CA')
salt.cmdRun(pepperEnv, 'I@salt:master', "ls -la /etc/pki/ca/salt_master_ca/")
}
- stage('MAAS Restore') {
- common.infoMsg('Verify pillar for MaaS backup')
- try {
- def maaSPillar = salt.getPillar(pepperEnv, "I@maas:server", 'maas:region:database:initial_data')
- if(maaSPillar['return'].isEmpty()) {
- throw new Exception('Problem with MaaS pillar.')
+ if (!maasNodes.isEmpty()) {
+ stage('MAAS Restore') {
+ common.infoMsg('Verify pillar for MaaS backup')
+ try {
+ def maaSPillar = salt.getPillar(pepperEnv, "I@maas:server", 'maas:region:database:initial_data')
+ if (maaSPillar['return'].isEmpty()) {
+ throw new Exception('Problem with MaaS pillar.')
+ }
}
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/backupninja-postgresql/backupninja-postgresql-restore.html')
+ return
+ }
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@maas:region', 'state': 'maas.region'])
}
- catch (Exception e){
- common.errorMsg(e.getMessage())
- common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/backupninja-postgresql/backupninja-postgresql-restore.html')
- return
- }
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@maas:region', 'state': 'maas.region'])
}
}
}