backupninja-backup-pipeline.groovy

def common = new com.mirantis.mk.Common()
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
def askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
def backupSaltMasterAndMaas = (env.getProperty('BACKUP_SALTMASTER_AND_MAAS') ?: true).toBoolean()
def backupDogtag = (env.getProperty('BACKUP_DOGTAG') ?: true).toBoolean()
def backupKeystone = (env.getProperty('BACKUP_KEYSTONE_CREDENTIAL_KEYS') ?: true).toBoolean()
def saltMasterTargetMatcher = "I@backupninja:client and I@salt:master"
def dogtagTagetMatcher = "I@backupninja:client and I@dogtag:server"
def keystoneTargetMatcher = "I@backupninja:client and I@keystone:server"
logBackupSuccess = []
logBackupFailure = []

def checkBackupninjaLog(output, backupName='', terminateOnFailure=true) {
    def common = new com.mirantis.mk.Common()
    def outputPattern = java.util.regex.Pattern.compile("\\d+")
    def outputMatcher = outputPattern.matcher(output)
    if (outputMatcher.find()) {
        try {
            result = outputMatcher.getAt([0, 1, 2, 3])
            if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)) {
                common.successMsg("[${backupName}] - Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
                logBackupSuccess.add(backupName)
            } else {
                common.errorMsg("[${backupName}] - Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
                logBackupFailure.add(backupName)
            }
        }
        catch (Exception e) {
            common.errorMsg(e.getMessage())
            common.errorMsg("[${backupName}] - Backupninja log parsing failed.")
            logBackupFailure.add(backupName)
        }
    }
}

timeout(time: 12, unit: 'HOURS') {
    node() {
        def saltMasterBackupNode = ''
        def dogtagBackupNode = ''
        def keystoneBackupNode = ''
        def backupServer = ''
        stage('Setup virtualenv for Pepper') {
            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
        }
        stage('Verify pillar for backups') {
            if (backupSaltMasterAndMaas) {
                try {
                    def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data').get('return')[0].values()[0]
                    if (masterPillar.isEmpty()) {
                        throw new Exception("Problem with salt-master pillar on 'I@salt:master' node.")
                    }
                    def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data').get('return')[0].values()[0]
                    if (minionPillar.isEmpty()) {
                        throw new Exception("Problem with salt-minion pillar on I@salt:master node.")
                    }
                }
                catch (Exception e) {
                    common.errorMsg(e.getMessage())
                    common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master.html')
                    throw e
                }
            }
            if (backupDogtag) {
                def barbicanBackendPresent = salt.getPillar(pepperEnv, "I@salt:master", "_param:barbican_backend").get('return')[0].values()[0]
                if (barbicanBackendPresent == 'dogtag') {
                    try {
                        def dogtagPillar = salt.getPillar(pepperEnv, "I@dogtag:server", "dogtag:server").get('return')[0].values()[0]
                        if (dogtagPillar.isEmpty()) {
                            throw new Exception("Problem with dogtag pillar on I@dogtag:server node.")
                        }
                    }
                    catch (Exception e) {
                        common.errorMsg(e.getMessage())
                        common.errorMsg("Looks like dogtag pillar is not defined. Fix your pillar or disable dogtag backup by setting the BACKUP_DOGTAG parameter to False if you're using different barbican backend.")
                        throw e
                    }
                } else {
                    backupDogtag = false
                    common.warningMsg('Backup for Dogtag is enabled, but service itself is not present. Skipping...')
                }
            }
        }
        stage('Check backup location') {
            if (backupSaltMasterAndMaas) {
                try {
                    saltMasterBackupNode = salt.getMinionsSorted(pepperEnv, saltMasterTargetMatcher)[0]
                    salt.minionsReachable(pepperEnv, "I@salt:master", saltMasterBackupNode)
                }
                catch (Exception e) {
                    common.errorMsg(e.getMessage())
                    common.errorMsg("Pipeline wasn't able to detect backupninja:client pillar on Salt master node or the minion is not reachable")
                    currentBuild.result = "FAILURE"
                    throw e
                }

                def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
                if (!maasNodes.isEmpty()) {
                    def postgresqlMajorVersion = salt.getPillar(pepperEnv, 'I@salt:master', '_param:postgresql_major_version').get('return')[0].values()[0]
                    if (! postgresqlMajorVersion) {
                            common.errorMsg("Can't get _param:postgresql_major_version parameter, which is required to determine postgresql-client version. Is it defined in pillar?")
                            if (askConfirmation) {
                                input message: "Confirm to proceed anyway."
                            }
                    } else {
                        def postgresqlClientPackages = "postgresql-client-${postgresqlMajorVersion}"
                        try {
                            if (!salt.isPackageInstalled(['saltId': pepperEnv, 'target': saltMasterBackupNode, 'packageName': postgresqlClientPackages, 'output': false])) {
                                if (askConfirmation) {
                                    input message: "Do you want to install ${postgresqlClientPackages} package on targeted nodes: ${saltMasterBackupNode}? It's required to make backup. Click to confirm."
                                } else {
                                    common.infoMsg("Package ${postgresqlClientPackages} will be installed. It's required to make backup.")
                                }
                                // update also common fake package
                                salt.runSaltProcessStep(pepperEnv, saltMasterBackupNode, 'pkg.install', ["postgresql-client,${postgresqlClientPackages}"])
                            }
                        } catch (Exception e) {
                            common.errorMsg("Unable to determine status of ${postgresqlClientPackages} packages on target nodes: ${saltMasterBackupNode}.")
                            if (askConfirmation) {
                                input message: "Do you want to continue? Click to confirm"
                            }
                        }
                    }
                }
            }
            if (backupDogtag) {
                try {
                    dogtagBackupNode = salt.getMinionsSorted(pepperEnv, dogtagTagetMatcher)[0]
                    salt.minionsReachable(pepperEnv, "I@salt:master", dogtagBackupNode)
                }
                catch (Exception e) {
                    common.errorMsg(e.getMessage())
                    common.errorMsg("Pipeline wasn't able to detect node with backupninja:client and dogtag:server pillars defined or the minion is not reachable")
                    currentBuild.result = "FAILURE"
                    throw e
                }
            }

            try {
                backupServer = salt.getMinions(pepperEnv, "I@backupninja:server")[0]
                salt.minionsReachable(pepperEnv, "I@salt:master", backupServer)
            }
            catch (Exception e) {
                common.errorMsg(e.getMessage())
                common.errorMsg("Pipeline wasn't able to detect backupninja:server pillar or the minion is not reachable")
                currentBuild.result = "FAILURE"
                throw e
            }
            if (backupKeystone) {
                try {
                    keystoneBackupNode = salt.getMinionsSorted(pepperEnv, keystoneTargetMatcher)[0]
                    salt.minionsReachable(pepperEnv, "I@salt:master", keystoneBackupNode)
                }
                catch (Exception e) {
                    common.errorMsg(e.getMessage())
                    common.errorMsg("Pipeline wasn't able to detect node with backupninja:client and keystone:server pillars defined or the minion is not reachable")
                    currentBuild.result = "FAILURE"
                    throw e
                }
            }

            try {
                backupServer = salt.getMinions(pepperEnv, "I@backupninja:server")[0]
                salt.minionsReachable(pepperEnv, "I@salt:master", backupServer)
            }
            catch (Exception e) {
                common.errorMsg(e.getMessage())
                common.errorMsg("Pipeline wasn't able to detect backupninja:server pillar or the minion is not reachable")
                currentBuild.result = "FAILURE"
                throw e
            }
        }
        stage('Prepare for backup') {
            if (backupSaltMasterAndMaas) {
                salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
                salt.enforceState(['saltId': pepperEnv, 'target': saltMasterTargetMatcher, 'state': 'backupninja'])
                def backupMasterSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:master:initial_data:source'))
                def backupMinionSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:minion:initial_data:source'))
                // TODO: Remove ssh-keyscan once we have openssh meta for backupninja implemented
                [backupServer, backupMasterSource, backupMinionSource].unique().each {
                    salt.cmdRun(pepperEnv, saltMasterBackupNode, "ssh-keygen -F ${it} || ssh-keyscan -H ${it} >> /root/.ssh/known_hosts")
                }
                def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
                if (!maasNodes.isEmpty()) {
                    common.infoMsg("Trying to save maas file permissions on ${maasNodes} if possible")
                    salt.cmdRun(pepperEnv, 'I@maas:region', 'which getfacl && ' +
                            'getfacl -pR /var/lib/maas/ > /var/lib/maas/file_permissions.txt &&' +
                            'getfacl -pR /etc/maas/ > /etc/maas/file_permissions.txt || true')
                }
            }
            if (backupDogtag) {
                salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
                salt.enforceState(['saltId': pepperEnv, 'target': dogtagTagetMatcher, 'state': 'backupninja'])
            }
            if (backupKeystone) {
                salt.enforceState(['saltId': pepperEnv, 'target': keystoneTargetMatcher, 'state': 'backupninja'])
                salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
            }
        }
        stage('Backup') {
            if (backupSaltMasterAndMaas) {
                def output = salt.getReturnValues(salt.cmdRun(pepperEnv, saltMasterBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
                checkBackupninjaLog(output, "Salt Master/MAAS")
            }
            if (backupDogtag) {
                def output = salt.getReturnValues(salt.cmdRun(pepperEnv, dogtagBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
                checkBackupninjaLog(output, "Dogtag")
            }
            if (backupKeystone) {
                def output = salt.getReturnValues(salt.cmdRun(pepperEnv, keystoneBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
                checkBackupninjaLog(output, "Keystone")
            }
        }
        stage('Results') {
            if (logBackupSuccess.size() > 0) {
                common.infoMsg("Following backups finished successfully: ${logBackupSuccess.join(",")}")
            }
            if (logBackupFailure.size() > 0) {
                common.errorMsg("Following backups has failed: ${logBackupFailure.join(",")}. Make sure to check the logs.")
                currentBuild.result = "FAILURE"
            }
        }
    }
}