Merge "remove ceph osd pipeline - fix partition removal while using nvme drives" into release/proposed/2019.2.0
diff --git a/backupninja-backup-pipeline.groovy b/backupninja-backup-pipeline.groovy
index 4410ea9..cbb64ca 100644
--- a/backupninja-backup-pipeline.groovy
+++ b/backupninja-backup-pipeline.groovy
@@ -3,61 +3,128 @@
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
def askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
+def backupSaltMasterAndMaas = (env.getProperty('BACKUP_SALTMASTER_AND_MAAS') ?: true).toBoolean()
+def backupDogtag = (env.getProperty('BACKUP_DOGTAG') ?: true).toBoolean()
+def saltMasterTargetMatcher = "I@backupninja:client and I@salt:master"
+def dogtagTagetMatcher = "I@backupninja:client and I@dogtag:server"
+logBackupSuccess = []
+logBackupFailure = []
+
+def checkBackupninjaLog(output, backupName='', terminateOnFailure=true) {
+ def common = new com.mirantis.mk.Common()
+ def outputPattern = java.util.regex.Pattern.compile("\\d+")
+ def outputMatcher = outputPattern.matcher(output)
+ if (outputMatcher.find()) {
+ try {
+ result = outputMatcher.getAt([0, 1, 2, 3])
+ if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)) {
+ common.successMsg("[${backupName}] - Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
+ logBackupSuccess.add(backupName)
+ } else {
+ common.errorMsg("[${backupName}] - Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
+ logBackupFailure.add(backupName)
+ }
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("[${backupName}] - Backupninja log parsing failed.")
+ logBackupFailure.add(backupName)
+ }
+ }
+}
timeout(time: 12, unit: 'HOURS') {
node() {
- def backupNode = ''
+ def saltMasterBackupNode = ''
+ def dogtagBackupNode = ''
def backupServer = ''
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
stage('Verify pillar for backups') {
- try {
- def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
- if (masterPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-master pillar.')
+ if (backupSaltMasterAndMaas) {
+ try {
+ def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
+ if (masterPillar['return'].isEmpty()) {
+ throw new Exception("Problem with salt-master pillar on 'I@salt:master' node.")
+ }
+ def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
+ if (minionPillar['return'].isEmpty()) {
+ throw new Exception("Problem with salt-minion pillar on I@salt:master node.")
+ }
}
- def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
- if (minionPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-minion pillar.')
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master.html')
+ return
}
}
- catch (Exception e) {
- common.errorMsg(e.getMessage())
- common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master.html')
- return
+ if (backupDogtag) {
+ try {
+ def dogtagPillar = salt.getPillar(pepperEnv, "I@salt:master", "dogtag:server")
+ if (dogtagPillar['return'].isEmpty()) {
+ throw new Exception("Problem with dogtag pillar on I@dogtag:server node.")
+ }
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Looks like dogtag pillar is not defined. Fix your pillar or disable dogtag backup by setting the BACKUP_DOGTAG parameter to False if you're using different barbican backend.")
+ return
+ }
}
}
stage('Check backup location') {
- try {
- backupNode = salt.getMinions(pepperEnv, "I@backupninja:client")[0]
- salt.minionsReachable(pepperEnv, "I@salt:master", backupNode)
- }
- catch (Exception e) {
- common.errorMsg(e.getMessage())
- common.errorMsg("Pipeline wasn't able to detect backupninja:client pillar or the minion is not reachable")
- currentBuild.result = "FAILURE"
- return
- }
-
- def postgresqlMajorVersion = salt.getPillar(pepperEnv, 'I@salt:master', '_param:postgresql_major_version').get('return')[0].values()[0]
- if (! postgresqlMajorVersion) {
- input message: "Can't get _param:postgresql_major_version parameter, which is required to determine postgresql-client version. Is it defined in pillar? Confirm to proceed anyway."
- } else {
- def postgresqlClientPackage = "postgresql-client-${postgresqlMajorVersion}"
+ if (backupSaltMasterAndMaas) {
try {
- if (!salt.isPackageInstalled(['saltId': pepperEnv, 'target': backupNode, 'packageName': postgresqlClientPackage, 'output': false])) {
- if (askConfirmation) {
- input message: "Do you want to install ${postgresqlClientPackage} package on targeted nodes: ${backupNode}? It's required to make backup. Click to confirm"
+ saltMasterBackupNode = salt.getMinionsSorted(pepperEnv, saltMasterTargetMatcher)[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", saltMasterBackupNode)
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Pipeline wasn't able to detect backupninja:client pillar on Salt master node or the minion is not reachable")
+ currentBuild.result = "FAILURE"
+ return
+ }
+
+ def maasNodes = salt.getMinions(pepperEnv, 'I@maas:server')
+ if (!maasNodes.isEmpty()) {
+ def postgresqlMajorVersion = salt.getPillar(pepperEnv, 'I@salt:master', '_param:postgresql_major_version').get('return')[0].values()[0]
+ if (! postgresqlMajorVersion) {
+ common.errorMsg("Can't get _param:postgresql_major_version parameter, which is required to determine postgresql-client version. Is it defined in pillar?")
+ if (askConfirmation) {
+ input message: "Confirm to proceed anyway."
+ }
+ } else {
+ def postgresqlClientPackage = "postgresql-client-${postgresqlMajorVersion}"
+ try {
+ if (!salt.isPackageInstalled(['saltId': pepperEnv, 'target': saltMasterBackupNode, 'packageName': postgresqlClientPackage, 'output': false])) {
+ if (askConfirmation) {
+ input message: "Do you want to install ${postgresqlClientPackages} package on targeted nodes: ${saltMasterBackupNode}? It's required to make backup. Click to confirm."
+ } else {
+ common.infoMsg("Package ${postgresqlClientPackages} will be installed. It's required to make backup.")
+ }
+ // update also common fake package
+ salt.runSaltProcessStep(pepperEnv, saltMasterBackupNode, 'pkg.install', ["postgresql-client,${postgresqlClientPackage}"])
+ }
+ } catch (Exception e) {
+ common.errorMsg("Unable to determine status of ${postgresqlClientPackages} packages on target nodes: ${saltMasterBackupNode}.")
+ if (askConfirmation) {
+ input message: "Do you want to continue? Click to confirm"
+ }
}
- // update also common fake package
- salt.runSaltProcessStep(pepperEnv, backupNode, 'pkg.install', ["postgresql-client,${postgresqlClientPackage}"])
}
- } catch (Exception e) {
- common.errorMsg("Unable to determine status of ${postgresqlClientPackage} packages on target nodes: ${backupNode}.")
- if (askConfirmation) {
- input message: "Do you want to continue? Click to confirm"
- }
+ }
+ }
+ if (backupDogtag) {
+ try {
+ dogtagBackupNode = salt.getMinionsSorted(pepperEnv, dogtagTagetMatcher)[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", dogtagBackupNode)
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Pipeline wasn't able to detect node with backupninja:client and dogtag:server pillars defined or the minion is not reachable")
+ currentBuild.result = "FAILURE"
+ return
}
}
@@ -73,41 +140,44 @@
}
}
stage('Prepare for backup') {
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:client', 'state': 'backupninja'])
- def backupMasterSource = salt.getReturnValues(salt.getPillar(pepperEnv, backupNode, 'salt:master:initial_data:source'))
- def backupMinionSource = salt.getReturnValues(salt.getPillar(pepperEnv, backupNode, 'salt:minion:initial_data:source'))
- [backupServer, backupMasterSource, backupMinionSource].unique().each {
- salt.cmdRun(pepperEnv, backupNode, "ssh-keygen -F ${it} || ssh-keyscan -H ${it} >> /root/.ssh/known_hosts")
+ if (backupSaltMasterAndMaas) {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
+ salt.enforceState(['saltId': pepperEnv, 'target': saltMasterTargetMatcher, 'state': 'backupninja'])
+ def backupMasterSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:master:initial_data:source'))
+ def backupMinionSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:minion:initial_data:source'))
+ // TODO: Remove ssh-keyscan once we have openssh meta for backupninja implemented
+ [backupServer, backupMasterSource, backupMinionSource].unique().each {
+ salt.cmdRun(pepperEnv, saltMasterBackupNode, "ssh-keygen -F ${it} || ssh-keyscan -H ${it} >> /root/.ssh/known_hosts")
+ }
+ def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
+ if (!maasNodes.isEmpty()) {
+ common.infoMsg("Trying to save maas file permissions on ${maasNodes} if possible")
+ salt.cmdRun(pepperEnv, 'I@maas:region', 'which getfacl && getfacl -pR /var/lib/maas/ > /var/lib/maas/file_permissions.txt || true')
+ }
}
- def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
- if (!maasNodes.isEmpty()) {
- common.infoMsg("Trying to save maas file permissions on ${maasNodes} if possible")
- salt.cmdRun(pepperEnv, 'I@maas:region', 'which getfacl && getfacl -pR /var/lib/maas/ > /var/lib/maas/file_permissions.txt || true')
+ if (backupDogtag) {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
+ salt.enforceState(['saltId': pepperEnv, 'target': dogtagTagetMatcher, 'state': 'backupninja'])
}
}
stage('Backup') {
- def output = salt.getReturnValues(salt.cmdRun(pepperEnv, backupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
- def outputPattern = java.util.regex.Pattern.compile("\\d+")
- def outputMatcher = outputPattern.matcher(output)
- if (outputMatcher.find()) {
- try {
- result = outputMatcher.getAt([0, 1, 2, 3])
- }
- catch (Exception e) {
- common.errorMsg(e.getMessage())
- common.errorMsg("Parsing failed.")
- currentBuild.result = "FAILURE"
- return
- }
+ if (backupSaltMasterAndMaas) {
+ def output = salt.getReturnValues(salt.cmdRun(pepperEnv, saltMasterBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
+ checkBackupninjaLog(output, "Salt Master/MAAS")
}
- if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)) {
- common.successMsg("Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
- } else {
- common.errorMsg("Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
+ if (backupDogtag) {
+ def output = salt.getReturnValues(salt.cmdRun(pepperEnv, dogtagBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
+ checkBackupninjaLog(output, "Dogtag")
+ }
+ }
+ stage('Results') {
+ if (logBackupSuccess.size() > 0) {
+ common.infoMsg("Following backups finished successfully: ${logBackupSuccess.join(",")}")
+ }
+ if (logBackupFailure.size() > 0) {
+ common.errorMsg("Following backups has failed: ${logBackupFailure.join(",")}. Make sure to check the logs.")
currentBuild.result = "FAILURE"
- return
}
}
}
-}
+}
\ No newline at end of file
diff --git a/backupninja-restore-pipeline.groovy b/backupninja-restore-pipeline.groovy
index b58756e..a869fe3 100644
--- a/backupninja-restore-pipeline.groovy
+++ b/backupninja-restore-pipeline.groovy
@@ -2,57 +2,98 @@
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
-def maasNodes
+def maasNodes = {}
+def restoreSaltMasterAndMaas = (env.getProperty('RESTORE_SALTMASTER_AND_MAAS') ?: true).toBoolean()
+def restoreDogtag = (env.getProperty('RESTORE_DOGTAG') ?: true).toBoolean()
timeout(time: 12, unit: 'HOURS') {
node() {
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
- stage('Salt-Master restore') {
- common.infoMsg('Verify pillar for salt-master backups')
- try {
- def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
- if(masterPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-master pillar.')
+ stage('Verify pillar for restore') {
+ if (restoreSaltMasterAndMaas) {
+ try {
+ def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
+ if(masterPillar['return'].isEmpty()) {
+ throw new Exception("Problem with salt-master pillar on 'I@salt:master' node.")
+ }
+ def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
+ if(minionPillar['return'].isEmpty()) {
+ throw new Exception("Problem with salt-minion pillar on 'I@salt:master' node.")
+ }
}
- def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
- if(minionPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-minion pillar.')
+ catch (Exception e){
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master/salt-master-restore.html')
+ return
}
+ maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
}
- catch (Exception e){
- common.errorMsg(e.getMessage())
- common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master/salt-master-restore.html')
- return
- }
- maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
- common.infoMsg('Performing restore')
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.master.restore'])
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.minion.restore'])
- salt.fullRefresh(pepperEnv, '*')
-
- common.infoMsg('Validating output')
- common.infoMsg('Salt-Keys')
- salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key")
- common.infoMsg('Salt-master CA')
- salt.cmdRun(pepperEnv, 'I@salt:master', "ls -la /etc/pki/ca/salt_master_ca/")
- }
- if (!maasNodes.isEmpty()) {
- stage('MAAS Restore') {
- common.infoMsg('Verify pillar for MaaS backup')
+ if (!maasNodes.isEmpty()) {
try {
def maaSPillar = salt.getPillar(pepperEnv, "I@maas:region", 'maas:region:database:initial_data')
if (maaSPillar['return'].isEmpty()) {
- throw new Exception('Problem with MaaS pillar.')
+ throw new Exception("Problem with MaaS pillar on 'I@maas:region' node.")
}
}
catch (Exception e) {
common.errorMsg(e.getMessage())
- common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/backupninja-postgresql/backupninja-postgresql-restore.html')
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/maas-postgresql/backupninja-postgresql-restore.html')
return
}
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@maas:region', 'state': 'maas.region'])
+ } else {
+ common.warningMsg("No MaaS Pillar was found. You can ignore this if it's expected. Otherwise you should fix you pillar. Check: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/maas-postgresql/backupninja-postgresql-restore.html")
+ }
+ if (restoreDogtag) {
+ try {
+ def dogtagPillar = salt.getPillar(pepperEnv, "I@dogtag:server:role:master", 'dogtag:server:initial_data')
+ if (dogtagPillar['return'].isEmpty()) {
+ throw new Exception("Problem with Dogtag pillar on 'I@dogtag:server:role:master' node.")
+ }
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/dogtag/restore-dogtag.html')
+ return
+ }
+ }
+ }
+ stage('Restore') {
+ if (restoreSaltMasterAndMaas) {
+ common.infoMsg('Starting salt-master restore')
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.master.restore'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.minion.restore'])
+ salt.fullRefresh(pepperEnv, '*')
+ common.infoMsg('Validating output')
+ common.infoMsg('Salt-Keys')
+ salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key")
+ common.infoMsg('Salt-master CA')
+ salt.cmdRun(pepperEnv, 'I@salt:master', "ls -la /etc/pki/ca/salt_master_ca/")
+ if (!maasNodes.isEmpty()) {
+ common.infoMsg('Starting MaaS restore')
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@maas:region', 'state': 'maas.region'])
+ }
+ }
+ if (restoreDogtag) {
+ salt.runSaltProcessStep(pepperEnv, 'I@dogtag:server:role:slave', 'service.stop', ['dirsrv@pki-tomcat.service'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@dogtag:server:role:master', 'state': 'dogtag.server.restore'])
+ salt.runSaltProcessStep(pepperEnv, 'I@dogtag:server:role:slave', 'service.start', ['dirsrv@pki-tomcat.service'])
+ }
+ }
+ stage('After restore steps') {
+ if (restoreSaltMasterAndMaas) {
+ common.infoMsg("No more steps for Salt Master and MaaS restore are required.")
+ }
+ if (restoreDogtag) {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': ['salt', 'reclass']])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@dogtag:server:role:master', 'state': 'dogtag.server'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@dogtag:server', 'state': 'dogtag.server'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@haproxy:proxy', 'state': 'haproxy'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@barbican:server:role:primary', 'state': 'barbican.server'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@barbican:server', 'state': 'barbican.server'])
+ salt.cmdRun(pepperEnv, 'I@barbican:server', 'rm /etc/barbican/alias/*')
+ salt.runSaltProcessStep(pepperEnv, 'I@barbican:server', 'service.restart', 'apache2')
}
}
}
diff --git a/ceph-upgrade.groovy b/ceph-upgrade.groovy
index c4881bc..dd75875 100644
--- a/ceph-upgrade.groovy
+++ b/ceph-upgrade.groovy
@@ -33,20 +33,27 @@
return salt.cmdRun(master, target, cmd)
}
-def waitForHealthy(master, count=0, attempts=300) {
+def waitForHealthy(master, flags, count=0, attempts=300) {
// wait for healthy cluster
while (count<attempts) {
def health = runCephCommand(master, ADMIN_HOST, 'ceph health')['return'][0].values()[0]
if (health.contains('HEALTH_OK')) {
common.infoMsg('Cluster is healthy')
break;
+ } else {
+ for (flag in flags) {
+ if (health.contains(flag + ' flag(s) set') && !(health.contains('down'))) {
+ common.infoMsg('Cluster is healthy')
+ return;
+ }
+ }
}
count++
sleep(10)
}
}
-def backup(master, target) {
+def backup(master, flags, target) {
stage("backup ${target}") {
if (target == 'osd') {
@@ -72,7 +79,7 @@
def provider_pillar = salt.getPillar(master, "${kvm01}", "salt:control:cluster:internal:node:${minion_name}:provider")
def minionProvider = provider_pillar['return'][0].values()[0]
- waitForHealthy(master)
+ waitForHealthy(master, flags)
try {
salt.cmdRun(master, "${minionProvider}", "[ ! -f ${BACKUP_DIR}/${minion_name}.${domain}.qcow2.bak ] && virsh destroy ${minion_name}.${domain}")
} catch (Exception e) {
@@ -89,14 +96,14 @@
common.warningMsg(e)
}
salt.minionsReachable(master, 'I@salt:master', "${minion_name}*")
- waitForHealthy(master)
+ waitForHealthy(master, flags)
}
}
}
return
}
-def upgrade(master, target) {
+def upgrade(master, target, flags) {
stage("Change ${target} repos") {
salt.runSaltProcessStep(master, "I@ceph:${target}", 'saltutil.refresh_pillar', [], null, true, 5)
@@ -127,12 +134,21 @@
}
// restart services
stage("Restart ${target} services on ${minion}") {
- runCephCommand(master, "${minion}", "systemctl restart ceph-${target}.target")
+ if (target == 'osd') {
+ def osds = salt.getGrain(master, "${minion}", 'ceph:ceph_disk').values()[0]
+ osds[0].values()[0].values()[0].each { osd,param ->
+ runCephCommand(master, "${minion}", "systemctl restart ceph-${target}@${osd}")
+ waitForHealthy(master, flags)
+ }
+ } else {
+ runCephCommand(master, "${minion}", "systemctl restart ceph-${target}.target")
+ waitForHealthy(master, flags)
+ }
}
stage("Verify services for ${minion}") {
sleep(10)
- runCephCommand(master, ADMIN_HOST, "ceph -s")
+ runCephCommand(master, "${minion}", "systemctl status ceph-${target}.target")
}
stage('Ask for manual confirmation') {
@@ -196,23 +212,23 @@
}
if (STAGE_UPGRADE_MON.toBoolean() == true) {
- upgrade(pepperEnv, 'mon')
+ upgrade(pepperEnv, 'mon', flags)
}
if (STAGE_UPGRADE_MGR.toBoolean() == true) {
- upgrade(pepperEnv, 'mgr')
+ upgrade(pepperEnv, 'mgr', flags)
}
if (STAGE_UPGRADE_OSD.toBoolean() == true) {
- upgrade(pepperEnv, 'osd')
+ upgrade(pepperEnv, 'osd', flags)
}
if (STAGE_UPGRADE_RGW.toBoolean() == true) {
- upgrade(pepperEnv, 'radosgw')
+ upgrade(pepperEnv, 'radosgw', flags)
}
if (STAGE_UPGRADE_CLIENT.toBoolean() == true) {
- upgrade(pepperEnv, 'common')
+ upgrade(pepperEnv, 'common', flags)
}
// remove cluster flags
@@ -246,7 +262,7 @@
// wait for healthy cluster
if (WAIT_FOR_HEALTHY.toBoolean() == true) {
- waitForHealthy(pepperEnv)
+ waitForHealthy(pepperEnv, flags)
}
}
}
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 5159960..5f19480 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -342,10 +342,7 @@
// Install
//
if (!batch_size) {
- def workerThreads = salt.getReturnValues(salt.getPillar(venvPepper, "I@salt:master", "salt:master:worker_threads", null)).toString()
- if (workerThreads.isInteger() && workerThreads.toInteger() > 0) {
- batch_size = workerThreads
- }
+ batch_size = salt.getWorkerThreads(venvPepper)
}
// Check if all minions are reachable and ready
diff --git a/galera-cluster-verify-restore.groovy b/galera-cluster-verify-restore.groovy
index 3faedc7..459bfc4 100644
--- a/galera-cluster-verify-restore.groovy
+++ b/galera-cluster-verify-restore.groovy
@@ -20,6 +20,7 @@
def restoreType = env.RESTORE_TYPE
def runRestoreDb = false
def runBackupDb = false
+def restartCluster = false
askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
checkTimeSync = (env.getProperty('CHECK_TIME_SYNC') ?: true).toBoolean()
@@ -35,12 +36,17 @@
if (restoreType.equals("BACKUP_AND_RESTORE")) {
runBackupDb = true
}
+if (restoreType.equals("RESTART_CLUSTER")) {
+ restartCluster = true
+}
timeout(time: 12, unit: 'HOURS') {
node() {
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
+
+ def galeraStatus = [:]
stage('Verify status') {
def sysstatTargets = 'I@xtrabackup:client or I@xtrabackup:server'
def sysstatTargetsNodes = salt.getMinions(pepperEnv, sysstatTargets)
@@ -58,50 +64,48 @@
input message: "Do you want to continue? Click to confirm"
}
}
- resultCode = galera.verifyGaleraStatus(pepperEnv, false, checkTimeSync)
- if (resultCode == 128) {
- common.errorMsg("Unable to connect to Galera Master. Trying slaves...")
- resultCode = galera.verifyGaleraStatus(pepperEnv, true, checkTimeSync)
- if (resultCode == 129) {
- common.errorMsg("Unable to obtain Galera slave minions list. Without fixing this issue, pipeline cannot continue in verification, backup and restoration. This may be caused by wrong Galera configuration or corrupted pillar data.")
+ galeraStatus = galera.verifyGaleraStatus(pepperEnv, checkTimeSync)
+
+ switch (galeraStatus.error) {
+ case 128:
+ common.errorMsg("Unable to obtain Galera members minions list. Without fixing this issue, pipeline cannot continue in verification, backup and restoration. This may be caused by wrong Galera configuration or corrupted pillar data.")
currentBuild.result = "FAILURE"
return
- } else if (resultCode == 130) {
+ case 130:
common.errorMsg("Neither master or slaves are reachable. Without fixing this issue, pipeline cannot continue in verification, backup and restoration. Is at least one member of the Galera cluster up and running?")
currentBuild.result = "FAILURE"
return
- }
- }
- if (resultCode == 131) {
- common.errorMsg("Time desynced - Please fix this issue and rerun the pipeline.")
- currentBuild.result = "FAILURE"
- return
- }
- if (resultCode == 140 || resultCode == 141) {
- common.errorMsg("Disk utilization check failed - Please fix this issue and rerun the pipeline.")
- currentBuild.result = "FAILURE"
- return
- }
- if (resultCode == 1) {
- if (askConfirmation) {
- input message: "There was a problem with parsing the status output or with determining it. Do you want to run a restore?"
- } else {
- common.warningMsg("There was a problem with parsing the status output or with determining it. Try to restore.")
- }
- } else if (resultCode > 1) {
- if (askConfirmation) {
- input message: "There's something wrong with the cluster, do you want to continue with backup and/or restore?"
- } else {
- common.warningMsg("There's something wrong with the cluster, try to backup and/or restore.")
- }
- } else {
- if (askConfirmation) {
- input message: "There seems to be everything alright with the cluster, do you still want to continue with backup and/or restore?"
- } else {
- common.warningMsg("There seems to be everything alright with the cluster, no backup and no restoration will be done.")
- currentBuild.result = "SUCCESS"
+ case 131:
+ common.errorMsg("Time desynced - Please fix this issue and rerun the pipeline.")
+ currentBuild.result = "FAILURE"
return
- }
+ case 140..141:
+ common.errorMsg("Disk utilization check failed - Please fix this issue and rerun the pipeline.")
+ currentBuild.result = "FAILURE"
+ return
+ case 1:
+ if (askConfirmation) {
+ input message: "There was a problem with parsing the status output or with determining it. Do you want to run a next action: ${restoreType}?"
+ } else {
+ common.warningMsg("There was a problem with parsing the status output or with determining it. Trying to perform action: ${restoreType}.")
+ }
+ break
+ case 0:
+ if (askConfirmation) {
+ input message: "There seems to be everything alright with the cluster, do you still want to continue with next action: ${restoreType}?"
+ break
+ } else {
+ common.warningMsg("There seems to be everything alright with the cluster, no backup and no restoration will be done.")
+ currentBuild.result = "SUCCESS"
+ return
+ }
+ default:
+ if (askConfirmation) {
+ input message: "There's something wrong with the cluster, do you want to continue with action: ${restoreType}?"
+ } else {
+ common.warningMsg("There's something wrong with the cluster, trying to perform action: ${restoreType}")
+ }
+ break
}
}
if (runBackupDb) {
@@ -117,24 +121,41 @@
)
}
}
- if (runRestoreDb) {
- stage('Restore') {
- if (askConfirmation) {
- input message: "Are you sure you want to run a restore? Click to confirm"
- }
- try {
- if ((!askConfirmation && resultCode > 0) || askConfirmation) {
- galera.restoreGaleraCluster(pepperEnv, runRestoreDb)
+ if (runRestoreDb || restartCluster) {
+ if (runRestoreDb) {
+ stage('Restore') {
+ if (askConfirmation) {
+ input message: "Are you sure you want to run a restore? Click to confirm"
}
- } catch (Exception e) {
- common.errorMsg("Restoration process has failed.")
- common.errorMsg(e.getMessage())
+ try {
+ if ((!askConfirmation && resultCode > 0) || askConfirmation) {
+ galera.restoreGaleraCluster(pepperEnv, galeraStatus)
+ }
+ } catch (Exception e) {
+ common.errorMsg("Restoration process has failed.")
+ common.errorMsg(e.getMessage())
+ }
+ }
+ }
+ if (restartCluster) {
+ stage('Restart cluster') {
+ if (askConfirmation) {
+ input message: "Are you sure you want to run a restart? Click to confirm"
+ }
+ try {
+ if ((!askConfirmation && resultCode > 0) || askConfirmation) {
+ galera.restoreGaleraCluster(pepperEnv, galeraStatus, false)
+ }
+ } catch (Exception e) {
+ common.errorMsg("Restart process has failed.")
+ common.errorMsg(e.getMessage())
+ }
}
}
stage('Verify restoration result') {
common.retry(verificationRetries, 15) {
- exitCode = galera.verifyGaleraStatus(pepperEnv, false, false)
- if (exitCode >= 1) {
+ def status = galera.verifyGaleraStatus(pepperEnv, false)
+ if (status.error >= 1) {
error("Verification attempt finished with an error. This may be caused by cluster not having enough time to come up or to sync. Next verification attempt in 5 seconds.")
} else {
common.infoMsg("Restoration procedure seems to be successful. See verification report to be sure.")
diff --git a/galera-database-backup-pipeline.groovy b/galera-database-backup-pipeline.groovy
index a6d0af5..8239aa6 100644
--- a/galera-database-backup-pipeline.groovy
+++ b/galera-database-backup-pipeline.groovy
@@ -57,5 +57,19 @@
stage('Clean-up') {
salt.cmdRun(pepperEnv, backupNode, "su root -c '/usr/local/bin/innobackupex-runner.sh -c'")
}
+ stage('Backup Dogtag') {
+ if (!salt.getMinions(pepperEnv, "I@dogtag:server:enabled").isEmpty()) {
+ dogtagBackupBuild = build(job: 'backupninja_backup', parameters: [
+ [$class: 'StringParameterValue', name: 'SALT_MASTER_URL', value: SALT_MASTER_URL],
+ [$class: 'StringParameterValue', name: 'SALT_MASTER_CREDENTIALS', value: SALT_MASTER_CREDENTIALS],
+ [$class: 'BooleanParameterValue', name: 'ASK_CONFIRMATION', value: "false"],
+ [$class: 'BooleanParameterValue', name: 'BACKUP_SALTMASTER_AND_MAAS', value: "false"],
+ [$class: 'BooleanParameterValue', name: 'BACKUP_DOGTAG', value: "true"],
+ ]
+ )
+ } else {
+ common.warningMsg("Dogtag pillar not found. This is fine if you are using different Barbican backend.")
+ }
+ }
}
}
\ No newline at end of file
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index 7285c40..f1964ab 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -91,8 +91,11 @@
common.errorMsg('An error has been occurred during cassandra db startup on I@opencontrail:control and not I@cassandra:backup:client nodes: ' + err.getMessage())
throw err
}
- // another mantra, wait till all services are up
- sleep(60)
+ // wait till outstanding cassandra dbs are up
+ common.retry(6, 20){
+ common.infoMsg("Trying to connect to casandra db on I@opencontrail:control and not I@cassandra:backup:client nodes ...")
+ salt.cmdRun(pepperEnv, 'I@opencontrail:control and not I@cassandra:backup:client', "nc -v -z -w2 ${configDbIp} ${configDbPort}")
+ }
try {
common.infoMsg("Start analytics containers node")
salt.cmdRun(pepperEnv, 'I@opencontrail:collector', 'cd /etc/docker/compose/opencontrail/; docker-compose up -d')
diff --git a/update-package.groovy b/update-package.groovy
index 851c376..14c2056 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -54,10 +54,14 @@
common.infoMsg("Listing all the packages that have a new update available on nodes: ${targetLiveAll}")
salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'pkg.list_upgrades', [], batch_size, true)
if (TARGET_PACKAGES != '' && TARGET_PACKAGES != '*') {
- common.warningMsg("Note that only the \"${TARGET_PACKAGES}\" would be installed from the above list of available updates on the ${targetLiveAll}")
+ if (ALLOW_DEPENDENCY_UPDATE.toBoolean()) {
+ common.warningMsg("Note that the \"${TARGET_PACKAGES}\" and it new dependencies would be installed from the above list of available updates on the ${targetLiveAll}")
+ } else {
+ common.warningMsg("Note that only the \"${TARGET_PACKAGES}\" would be installed from the above list of available updates on the ${targetLiveAll}")
+ commandKwargs = ['only_upgrade': 'true']
+ }
command = "pkg.install"
packages = TARGET_PACKAGES.tokenize(' ')
- commandKwargs = ['only_upgrade': 'true']
}
}
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 5d5f5ae..fc76277 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -70,6 +70,14 @@
}
}
+def getWorkerThreads(saltId) {
+ if (env.getEnvironment().containsKey('SALT_MASTER_OPT_WORKER_THREADS')) {
+ return env['SALT_MASTER_OPT_WORKER_THREADS'].toString()
+ }
+ def threads = salt.cmdRun(saltId, "I@salt:master", "cat /etc/salt/master.d/master.conf | grep worker_threads | cut -f 2 -d ':'", true, null, true)
+ return threads['return'][0].values()[0].replaceAll('Salt command execution success','').trim()
+}
+
def wa29352(ArrayList saltMinions, String cname) {
// WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/12/openssh/client/root.yml
// Default soft-param has been removed, what now makes not possible to render some old env's.
@@ -83,7 +91,7 @@
salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29352File}", true, null, false)
}
catch (Exception ex) {
- common.infoMsg('Work-around for PROD-29352 already apply, nothing todo')
+ common.infoMsg('Work-around for PROD-29352 already applied, nothing todo')
return
}
def rKeysDict = [
@@ -112,6 +120,9 @@
"grep -q '${wa29352ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29352ClassName' infra/secrets.yml")
salt.fullRefresh(venvPepper, '*')
sh('rm -fv ' + _tempFile)
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && git status && " +
+ "git add ${wa29352File} && git add -u && git commit --allow-empty -m 'Cluster model updated with WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/ at ${common.getDatetime()}' ")
+ common.infoMsg('Work-around for PROD-29352 successfully applied')
}
def wa29155(ArrayList saltMinions, String cname) {
@@ -206,6 +217,30 @@
}
}
+def wa32182(String cluster_name) {
+ if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
+ def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
+ def fixFile = "${clusterModelPath}/opencontrail/common_wa32182.yml"
+ def usualFile = "${clusterModelPath}/opencontrail/common.yml"
+ def fixFileContent = "classes:\n- system.opencontrail.common\n"
+ salt.cmdRun(venvPepper, 'I@salt:master', "test -f ${fixFile} -o -f ${usualFile} || echo '${fixFileContent}' > ${fixFile}")
+ def contrailFiles = ['opencontrail/analytics.yml', 'opencontrail/control.yml', 'openstack/compute/init.yml']
+ if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
+ contrailFiles.add('kubernetes/compute.yml')
+ }
+ for(String contrailFile in contrailFiles) {
+ contrailFile = "${clusterModelPath}/${contrailFile}"
+ def containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- cluster\\.${cluster_name}\\.opencontrail\\.common(_wa32182)?\$' ${contrailFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ if (containsFix) {
+ continue
+ } else {
+ salt.cmdRun(venvPepper, 'I@salt:master', "grep -q -E '^parameters:' ${contrailFile} && sed -i '/^parameters:/i - cluster.${cluster_name}.opencontrail.common_wa32182' ${contrailFile} || " +
+ "echo '- cluster.${cluster_name}.opencontrail.common_wa32182' >> ${contrailFile}")
+ }
+ }
+ }
+}
+
def archiveReclassInventory(filename) {
def _tmp_file = '/tmp/' + filename + UUID.randomUUID().toString().take(8)
// jenkins may fail at overheap. Compress data with gzip like WA
@@ -304,16 +339,16 @@
if (gitTargetMcpVersion != 'proposed') {
reclassSystemBranchDefault = "origin/${gitTargetMcpVersion}"
}
- def driteTrainParamsYaml = env.getProperty('DRIVE_TRAIN_PARAMS')
- if (driteTrainParamsYaml) {
- def driteTrainParams = readYaml text: driteTrainParamsYaml
- saltMastURL = driteTrainParams.get('SALT_MASTER_URL')
- saltMastCreds = driteTrainParams.get('SALT_MASTER_CREDENTIALS')
- upgradeSaltStack = driteTrainParams.get('UPGRADE_SALTSTACK', false).toBoolean()
- updateClusterModel = driteTrainParams.get('UPDATE_CLUSTER_MODEL', false).toBoolean()
- updatePipelines = driteTrainParams.get('UPDATE_PIPELINES', false).toBoolean()
- updateLocalRepos = driteTrainParams.get('UPDATE_LOCAL_REPOS', false).toBoolean()
- reclassSystemBranch = driteTrainParams.get('RECLASS_SYSTEM_BRANCH', reclassSystemBranchDefault)
+ def driveTrainParamsYaml = env.getProperty('DRIVE_TRAIN_PARAMS')
+ if (driveTrainParamsYaml) {
+ def driveTrainParams = readYaml text: driveTrainParamsYaml
+ saltMastURL = driveTrainParams.get('SALT_MASTER_URL')
+ saltMastCreds = driveTrainParams.get('SALT_MASTER_CREDENTIALS')
+ upgradeSaltStack = driveTrainParams.get('UPGRADE_SALTSTACK', false).toBoolean()
+ updateClusterModel = driveTrainParams.get('UPDATE_CLUSTER_MODEL', false).toBoolean()
+ updatePipelines = driveTrainParams.get('UPDATE_PIPELINES', false).toBoolean()
+ updateLocalRepos = driveTrainParams.get('UPDATE_LOCAL_REPOS', false).toBoolean()
+ reclassSystemBranch = driveTrainParams.get('RECLASS_SYSTEM_BRANCH', reclassSystemBranchDefault)
batchSize = driveTrainParams.get('BATCH_SIZE', '')
} else {
// backward compatibility for 2018.11.0
@@ -331,11 +366,8 @@
if (cluster_name == '' || cluster_name == 'null' || cluster_name == null) {
error('Pillar data is broken for Salt master node! Please check it manually and re-run pipeline.')
}
- if (!batch_size) {
- def workerThreads = salt.getReturnValues(salt.getPillar(venvPepper, "I@salt:master", "salt:master:worker_threads", null))
- if (workerThreads.isInteger() && workerThreads.toInteger() > 0) {
- batch_size = workerThreads
- }
+ if (!batchSize) {
+ batchSize = getWorkerThreads(venvPepper)
}
stage('Update Reclass and Salt-Formulas') {
@@ -377,6 +409,9 @@
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/linux_system_repo_mcp_maas_url/d' maas.yml")
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/maas_region_main_archive/d' maas.yml")
+
// Switch Jenkins/Gerrit to use LDAP SSL/TLS
def gerritldapURI = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly 'gerrit_ldap_server: .*' * | grep -Po 'gerrit_ldap_server: \\K.*' | tr -d '\"'", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
@@ -413,7 +448,7 @@
}
// Add all update repositories
def repoIncludeBase = '- system.linux.system.repo.mcp.apt_mirantis.'
- def updateRepoList = ['cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'percona', 'salt-formulas', 'saltstack', 'ubuntu']
+ def updateRepoList = ['cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'maas', 'percona', 'salt-formulas', 'saltstack', 'ubuntu']
updateRepoList.each { repo ->
def repoNameUpdateInclude = "${repoIncludeBase}update.${repo}"
def filesWithInclude = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -Plr '\\${repoIncludeBase}${repo}\$' . || true", false).get('return')[0].values()[0].trim().tokenize('\n')
@@ -427,6 +462,7 @@
}
}
}
+ wa32182(cluster_name)
// Add new defaults
common.infoMsg("Add new defaults")
salt.cmdRun(venvPepper, 'I@salt:master', "grep '^ mcp_version: ' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +