Merge the tip of origin/release/proposed/2019.2.0 into origin/release/2019.2.0
09fa3ce Added check for Alerta admin API key generation
984c82e Add keystone credentials key backup-restore steps
05c4983 Add check for galera clustercheck password presence
cff0741 Adding OS db cleanup job pipeline
Change-Id: I9248cbc476dcd69880f9252733d45d9a9f42d829
diff --git a/backupninja-backup-pipeline.groovy b/backupninja-backup-pipeline.groovy
index 028ccbe..a55a5bb 100644
--- a/backupninja-backup-pipeline.groovy
+++ b/backupninja-backup-pipeline.groovy
@@ -5,8 +5,10 @@
def askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
def backupSaltMasterAndMaas = (env.getProperty('BACKUP_SALTMASTER_AND_MAAS') ?: true).toBoolean()
def backupDogtag = (env.getProperty('BACKUP_DOGTAG') ?: true).toBoolean()
+def backupKeystone = (env.getProperty('BACKUP_KEYSTONE_CREDENTIAL_KEYS') ?: true).toBoolean()
def saltMasterTargetMatcher = "I@backupninja:client and I@salt:master"
def dogtagTagetMatcher = "I@backupninja:client and I@dogtag:server"
+def keystoneTargetMatcher = "I@backupninja:client and I@keystone:server"
logBackupSuccess = []
logBackupFailure = []
@@ -37,6 +39,7 @@
node() {
def saltMasterBackupNode = ''
def dogtagBackupNode = ''
+ def keystoneBackupNode = ''
def backupServer = ''
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
@@ -144,6 +147,29 @@
currentBuild.result = "FAILURE"
throw e
}
+ if (backupKeystone) {
+ try {
+ keystoneBackupNode = salt.getMinionsSorted(pepperEnv, keystoneTargetMatcher)[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", keystoneBackupNode)
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Pipeline wasn't able to detect node with backupninja:client and keystone:server pillars defined or the minion is not reachable")
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+ }
+
+ try {
+ backupServer = salt.getMinions(pepperEnv, "I@backupninja:server")[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", backupServer)
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Pipeline wasn't able to detect backupninja:server pillar or the minion is not reachable")
+ currentBuild.result = "FAILURE"
+ throw e
+ }
}
stage('Prepare for backup') {
if (backupSaltMasterAndMaas) {
@@ -167,6 +193,10 @@
salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
salt.enforceState(['saltId': pepperEnv, 'target': dogtagTagetMatcher, 'state': 'backupninja'])
}
+ if (backupKeystone) {
+ salt.enforceState(['saltId': pepperEnv, 'target': keystoneTargetMatcher, 'state': 'backupninja'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
+ }
}
stage('Backup') {
if (backupSaltMasterAndMaas) {
@@ -177,6 +207,10 @@
def output = salt.getReturnValues(salt.cmdRun(pepperEnv, dogtagBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
checkBackupninjaLog(output, "Dogtag")
}
+ if (backupKeystone) {
+ def output = salt.getReturnValues(salt.cmdRun(pepperEnv, keystoneBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
+ checkBackupninjaLog(output, "Keystone")
+ }
}
stage('Results') {
if (logBackupSuccess.size() > 0) {
diff --git a/backupninja-restore-pipeline.groovy b/backupninja-restore-pipeline.groovy
index 32f3962..cbd1f43 100644
--- a/backupninja-restore-pipeline.groovy
+++ b/backupninja-restore-pipeline.groovy
@@ -5,6 +5,7 @@
def maasNodes = []
def restoreSaltMasterAndMaas = (env.getProperty('RESTORE_SALTMASTER_AND_MAAS') ?: true).toBoolean()
def restoreDogtag = (env.getProperty('RESTORE_DOGTAG') ?: true).toBoolean()
+def restoreKeystone = (env.getProperty('RESTORE_KEYSTONE_CREDENTIAL_KEYS') ?: true).toBoolean()
timeout(time: 12, unit: 'HOURS') {
node() {
@@ -90,6 +91,9 @@
salt.enforceState(['saltId': pepperEnv, 'target': 'I@dogtag:server:role:master', 'state': 'dogtag.server.restore'])
salt.runSaltProcessStep(pepperEnv, 'I@dogtag:server:role:slave', 'service.start', ['dirsrv@pki-tomcat.service'])
}
+ if (restoreKeystone) {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@keystone:server:role:primary', 'state': 'keystone.restore'])
+ }
}
stage('After restore steps') {
if (restoreSaltMasterAndMaas) {
diff --git a/openstack-database-cleanup.groovy b/openstack-database-cleanup.groovy
new file mode 100644
index 0000000..0654d54
--- /dev/null
+++ b/openstack-database-cleanup.groovy
@@ -0,0 +1,48 @@
+/**
+ *
+ * Cleanup OpenStack databases from stale records (archived records or records marked as deleted).
+ * Cleanup OpenStack service databases.
+ *
+ * Expected parameters:
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
+ * SALT_MASTER_URL Full Salt API address [http://10.10.10.15:6969].
+ *
+**/
+
+def common = new com.mirantis.mk.Common()
+def salt = new com.mirantis.mk.Salt()
+def python = new com.mirantis.mk.Python()
+
+def os_services = [ 'nova:controller', 'heat:server', 'cinder:controller' ]
+
+def slave_node = 'python'
+
+if (common.validInputParam('SLAVE_NODE')) {
+ slave_node = SLAVE_NODE
+}
+
+def env = "pepperEnv"
+timeout(time: 12, unit: 'HOURS') {
+
+ node(slave_node) {
+
+ stage('Setup virtualenv for Pepper') {
+ python.setupPepperVirtualenv(env, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ stage('Databases cleanup') {
+
+ for (os_service in os_services) {
+
+ formula = os_service.split(":")[0]
+ os_state = "${formula}.db.db_cleanup"
+ os_file = "/usr/share/salt-formulas/env/${formula}/db/db_cleanup.sls"
+
+ if (salt.runSaltProcessStep(env, 'I@salt:master', 'file.file_exists', [os_file], null, true, 5)['return'][0].values()[0].toBoolean()) {
+ salt.enforceStateWithTest([saltId: env, target: "I@${os_service}:role:primary", state: [os_state]])
+ }
+
+ }
+ }
+ }
+}
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 9d3981b..e616101 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -247,6 +247,24 @@
}
}
+def check_35705(String cluster_name) {
+ def galeracheckpasswordPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:galera_clustercheck_password').get("return")[0].values()[0]
+ if (galeracheckpasswordPillar == '' || galeracheckpasswordPillar == 'null' || galeracheckpasswordPillar == null) {
+ error('Galera clustercheck password is not defined.\n' +
+ 'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#improper-operation-of-galera-ha for more info')
+ }
+}
+
+def check_35884(String cluster_name) {
+ def alertaApiKeyGenPillar = salt.getPillar(venvPepper, 'I@salt:master', '_param:alerta_admin_api_key_generated').get("return")[0].values()[0]
+ def alertaApiKeyPillar = salt.getPillar(venvPepper, 'I@prometheus:alerta or I@prometheus:alertmanager', '_param:alerta_admin_key').get("return")[0].values()[0]
+
+ if (alertaApiKeyGenPillar == '' || alertaApiKeyGenPillar == 'null' || alertaApiKeyGenPillar == null || alertaApiKeyPillar == '' || alertaApiKeyPillar == 'null' || alertaApiKeyPillar == null) {
+ error('Alerta admin API key not defined.\n' +
+ 'See https://docs.mirantis.com/mcp/q4-18/mcp-release-notes/mu/mu-12/mu-12-addressed/mu-12-dtrain/mu-12-dt-manual.html#i-35884 for more info')
+ }
+}
+
def wa32182(String cluster_name) {
if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
@@ -659,6 +677,8 @@
fullRefreshOneByOne(venvPepper, allMinions)
check_34406(cluster_name)
+ check_35705(cluster_name)
+ check_35884(cluster_name)
common.infoMsg('Perform: Validate reclass medata before processing')
validateReclassModel(minions, 'before')