Merge "Add extra repo replacement in upgrade-mcp-release pipeline" into release/2019.2.0
diff --git a/galera-cluster-verify-restore.groovy b/galera-cluster-verify-restore.groovy
index 2a575cc..c98c336 100644
--- a/galera-cluster-verify-restore.groovy
+++ b/galera-cluster-verify-restore.groovy
@@ -4,6 +4,9 @@
* Expected parameters:
* SALT_MASTER_CREDENTIALS Credentials to the Salt API.
* SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
+ * ASK_CONFIRMATION Ask confirmation for restore
+ * CHECK_TIME_SYNC Set to true to check time synchronization accross selected nodes.
+ * VERIFICATION_RETRIES Number of restries to verify the restoration.
*
**/
@@ -15,6 +18,14 @@
def pepperEnv = "pepperEnv"
def resultCode = 99
+askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
+checkTimeSync = (env.getProperty('CHECK_TIME_SYNC') ?: true).toBoolean()
+if (common.validInputParam(VERIFICATION_RETRIES) && VERIFICATION_RETRIES.isInteger()) {
+ verificationRetries = VERIFICATION_RETRIES.toInteger()
+} else {
+ verificationRetries = 5
+}
+
timeout(time: 12, unit: 'HOURS') {
node() {
stage('Setup virtualenv for Pepper') {
@@ -29,9 +40,11 @@
if (resultCode == 129) {
common.errorMsg("Unable to obtain Galera slave minions list". "Without fixing this issue, pipeline cannot continue in verification and restoration.")
currentBuild.result = "FAILURE"
+ return
} else if (resultCode == 130) {
common.errorMsg("Neither master or slaves are reachable. Without fixing this issue, pipeline cannot continue in verification and restoration.")
currentBuild.result = "FAILURE"
+ return
}
}
if (resultCode == 1) {
@@ -49,13 +62,14 @@
}
}
stage('Verify restoration result') {
- exitCode = openstack.verifyGaleraStatus(pepperEnv, false)
- if (exitCode >= 1) {
- common.errorMsg("Restoration procedure was probably not successful. See verification report for more information.")
- currentBuild.result = "FAILURE"
- } else {
- common.infoMsg("Restoration procedure seems to be successful. See verification report to be sure.")
- currentBuild.result = "SUCCESS"
+ common.retry(verificationRetries, 15) {
+ exitCode = openstack.verifyGaleraStatus(pepperEnv, false, false)
+ if (exitCode >= 1) {
+ error("Verification attempt finished with an error. This may be caused by cluster not having enough time to come up or to sync. Next verification attempt in 5 seconds.")
+ } else {
+ common.infoMsg("Restoration procedure seems to be successful. See verification report to be sure.")
+ currentBuild.result = "SUCCESS"
+ }
}
}
}