Add openstack-galera-upgrade.groovy pipeline
Change-Id: Iae8d3e50fbe3c6bb608a30ac0bc4f87b288b933c
Related-Prod: PROD-29630
(cherry picked from commit f4adf25bc82d5e154e3214859fcb0e9778c30947)
diff --git a/openstack-galera-upgrade.groovy b/openstack-galera-upgrade.groovy
new file mode 100644
index 0000000..f124051
--- /dev/null
+++ b/openstack-galera-upgrade.groovy
@@ -0,0 +1,206 @@
+/**
+ * Upgrade MySQL and Galera packages on dbs nodes.
+ * Update packages on given nodes
+ *
+ * Expected parameters:
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
+ * SALT_MASTER_URL Full Salt API address [http://10.10.10.15:6969].
+ * SHUTDOWN_CLUSTER Shutdown all mysql instances on target nodes at the same time.
+ * OS_DIST_UPGRADE Upgrade system packages including kernel (apt-get dist-upgrade).
+ * OS_UPGRADE Upgrade all installed applications (apt-get upgrade)
+ * TARGET_SERVERS Comma separated list of salt compound definitions to upgrade.
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
+ *
+**/
+
+def common = new com.mirantis.mk.Common()
+def salt = new com.mirantis.mk.Salt()
+def python = new com.mirantis.mk.Python()
+def debian = new com.mirantis.mk.Debian()
+def openstack = new com.mirantis.mk.Openstack()
+def galera = new com.mirantis.mk.Galera()
+def shutdownCluster = SHUTDOWN_CLUSTER.toBoolean()
+def interactive = INTERACTIVE.toBoolean()
+def LinkedHashMap upgradeStageMap = [:]
+
+upgradeStageMap.put('Pre upgrade',
+ [
+ 'Description': 'Only non destructive actions will be applied during this phase. Basic service verification will be performed.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * No service downtime
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Verify API, perform basic CRUD operations for services.
+ * Verify MySQL is running and Galera cluster is operational.''',
+ 'State result': 'Basic checks around wsrep Galera status are passed.'
+ ])
+
+upgradeStageMap.put('Stop MySQL service',
+ [
+ 'Description': 'All MySQL services will be stopped on All TARGET_SERVERS nodes.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * MySQL services are stopped.
+ * OpenStack APIs are not accessible from this point.
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Stop MySQL services''',
+ 'State result': 'MySQL service is stopped',
+ ])
+
+upgradeStageMap.put('Upgrade OS',
+ [
+ 'Description': 'Optional step. OS packages will be upgraded during this phase, depending on the job parameters dist-upgrade might be called. And reboot of node executed.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * No workload downtime
+ * The nodes might be rebooted''',
+ 'Launched actions': '''
+ * Install new version of system packages
+ * If doing dist-upgrade new kernel might be installed and node rebooted
+ * System packages are updated
+ * Node might be rebooted
+'''
+ ])
+
+upgradeStageMap.put('Upgrade MySQL server',
+ [
+ 'Description': 'MySQL and Erlang code will be upgraded during this stage. No workload downtime is expected.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * OpenStack services loose connection to MySQL server
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Install new version of MySQL and Galera packages
+ * Render version of configs''',
+ 'State result': '''
+ * MySQL packages are upgraded''',
+ ])
+
+upgradeStageMap.put('Start MySQL service',
+ [
+ 'Description': 'All MySQL services will be running on All TARGET_SERVERS nodes.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * MySQL service is running.
+ * OpenStack API are accessible from this point.
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Start MySQL service''',
+ 'State result': 'MySQL service is running',
+ ])
+
+def env = "env"
+timeout(time: 12, unit: 'HOURS') {
+ node() {
+
+ stage('Setup virtualenv for Pepper') {
+ python.setupPepperVirtualenv(env, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ def upgradeTargets = salt.getMinionsSorted(env, TARGET_SERVERS)
+
+ if (upgradeTargets.isEmpty()) {
+ error("No servers for upgrade matched by ${TARGET_SERVERS}")
+ }
+
+ def targetSecMapping = [:]
+ def secNoList = []
+ def out
+ def stopTargets = upgradeTargets.reverse()
+ common.printStageMap(upgradeStageMap)
+
+ if (interactive){
+ input message: common.getColorizedString(
+ "Above you can find detailed info this pipeline will execute.\nThe info provides brief description of each stage, actions that will be performed and service/workload impact during each stage.\nPlease read it carefully.", "yellow")
+ }
+
+ for (target in upgradeTargets) {
+ salt.runSaltProcessStep(env, target, 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(env, target, ['linux.system.repo'])
+ common.stageWrapper(upgradeStageMap, "Pre upgrade", target, interactive) {
+ openstack.runOpenStackUpgradePhase(env, target, 'pre')
+ openstack.runOpenStackUpgradePhase(env, target, 'verify')
+ }
+ }
+
+ if (shutdownCluster){
+ for (target in stopTargets) {
+ common.stageWrapper(upgradeStageMap, "Stop MySQL service", target, interactive) {
+ openstack.runOpenStackUpgradePhase(env, target, 'service_stopped')
+ }
+ }
+ }
+
+ for (target in upgradeTargets) {
+ out = salt.cmdRun(env, target, 'cat /var/lib/mysql/grastate.dat | grep "seqno" | cut -d ":" -f2', true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ common.infoMsg("Get seqno: ${out} for node ${target}")
+ if (!out.isNumber()){
+ out = -2
+ }
+ targetSecMapping[out.toInteger()] = target
+ secNoList.add(out.toInteger())
+ }
+
+ def masterNode = targetSecMapping[secNoList.max()]
+ common.infoMsg("Master node is: ${masterNode}")
+
+ // Make sure we start upgrade always from master node
+ upgradeTargets.remove(masterNode)
+ upgradeTargets = [masterNode] + upgradeTargets
+ common.infoMsg("Upgrade targets are: ${upgradeTargets}")
+
+ for (target in upgradeTargets) {
+
+ common.stageWrapper(upgradeStageMap, "Stop MySQL service", target, interactive) {
+ openstack.runOpenStackUpgradePhase(env, target, 'service_stopped')
+ }
+
+ common.stageWrapper(upgradeStageMap, "Upgrade OS", target, interactive) {
+ if (OS_DIST_UPGRADE.toBoolean() == true){
+ upgrade_mode = 'dist-upgrade'
+ } else if (OS_UPGRADE.toBoolean() == true){
+ upgrade_mode = 'upgrade'
+ }
+ if (OS_DIST_UPGRADE.toBoolean() == true || OS_UPGRADE.toBoolean() == true) {
+ debian.osUpgradeNode(env, target, upgrade_mode, false)
+ }
+ }
+
+ common.stageWrapper(upgradeStageMap, "Upgrade MySQL server", target, interactive) {
+ openstack.runOpenStackUpgradePhase(env, target, 'pkgs_latest')
+ openstack.runOpenStackUpgradePhase(env, target, 'render_config')
+ }
+
+ if (shutdownCluster && target == masterNode){
+ //Start first node.
+ common.stageWrapper(upgradeStageMap, "Start MySQL service", target, interactive) {
+ galera.startFirstNode(env, target)
+ }
+ }
+
+ common.stageWrapper(upgradeStageMap, "Start MySQL service", target, interactive) {
+ openstack.runOpenStackUpgradePhase(env, target, 'service_running')
+ openstack.runOpenStackUpgradePhase(env, target, 'verify')
+ }
+ }
+
+ // restart first node by applying state.
+
+ if (shutdownCluster) {
+ openstack.runOpenStackUpgradePhase(env, masterNode, 'render_config')
+ salt.cmdRun(env, masterNode, "service mysql reload")
+ openstack.runOpenStackUpgradePhase(env, masterNode, 'verify')
+ }
+
+ for (target in upgradeTargets) {
+ ensureClusterState = galera.getWsrepParameters(env, target, 'wsrep_evs_state')
+ if (ensureClusterState['wsrep_evs_state'] == 'OPERATIONAL') {
+ common.infoMsg('Node is in OPERATIONAL state.')
+ } else {
+ throw new Exception("Node is NOT in OPERATIONAL state.")
+ }
+ }
+ }
+}