| /** |
| * Upgrade MySQL and Galera packages on dbs nodes. |
| * Update packages on given nodes |
| * |
| * Expected parameters: |
| * SALT_MASTER_CREDENTIALS Credentials to the Salt API. |
| * SALT_MASTER_URL Full Salt API address [http://10.10.10.15:6969]. |
| * SHUTDOWN_CLUSTER Shutdown all mysql instances on target nodes at the same time. |
| * OS_DIST_UPGRADE Upgrade system packages including kernel (apt-get dist-upgrade). |
| * OS_UPGRADE Upgrade all installed applications (apt-get upgrade) |
| * TARGET_SERVERS Comma separated list of salt compound definitions to upgrade. |
| * INTERACTIVE Ask interactive questions during pipeline run (bool). |
| * |
| **/ |
| |
| def common = new com.mirantis.mk.Common() |
| def salt = new com.mirantis.mk.Salt() |
| def python = new com.mirantis.mk.Python() |
| def debian = new com.mirantis.mk.Debian() |
| def openstack = new com.mirantis.mk.Openstack() |
| def galera = new com.mirantis.mk.Galera() |
| def shutdownCluster = SHUTDOWN_CLUSTER.toBoolean() |
| def interactive = INTERACTIVE.toBoolean() |
| def LinkedHashMap upgradeStageMap = [:] |
| |
| upgradeStageMap.put('Pre upgrade', |
| [ |
| 'Description': 'Only non destructive actions will be applied during this phase. Basic service verification will be performed.', |
| 'Status': 'NOT_LAUNCHED', |
| 'Expected behaviors': ''' |
| * No service downtime |
| * No workload downtime''', |
| 'Launched actions': ''' |
| * Verify API, perform basic CRUD operations for services. |
| * Verify MySQL is running and Galera cluster is operational.''', |
| 'State result': 'Basic checks around wsrep Galera status are passed.' |
| ]) |
| |
| upgradeStageMap.put('Stop MySQL service', |
| [ |
| 'Description': 'All MySQL services will be stopped on All TARGET_SERVERS nodes.', |
| 'Status': 'NOT_LAUNCHED', |
| 'Expected behaviors': ''' |
| * MySQL services are stopped. |
| * OpenStack APIs are not accessible from this point. |
| * No workload downtime''', |
| 'Launched actions': ''' |
| * Stop MySQL services''', |
| 'State result': 'MySQL service is stopped', |
| ]) |
| |
| upgradeStageMap.put('Upgrade OS', |
| [ |
| 'Description': 'Optional step. OS packages will be upgraded during this phase, depending on the job parameters dist-upgrade might be called. And reboot of node executed.', |
| 'Status': 'NOT_LAUNCHED', |
| 'Expected behaviors': ''' |
| * No workload downtime |
| * The nodes might be rebooted''', |
| 'Launched actions': ''' |
| * Install new version of system packages |
| * If doing dist-upgrade new kernel might be installed and node rebooted |
| * System packages are updated |
| * Node might be rebooted |
| ''' |
| ]) |
| |
| upgradeStageMap.put('Upgrade MySQL server', |
| [ |
| 'Description': 'MySQL and Erlang code will be upgraded during this stage. No workload downtime is expected.', |
| 'Status': 'NOT_LAUNCHED', |
| 'Expected behaviors': ''' |
| * OpenStack services loose connection to MySQL server |
| * No workload downtime''', |
| 'Launched actions': ''' |
| * Install new version of MySQL and Galera packages |
| * Render version of configs''', |
| 'State result': ''' |
| * MySQL packages are upgraded''', |
| ]) |
| |
| upgradeStageMap.put('Start MySQL service', |
| [ |
| 'Description': 'All MySQL services will be running on All TARGET_SERVERS nodes.', |
| 'Status': 'NOT_LAUNCHED', |
| 'Expected behaviors': ''' |
| * MySQL service is running. |
| * OpenStack API are accessible from this point. |
| * No workload downtime''', |
| 'Launched actions': ''' |
| * Start MySQL service''', |
| 'State result': 'MySQL service is running', |
| ]) |
| |
| def env = "env" |
| timeout(time: 12, unit: 'HOURS') { |
| node() { |
| |
| stage('Setup virtualenv for Pepper') { |
| python.setupPepperVirtualenv(env, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS) |
| } |
| |
| def upgradeTargets = salt.getMinionsSorted(env, TARGET_SERVERS) |
| |
| if (upgradeTargets.isEmpty()) { |
| error("No servers for upgrade matched by ${TARGET_SERVERS}") |
| } |
| |
| def targetSecMapping = [:] |
| def secNoList = [] |
| def out |
| def stopTargets = upgradeTargets.reverse() |
| common.printStageMap(upgradeStageMap) |
| |
| if (interactive){ |
| input message: common.getColorizedString( |
| "Above you can find detailed info this pipeline will execute.\nThe info provides brief description of each stage, actions that will be performed and service/workload impact during each stage.\nPlease read it carefully.", "yellow") |
| } |
| |
| for (target in upgradeTargets) { |
| salt.runSaltProcessStep(env, target, 'saltutil.refresh_pillar', [], null, true) |
| salt.enforceState(env, target, ['linux.system.repo']) |
| common.stageWrapper(upgradeStageMap, "Pre upgrade", target, interactive) { |
| openstack.runOpenStackUpgradePhase(env, target, 'pre') |
| openstack.runOpenStackUpgradePhase(env, target, 'verify') |
| } |
| } |
| |
| if (shutdownCluster){ |
| for (target in stopTargets) { |
| common.stageWrapper(upgradeStageMap, "Stop MySQL service", target, interactive) { |
| openstack.runOpenStackUpgradePhase(env, target, 'service_stopped') |
| } |
| } |
| } |
| |
| for (target in upgradeTargets) { |
| out = salt.cmdRun(env, target, 'cat /var/lib/mysql/grastate.dat | grep "seqno" | cut -d ":" -f2', true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim() |
| common.infoMsg("Get seqno: ${out} for node ${target}") |
| if (!out.isNumber()){ |
| out = -2 |
| } |
| targetSecMapping[out.toInteger()] = target |
| secNoList.add(out.toInteger()) |
| } |
| |
| def masterNode = targetSecMapping[secNoList.max()] |
| common.infoMsg("Master node is: ${masterNode}") |
| |
| // Make sure we start upgrade always from master node |
| upgradeTargets.remove(masterNode) |
| upgradeTargets = [masterNode] + upgradeTargets |
| common.infoMsg("Upgrade targets are: ${upgradeTargets}") |
| |
| for (target in upgradeTargets) { |
| |
| common.stageWrapper(upgradeStageMap, "Stop MySQL service", target, interactive) { |
| openstack.runOpenStackUpgradePhase(env, target, 'service_stopped') |
| } |
| |
| common.stageWrapper(upgradeStageMap, "Upgrade OS", target, interactive) { |
| if (OS_DIST_UPGRADE.toBoolean() == true){ |
| upgrade_mode = 'dist-upgrade' |
| } else if (OS_UPGRADE.toBoolean() == true){ |
| upgrade_mode = 'upgrade' |
| } |
| if (OS_DIST_UPGRADE.toBoolean() == true || OS_UPGRADE.toBoolean() == true) { |
| debian.osUpgradeNode(env, target, upgrade_mode, false) |
| } |
| } |
| |
| common.stageWrapper(upgradeStageMap, "Upgrade MySQL server", target, interactive) { |
| openstack.runOpenStackUpgradePhase(env, target, 'pkgs_latest') |
| openstack.runOpenStackUpgradePhase(env, target, 'render_config') |
| } |
| |
| if (shutdownCluster && target == masterNode){ |
| //Start first node. |
| common.stageWrapper(upgradeStageMap, "Start MySQL service", target, interactive) { |
| galera.startFirstNode(env, target) |
| } |
| } |
| |
| common.stageWrapper(upgradeStageMap, "Start MySQL service", target, interactive) { |
| openstack.runOpenStackUpgradePhase(env, target, 'service_running') |
| openstack.runOpenStackUpgradePhase(env, target, 'verify') |
| } |
| } |
| |
| // restart first node by applying state. |
| |
| if (shutdownCluster) { |
| openstack.runOpenStackUpgradePhase(env, masterNode, 'render_config') |
| salt.cmdRun(env, masterNode, "service mysql reload") |
| openstack.runOpenStackUpgradePhase(env, masterNode, 'verify') |
| } |
| |
| for (target in upgradeTargets) { |
| ensureClusterState = galera.getWsrepParameters(env, target, 'wsrep_evs_state') |
| if (ensureClusterState['wsrep_evs_state'] == 'OPERATIONAL') { |
| common.infoMsg('Node is in OPERATIONAL state.') |
| } else { |
| throw new Exception("Node is NOT in OPERATIONAL state.") |
| } |
| } |
| } |
| } |