Adjust OpenStack upgrade pipelines
Update the following pipelines to use formula based upgrade blocks.
* openstack-control-upgrade.groovy
* openstack-data-upgrade.groovy
Remove redundant pipelines:
* openstack-compute-upgrade.groovy
* ovs-gateway-upgrade.groovy
Related-Prod: PROD-22636
Change-Id: I6e505df2dd15c66946f9e9a770c73e9f897f9b2e
diff --git a/openstack-data-upgrade.groovy b/openstack-data-upgrade.groovy
new file mode 100644
index 0000000..88bbf57
--- /dev/null
+++ b/openstack-data-upgrade.groovy
@@ -0,0 +1,185 @@
+/**
+ * Upgrade OpenStack packages on gateway nodes.
+ * Update packages on given nodes
+ *
+ * Expected parameters:
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
+ * SALT_MASTER_URL Full Salt API address [http://10.10.10.1:8000].
+ * OS_DIST_UPGRADE Upgrade system packages including kernel (apt-get dist-upgrade)
+ * OS_UPGRADE Upgrade all installed applications (apt-get upgrade)
+ * TARGET_SERVERS Comma separated list of salt compound definitions to upgrade.
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
+ *
+ * TODO:
+ * * Add OS_RELEASE_UPGRADE
+**/
+
+def common = new com.mirantis.mk.Common()
+def salt = new com.mirantis.mk.Salt()
+def python = new com.mirantis.mk.Python()
+def openstack = new com.mirantis.mk.Openstack()
+def debian = new com.mirantis.mk.Debian()
+
+def interactive = INTERACTIVE.toBoolean()
+def LinkedHashMap upgradeStageMap = [:]
+
+upgradeStageMap.put('Pre upgrade',
+ [
+ 'Description': 'Only non destructive actions will be applied during this phase. Basic api, service verification will be performed.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * No service downtime
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Verify API, perform basic CRUD operations for services.
+ * Verify that compute/neutron agents on hosts are up.
+ * Run some service built in checkers like keystone-manage doctor or nova-status upgrade.''',
+ 'State result': 'Basic checks around services API are passed.'
+ ])
+upgradeStageMap.put('Upgrade pre: migrate resources',
+ [
+ 'Description': 'In order to minimize workload downtime smooth resource migration is happening during this phase. Neutron agents on node are set to admin_disabled state, to make sure they are quickly migrated to new node (1-2 ping loss). Instances might be live-migrated from host (this stage is optional) and configured from pillar.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * No service downtime
+ * Small workload downtime''',
+ 'Launched actions': '''
+ * Set neutron agents to admin disabled sate
+ * Migrate instances if allowed (optional).''',
+ 'State result': '''
+ * Hosts are being removed from scheduling to host new resources.
+ * If instance migration was performed no instances should be present.'''
+ ])
+upgradeStageMap.put('Upgrade OpenStack',
+ [
+ 'Description': 'OpenStack python code will be upgraded during this stage. No workload downtime is expected.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * OpenStack services might flap
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Install new version of OpenStack packages
+ * Render version of configs
+ * Apply offline dbsync
+ * Start OpenStack services
+ * Verify agents are alive/connected
+ * Run basic API validation''',
+ 'State result': '''
+ * OpenStack packages are upgraded
+ * Services are running
+ * Basic checks around services API are passed
+ * Verified that agents/services on data plane nodes are connected to new control plane
+'''
+ ])
+upgradeStageMap.put('Upgrade OS',
+ [
+ 'Description': 'Optional step. OS packages will be upgraded during this phase, depending on the job parameters dist-upgrade might be called. And reboot of node executed.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * OpenStack services might flap
+ * No workload downtime
+ * The nodes might be rebooted''',
+ 'Launched actions': '''
+ * Install new version of system packages
+ * If doing dist-upgrade new kernel might be installed and node rebooted
+ * Verify agents are alive/connected
+ * Run basic API validation''',
+ 'State result': '''
+ * System packages are updated
+ * Services are running
+ * Basic checks around services API are passed
+ * Verified that agents/services on data plane nodes are connected
+ * Node might be rebooted
+'''
+ ])
+upgradeStageMap.put('Upgrade post: enable resources',
+ [
+ 'Description': 'Verify that agents/services on node are up, add them back to scheduling.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * No service downtime
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Set neutron agents to admin sate enabled
+ * Enable nova-compute services''',
+ 'State result': 'Hosts are being added to scheduling to host new resources',
+ ])
+upgradeStageMap.put('Post upgrade',
+ [
+ 'Description': 'Only non destructive actions will be applied during this phase. Like cleanup old configs, cleanup temporary files. Online dbsyncs.',
+ 'Status': 'NOT_LAUNCHED',
+ 'Expected behaviors': '''
+ * No service downtime
+ * No workload downtime''',
+ 'Launched actions': '''
+ * Cleanup os client configs''',
+ 'State result': 'Temporary resources are being cleaned.'
+ ])
+
+
+def env = "env"
+timeout(time: 24, unit: 'HOURS') {
+ node() {
+
+ stage('Setup virtualenv for Pepper') {
+ python.setupPepperVirtualenv(env, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ def targetNodes = salt.getMinionsSorted(env, TARGET_SERVERS)
+ def migrateResources = true
+
+ if (targetNodes.isEmpty()) {
+ error("No servers for upgrade matched by ${TARGET_SERVERS}")
+ }
+ if (targetNodes.size() == 1 ){
+ migrateResources = false
+ }
+
+ common.printStageMap(upgradeStageMap)
+ if (interactive){
+ input message: common.getColorizedString(
+ "Above you can find detailed info this pipeline will execute.\nThe info provides brief description of each stage, actions that will be performed and service/workload impact during each stage.\nPlease read it carefully.", "yellow")
+ }
+
+ for (target in targetNodes){
+ common.stageWrapper(upgradeStageMap, "Pre upgrade", target, interactive) {
+ openstack.runOpenStackUpgradePhase(env, target, 'pre')
+ openstack.runOpenStackUpgradePhase(env, target, 'verify')
+ }
+
+ common.stageWrapper(upgradeStageMap, "Upgrade pre: migrate resources", target, interactive) {
+ if (migrateResources) {
+ common.infoMsg("Migrating neutron resources from ${target}")
+ openstack.runOpenStackUpgradePhase(env, target, 'upgrade.pre')
+ // Start upgrade only when resources were successfully migrated
+ }
+ }
+
+ common.stageWrapper(upgradeStageMap, "Upgrade OpenStack", target, interactive) {
+ // Stop services on node. //Do actual step by step orch here.
+ openstack.runOpenStackUpgradePhase(env, target, 'service_stopped')
+ openstack.runOpenStackUpgradePhase(env, target, 'pkgs_latest')
+ openstack.runOpenStackUpgradePhase(env, target, 'render_config')
+ openstack.runOpenStackUpgradePhase(env, target, 'service_running')
+ openstack.applyOpenstackAppsStates(env, target)
+ openstack.runOpenStackUpgradePhase(env, target, 'verify')
+ }
+ common.stageWrapper(upgradeStageMap, "Upgrade OS", target, interactive) {
+ if (OS_DIST_UPGRADE.toBoolean() == true){
+ upgrade_mode = 'dist-upgrade'
+ } else if (OS_UPGRADE.toBoolean() == true){
+ upgrade_mode = 'upgrade'
+ }
+ if (OS_DIST_UPGRADE.toBoolean() == true || OS_UPGRADE.toBoolean() == true) {
+ debian.osUpgradeNode(env, target, upgrade_mode, false)
+ }
+ openstack.applyOpenstackAppsStates(env, target)
+ openstack.runOpenStackUpgradePhase(env, target, 'verify')
+ }
+
+ common.stageWrapper(upgradeStageMap, "Upgrade post: enable resources", target, interactive) {
+ openstack.runOpenStackUpgradePhase(env, target, 'upgrade.post')
+ }
+ }
+ }
+}