Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 1 | import pytest |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 2 | import sys |
| 3 | import os |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 4 | |
| 5 | from tcp_tests import logger |
| 6 | from tcp_tests import settings |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 7 | |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 8 | sys.path.append(os.getcwd()) |
| 9 | try: |
| 10 | from tcp_tests.fixtures import config_fixtures |
| 11 | from tcp_tests.managers import underlay_ssh_manager |
| 12 | from tcp_tests.managers import saltmanager as salt_manager |
| 13 | except ImportError: |
| 14 | print("ImportError: Run the application from the tcp-qa directory or " |
| 15 | "set the PYTHONPATH environment variable to directory which contains" |
| 16 | " ./tcp_tests") |
| 17 | sys.exit(1) |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 18 | LOG = logger.logger |
| 19 | |
| 20 | |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 21 | def has_only_similar(values_by_nodes): |
| 22 | """ |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 23 | :param values_by_nodes: dict |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 24 | :return: bool, True if all items in the dict have similar values |
| 25 | """ |
| 26 | values = list(values_by_nodes.values()) |
| 27 | return all(value == values[0] for value in values) |
| 28 | |
| 29 | |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 30 | def get_control_plane_targets(): |
| 31 | config = config_fixtures.config() |
| 32 | underlay = underlay_ssh_manager.UnderlaySSHManager(config) |
| 33 | saltmanager = salt_manager.SaltManager(config, underlay) |
Hanna Arhipova | fbcea85 | 2019-10-03 13:24:53 +0300 | [diff] [blame] | 34 | targets = list() |
Hanna Arhipova | 52bb17d | 2019-10-28 19:15:17 +0200 | [diff] [blame] | 35 | telemetry_exists = False |
| 36 | barbican_exists = False |
Hanna Arhipova | fbcea85 | 2019-10-03 13:24:53 +0300 | [diff] [blame] | 37 | try: |
| 38 | targets += saltmanager.run_state( |
| 39 | "I@keystone:server", 'test.ping')[0]['return'][0].keys() |
| 40 | targets += saltmanager.run_state( |
| 41 | "I@nginx:server and not I@salt:master", |
| 42 | "test.ping")[0]['return'][0].keys() |
Hanna Arhipova | 4462dd0 | 2019-10-28 19:05:08 +0200 | [diff] [blame] | 43 | telemetry_exists = saltmanager.get_single_pillar( |
| 44 | "I@salt:master", |
| 45 | "_param:openstack_telemetry_hostname") |
| 46 | barbican_exists = saltmanager.get_single_pillar( |
| 47 | "I@salt:master", |
| 48 | "_param:barbican_enabled") |
Hanna Arhipova | fbcea85 | 2019-10-03 13:24:53 +0300 | [diff] [blame] | 49 | except BaseException as err: |
| 50 | LOG.warning("Can't retrieve data from Salt. \ |
| 51 | Maybe cluster is not deployed completely.\ |
| 52 | Err: {}".format(err)) |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 53 | |
Hanna Arhipova | 75f21bb | 2019-10-07 19:22:15 +0300 | [diff] [blame] | 54 | # check for Manila existence |
| 55 | # if saltmanager.get_single_pillar("I@salt:master", |
| 56 | # "_param:manila_service_protocol"): |
| 57 | # targets.append('share*') |
| 58 | |
| 59 | # check for Tenant Telemetry existence |
Hanna Arhipova | 4462dd0 | 2019-10-28 19:05:08 +0200 | [diff] [blame] | 60 | if telemetry_exists: |
Hanna Arhipova | 75f21bb | 2019-10-07 19:22:15 +0300 | [diff] [blame] | 61 | targets.append('mdb*') |
| 62 | |
| 63 | # check for Barbican existence |
Hanna Arhipova | 4462dd0 | 2019-10-28 19:05:08 +0200 | [diff] [blame] | 64 | if barbican_exists: |
Hanna Arhipova | 75f21bb | 2019-10-07 19:22:15 +0300 | [diff] [blame] | 65 | targets.append('kmn*') |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 66 | return targets |
| 67 | |
| 68 | |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 69 | @pytest.fixture(scope='class') |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 70 | def update_to_tag(request): |
| 71 | return request.config.getoption("--update-to-tag") |
Hanna Arhipova | 672efc6 | 2021-03-15 09:55:14 +0200 | [diff] [blame] | 72 | |
| 73 | |
| 74 | @pytest.fixture(scope='class') |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 75 | def update_mysql_to_version(request): |
| 76 | return request.config.getoption("--update-mysql-version") |
| 77 | |
| 78 | |
| 79 | @pytest.fixture(scope='class') |
Hanna Arhipova | 672efc6 | 2021-03-15 09:55:14 +0200 | [diff] [blame] | 80 | def switch_to_proposed_pipelines(reclass_actions, salt_actions, |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 81 | update_to_tag): |
| 82 | if update_to_tag: |
Hanna Arhipova | 672efc6 | 2021-03-15 09:55:14 +0200 | [diff] [blame] | 83 | return True |
| 84 | |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 85 | reclass = reclass_actions |
| 86 | proposed_repo = "http://mirror.mirantis.com/update/proposed/" |
| 87 | repo_param = "parameters._param.linux_system_repo_update_url" |
Hanna Arhipova | 7cfeb07 | 2019-10-11 15:04:10 +0300 | [diff] [blame] | 88 | |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 89 | proposed_pipeline_branch = "release/proposed/2019.2.0" |
| 90 | pipeline_branch_param = "parameters._param.jenkins_pipelines_branch" |
| 91 | infra_yml = "cluster/*/infra/init.yml" |
| 92 | |
| 93 | LOG.info("Check reclass has release/proposed/2019.2.0 branches") |
| 94 | if reclass.get_key(pipeline_branch_param, |
| 95 | infra_yml) == proposed_pipeline_branch \ |
| 96 | and reclass.get_key(repo_param, infra_yml) == proposed_repo: |
| 97 | return True |
| 98 | |
| 99 | LOG.info("Switch to release/proposed/2019.2.0 branches") |
| 100 | reclass.add_key(pipeline_branch_param, proposed_pipeline_branch, infra_yml) |
| 101 | |
| 102 | reclass.add_key(repo_param, proposed_repo, infra_yml) |
| 103 | reclass.add_key(repo_param, proposed_repo, "cluster/*/openstack/init.yml") |
| 104 | reclass.add_key(repo_param, proposed_repo, "cluster/*/stacklight/init.yml") |
| 105 | reclass.add_key(repo_param, proposed_repo, "cluster/*/ceph/init.yml") |
Hanna Arhipova | 7cfeb07 | 2019-10-11 15:04:10 +0300 | [diff] [blame] | 106 | |
| 107 | salt_actions.run_state("*", "saltutil.refresh_pillar") |
Hanna Arhipova | 9ee7590 | 2019-10-29 16:33:26 +0200 | [diff] [blame] | 108 | salt_actions.enforce_state("*", "salt.minion") |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 109 | salt_actions.enforce_state("I@jenkins:client", "jenkins.client") |
| 110 | |
| 111 | |
Hanna Arhipova | 745fdbb | 2020-03-30 14:40:44 +0300 | [diff] [blame] | 112 | @pytest.fixture |
Hanna Arhipova | ec201e7 | 2020-10-26 17:14:25 +0200 | [diff] [blame] | 113 | def wa_for_galera_clustercheck_password_prod35705(reclass_actions, |
| 114 | salt_actions): |
Hanna Arhipova | 0e816d2 | 2021-01-29 00:17:44 +0200 | [diff] [blame] | 115 | tgt = "I@galera:master or I@galera:slave" |
| 116 | if not salt_actions.get_pillar(tgt, |
| 117 | "_param:galera_clustercheck_password")[0]: |
| 118 | reclass_actions.add_key( |
| 119 | "parameters._param.galera_clustercheck_password", |
| 120 | "a"*32, |
| 121 | "cluster/*/infra/secrets.yml") |
| 122 | salt_actions.run_state(tgt, "saltutil.refresh_pillar") |
| 123 | salt_actions.enforce_state(tgt, "galera") |
| 124 | salt_actions.enforce_state(tgt, "haproxy") |
| 125 | reclass_actions.commit( |
| 126 | "[from TCP-QA] Add galera_clustercheck_password") |
| 127 | else: |
Hanna Arhipova | 672efc6 | 2021-03-15 09:55:14 +0200 | [diff] [blame] | 128 | |
Hanna Arhipova | 0e816d2 | 2021-01-29 00:17:44 +0200 | [diff] [blame] | 129 | LOG.info("Skipping WA for Galera Clustercheck Password") |
Hanna Arhipova | 745fdbb | 2020-03-30 14:40:44 +0300 | [diff] [blame] | 130 | |
| 131 | |
Hanna Arhipova | 97fac6e | 2020-12-03 13:10:56 +0200 | [diff] [blame] | 132 | @pytest.fixture |
PGlazov | f597729 | 2021-11-25 19:38:27 +0400 | [diff] [blame] | 133 | def wa_cve_2021_20288_global_id_reclaim(reclass_actions, |
| 134 | salt_actions): |
| 135 | tgt = "I@ceph:common" |
| 136 | context_file = "cluster/*/ceph/common.yml" |
| 137 | if not reclass_actions.check_existence( |
Pavel Glazov | 6ed8d44 | 2022-09-20 12:50:18 +0400 | [diff] [blame] | 138 | 'parameters.ceph.common.config.mon.' + |
| 139 | 'auth_allow_insecure_global_id_reclaim'): |
PGlazov | 4b01396 | 2022-01-31 14:01:42 +0400 | [diff] [blame] | 140 | if salt_actions.get_single_pillar( |
| 141 | "I@ceph:mgr and *01.*", |
| 142 | "_param.ceph_version") == 'luminous': |
| 143 | LOG.info('Skip reclaim set') |
| 144 | return |
PGlazov | f597729 | 2021-11-25 19:38:27 +0400 | [diff] [blame] | 145 | reclass_actions.add_bool_key( |
Pavel Glazov | 6ed8d44 | 2022-09-20 12:50:18 +0400 | [diff] [blame] | 146 | 'parameters.ceph.common.config.mon.' + |
| 147 | 'auth_allow_insecure_global_id_reclaim', "False", context_file) |
PGlazov | f597729 | 2021-11-25 19:38:27 +0400 | [diff] [blame] | 148 | salt_actions.run_state(tgt, "state.apply", "ceph.common") |
| 149 | salt_actions.cmd_run(tgt, "systemctl restart ceph-mon.target") |
| 150 | else: |
| 151 | LOG.info("Skipping WA ceph set auth_allow_insecure_global_id_reclaim") |
| 152 | |
| 153 | |
| 154 | @pytest.fixture |
Hanna Arhipova | 97fac6e | 2020-12-03 13:10:56 +0200 | [diff] [blame] | 155 | def wa_for_alerta_password_prod35958(reclass_actions, |
| 156 | salt_actions): |
Pavel Glazov | ea439c4 | 2021-07-25 00:20:20 +0000 | [diff] [blame] | 157 | |
Hanna Arhipova | 0e816d2 | 2021-01-29 00:17:44 +0200 | [diff] [blame] | 158 | if not salt_actions.get_pillar("I@prometheus:alerta", |
| 159 | "_param:alerta_admin_api_key_generated")[0]: |
| 160 | reclass_actions.add_key( |
| 161 | "parameters._param.alerta_admin_api_key_generated", |
| 162 | "a"*32, |
| 163 | "cluster/*/infra/secrets.yml") |
| 164 | reclass_actions.add_key( |
| 165 | "parameters._param.alerta_admin_key", |
| 166 | "${_param:alerta_admin_api_key_generated}", |
| 167 | "cluster/*/stacklight/init.yml") |
| 168 | reclass_actions.commit("[from TCP-QA] Add alerta_admin_key") |
| 169 | salt_actions.run_state( |
| 170 | "I@prometheus:alerta or I@prometheus:alertmanager", |
| 171 | "saltutil.refresh_pillar") |
| 172 | salt_actions.enforce_state( |
| 173 | "I@prometheus:alerta", "prometheus.alerta") |
| 174 | salt_actions.enforce_state( |
| 175 | "I@prometheus:alertmanager", "prometheus.alertmanager") |
| 176 | salt_actions.enforce_state( |
| 177 | "I@prometheus:alerta or I@prometheus:alertmanager", |
| 178 | "docker.client") |
| 179 | else: |
| 180 | LOG.info("Skipping WA for Alerta API key") |
Hanna Arhipova | 97fac6e | 2020-12-03 13:10:56 +0200 | [diff] [blame] | 181 | |
| 182 | |
Vladimir Khlyunev | 5fe0aaa | 2022-08-16 23:13:22 +0400 | [diff] [blame] | 183 | @pytest.fixture |
| 184 | def wa_redis_server_version_3_prod36960(reclass_actions, salt_actions): |
| 185 | version_pillar = "redis:server:version" |
| 186 | tgt = "I@{}".format(version_pillar) |
| 187 | pillar_file = "cluster/*/openstack/telemetry.yml" |
| 188 | redis_target_version = "5.0" |
| 189 | |
| 190 | msg_nodes = salt_actions.get_minions_by_target(tgt) |
| 191 | if not msg_nodes: |
| 192 | LOG.info("dbs* nodes not found, skipping") |
| 193 | return |
| 194 | else: |
| 195 | msg_node = msg_nodes[0] |
| 196 | |
| 197 | if str(salt_actions.get_single_pillar( |
| 198 | msg_node, |
| 199 | version_pillar)) == redis_target_version: |
| 200 | LOG.info("Redis-server is already '{}', skipping".format( |
| 201 | redis_target_version)) |
| 202 | return |
| 203 | |
| 204 | LOG.info("Setting redis-server version to '{}'".format( |
| 205 | redis_target_version)) |
| 206 | reclass_actions.add_key( |
| 207 | "parameters.{}".format(version_pillar.replace(":", ".")), |
| 208 | redis_target_version, |
| 209 | pillar_file) |
| 210 | salt_actions.run_state(" or ".join(msg_nodes), |
| 211 | "saltutil.refresh_pillar") |
| 212 | |
| 213 | |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 214 | @pytest.fixture(scope='class') |
| 215 | def enable_openstack_update(reclass_actions, salt_actions): |
| 216 | param = "parameters._param.openstack_upgrade_enabled" |
| 217 | context_file = "cluster/*/infra/init.yml" |
| 218 | |
| 219 | LOG.info("Enable openstack_upgrade_enabled in reclass") |
| 220 | reclass_actions.add_bool_key(param, "True", context_file) |
| 221 | salt_actions.run_state("*", "saltutil.refresh_pillar") |
| 222 | yield True |
| 223 | LOG.info("Disable openstack_upgrade_enabled in reclass") |
| 224 | reclass_actions.add_bool_key(param, "False", context_file) |
| 225 | salt_actions.run_state("*", "saltutil.refresh_pillar") |
| 226 | |
| 227 | |
Vladimir Khlyunev | 0ca3ba8 | 2022-10-31 12:12:30 +0400 | [diff] [blame] | 228 | @pytest.fixture(scope='class') |
Pavel Glazov | 6ee164c | 2022-12-12 06:42:28 +0400 | [diff] [blame] | 229 | def switch_to_staging_updates_mirantis_repo(update_to_tag, |
| 230 | reclass_actions, |
| 231 | salt_actions): |
Vladimir Khlyunev | 0ca3ba8 | 2022-10-31 12:12:30 +0400 | [diff] [blame] | 232 | version_param = "parameters._param.updates_mirantis_version" |
| 233 | infra_yml = "cluster/*/infra/init.yml" |
Vladimir Khlyunev | 0ca3ba8 | 2022-10-31 12:12:30 +0400 | [diff] [blame] | 234 | current_version = reclass_actions.get_key(version_param, infra_yml) |
Pavel Glazov | 6ee164c | 2022-12-12 06:42:28 +0400 | [diff] [blame] | 235 | |
| 236 | if update_to_tag is not None: |
| 237 | reclass_actions.add_key(version_param, update_to_tag, infra_yml) |
Vladimir Khlyunev | 0ca3ba8 | 2022-10-31 12:12:30 +0400 | [diff] [blame] | 238 | if current_version == 'staging': |
| 239 | LOG.info("updates.mirantis.com already set to 'staging', skipping") |
| 240 | return |
| 241 | if current_version is None: |
| 242 | LOG.info("updates.mirantis.com is not enabled, " |
| 243 | "skipping /staging repo pillar") |
| 244 | return |
Pavel Glazov | 6ee164c | 2022-12-12 06:42:28 +0400 | [diff] [blame] | 245 | if current_version is not None: |
| 246 | LOG.info("updates.mirantis.com is other tag " |
| 247 | "skipping /staging repo pillar") |
| 248 | salt_actions.run_state("*", "saltutil.refresh_pillar") |
| 249 | return |
Vladimir Khlyunev | 0ca3ba8 | 2022-10-31 12:12:30 +0400 | [diff] [blame] | 250 | |
| 251 | LOG.info("Switch to updates.mirantis.com/staging repo") |
| 252 | reclass_actions.add_key(version_param, 'staging', infra_yml) |
| 253 | |
| 254 | salt_actions.run_state("*", "saltutil.refresh_pillar") |
| 255 | |
| 256 | |
Hanna Arhipova | 745fdbb | 2020-03-30 14:40:44 +0300 | [diff] [blame] | 257 | @pytest.mark.usefixtures("switch_to_proposed_pipelines", |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 258 | "update_to_tag", |
Hanna Arhipova | 97fac6e | 2020-12-03 13:10:56 +0200 | [diff] [blame] | 259 | "wa_for_galera_clustercheck_password_prod35705", |
Vladimir Khlyunev | 5fe0aaa | 2022-08-16 23:13:22 +0400 | [diff] [blame] | 260 | "wa_for_alerta_password_prod35958", |
Vladimir Khlyunev | 0ca3ba8 | 2022-10-31 12:12:30 +0400 | [diff] [blame] | 261 | "wa_redis_server_version_3_prod36960", |
| 262 | "switch_to_staging_updates_mirantis_repo") |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 263 | class TestUpdateMcpCluster(object): |
| 264 | """ |
| 265 | Following the steps in |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 266 | https://docs.mirantis.com/mcp/master/mcp-operations-guide/update-upgrade/minor-update.html#minor-update |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 267 | """ |
| 268 | |
| 269 | @pytest.mark.grab_versions |
| 270 | @pytest.mark.parametrize("_", [settings.ENV_NAME]) |
| 271 | @pytest.mark.run_mcp_update |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 272 | def test_update_drivetrain(self, salt_actions, drivetrain_actions, |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 273 | show_step, update_to_tag, _): |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 274 | """Updating DriveTrain component to release/proposed/2019.2.0 version |
| 275 | |
| 276 | Scenario: |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 277 | 1. Add workaround for PROD-32751 |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 278 | 2. Run job git-mirror-downstream-mk-pipelines |
| 279 | 3. Run job git-mirror-downstream-pipeline-library |
dtsapikov | cbff233 | 2022-01-19 19:32:22 +0400 | [diff] [blame] | 280 | 4. Run job pre-upgrade-verify |
| 281 | 5. If jobs are passed then start 'Deploy - upgrade MCP Drivetrain' |
| 282 | 6. Wait till salt-master completes all async jobs |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 283 | |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 284 | Duration: ~70 min |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 285 | """ |
| 286 | salt = salt_actions |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 287 | dt = drivetrain_actions |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 288 | |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 289 | # #################### Add workaround for PROD-32751 ################# |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 290 | show_step(1) |
| 291 | |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 292 | # FIXME: workaround for PROD-32751 |
| 293 | salt.cmd_run("cfg01*", "cd /srv/salt/reclass; git add -u && \ |
| 294 | git commit --allow-empty -m 'Cluster model update'") |
| 295 | |
| 296 | # ################### Downstream mk-pipelines ######################### |
| 297 | show_step(2) |
| 298 | job_name = 'git-mirror-downstream-mk-pipelines' |
| 299 | job_parameters = { |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 300 | 'BRANCHES': '*' or |
| 301 | 'release/proposed/2019.2.0' |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 302 | } |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 303 | job_result, job_description = dt.start_job_on_jenkins( |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 304 | job_name=job_name, |
Hanna Arhipova | c3c5929 | 2020-04-23 16:29:09 +0300 | [diff] [blame] | 305 | job_parameters=job_parameters, |
| 306 | verbose=True) |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 307 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 308 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 309 | |
| 310 | # ################### Downstream pipeline-library #################### |
| 311 | show_step(3) |
| 312 | job_name = 'git-mirror-downstream-pipeline-library' |
| 313 | job_parameters = { |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 314 | 'BRANCHES': '*' or |
| 315 | 'release/proposed/2019.2.0' |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 316 | } |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 317 | job_result, job_description = dt.start_job_on_jenkins( |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 318 | job_name=job_name, |
Hanna Arhipova | c3c5929 | 2020-04-23 16:29:09 +0300 | [diff] [blame] | 319 | job_parameters=job_parameters, |
| 320 | verbose=True) |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 321 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 322 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 323 | |
dtsapikov | cbff233 | 2022-01-19 19:32:22 +0400 | [diff] [blame] | 324 | # ############### Start 'Pre-upgrade verify' job (since 2019.2.17)##### |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 325 | show_step(4) |
dtsapikov | cbff233 | 2022-01-19 19:32:22 +0400 | [diff] [blame] | 326 | job_name = 'pre-upgrade-verify' |
| 327 | job_parameters = {} |
| 328 | |
Pavel Glazov | 794b1a9 | 2022-10-03 16:33:04 +0400 | [diff] [blame] | 329 | if (dt.check_job_exists_on_jenkins(job_name)): |
dtsapikov | cbff233 | 2022-01-19 19:32:22 +0400 | [diff] [blame] | 330 | LOG.info("Pre-upgrade verify job exists on Jenkins!") |
| 331 | job_result, job_description = dt.start_job_on_jenkins( |
| 332 | job_name=job_name, |
| 333 | job_parameters=job_parameters, |
| 334 | verbose=True, |
| 335 | build_timeout=4 * 60 * 60) |
| 336 | |
| 337 | assert job_result == 'SUCCESS', job_description |
| 338 | |
| 339 | # ################### Start 'Deploy - upgrade MCP Drivetrain' job ##### |
| 340 | show_step(5) |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 341 | |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 342 | job_name = 'upgrade-mcp-release' |
| 343 | job_parameters = { |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 344 | 'GIT_REFSPEC': update_to_tag or 'release/proposed/2019.2.0', |
PGlazov | 1df9e54 | 2021-11-16 02:31:49 +0400 | [diff] [blame] | 345 | 'MK_PIPELINES_REFSPEC': |
| 346 | update_to_tag or 'release/proposed/2019.2.0', |
Hanna Arhipova | e1dd27a | 2021-07-28 12:40:40 +0300 | [diff] [blame] | 347 | 'TARGET_MCP_VERSION': update_to_tag or '2019.2.0', |
Hanna Arhipova | 874c68f | 2021-03-29 15:57:19 +0300 | [diff] [blame] | 348 | "DRIVE_TRAIN_PARAMS": { |
| 349 | "OS_DIST_UPGRADE": True, |
| 350 | "OS_UPGRADE": True, |
| 351 | "BATCH_SIZE": 10 |
| 352 | } |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 353 | } |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 354 | job_result, job_description = dt.start_job_on_jenkins( |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 355 | job_name=job_name, |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 356 | job_parameters=job_parameters, |
Hanna Arhipova | c3c5929 | 2020-04-23 16:29:09 +0300 | [diff] [blame] | 357 | verbose=True, |
Hanna Arhipova | e7f2e32 | 2021-02-25 13:53:18 +0200 | [diff] [blame] | 358 | build_timeout=4 * 60 * 60) |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 359 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 360 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | 7a593da | 2021-06-25 11:16:09 +0300 | [diff] [blame] | 361 | # ############### Wait till salt-master completes all async jobs ##### |
| 362 | show_step(5) |
| 363 | salt.wait_jobs_completed(timeout=20*60) |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 364 | |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 365 | @pytest.mark.grab_versions |
| 366 | @pytest.mark.parametrize("_", [settings.ENV_NAME]) |
Hanna Arhipova | 71ecc27 | 2019-08-20 14:54:22 +0300 | [diff] [blame] | 367 | @pytest.mark.run_mcp_update |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 368 | def test_update_glusterfs(self, salt_actions, reclass_actions, |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 369 | drivetrain_actions, show_step, |
| 370 | update_mysql_to_version, _): |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 371 | """ Upgrade GlusterFS |
| 372 | Scenario: |
| 373 | 1. In infra/init.yml in Reclass, add the glusterfs_version parameter |
| 374 | 2. Start linux.system.repo state |
| 375 | 3. Start "update-glusterfs" job |
| 376 | 4. Check version for GlusterFS servers |
| 377 | 5. Check version for GlusterFS clients |
| 378 | |
| 379 | """ |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 380 | |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 381 | salt = salt_actions |
| 382 | reclass = reclass_actions |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 383 | dt = drivetrain_actions |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 384 | |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 385 | # ############## Change reclass ###################################### |
| 386 | show_step(1) |
| 387 | reclass.add_key( |
| 388 | "parameters._param.linux_system_repo_mcp_glusterfs_version_number", |
| 389 | "5", |
| 390 | "cluster/*/infra/init.yml" |
| 391 | ) |
| 392 | # ################# Run linux.system state ########################### |
| 393 | show_step(2) |
| 394 | salt.enforce_state("*", "linux.system.repo") |
| 395 | |
| 396 | # ############## Start deploy-upgrade-galera job ##################### |
| 397 | show_step(3) |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 398 | job_name = 'update-glusterfs' |
| 399 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 400 | job_result, job_description = dt.start_job_on_jenkins( |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 401 | job_name=job_name, |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 402 | build_timeout=40 * 60) |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 403 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 404 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 405 | |
| 406 | # ################ Check GlusterFS version for servers ############## |
| 407 | show_step(4) |
| 408 | gluster_server_versions_by_nodes = salt.cmd_run( |
| 409 | "I@glusterfs:server", |
| 410 | "glusterd --version|head -n1")[0] |
| 411 | |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 412 | assert has_only_similar(gluster_server_versions_by_nodes), \ |
Hanna Arhipova | cc3759b | 2019-08-28 16:01:11 +0300 | [diff] [blame] | 413 | gluster_server_versions_by_nodes |
| 414 | |
| 415 | # ################ Check GlusterFS version for clients ############## |
| 416 | show_step(5) |
| 417 | gluster_client_versions_by_nodes = salt.cmd_run( |
| 418 | "I@glusterfs:client", |
| 419 | "glusterfs --version|head -n1")[0] |
| 420 | |
| 421 | assert has_only_similar(gluster_client_versions_by_nodes), \ |
| 422 | gluster_client_versions_by_nodes |
| 423 | |
| 424 | @pytest.mark.grab_versions |
| 425 | @pytest.mark.parametrize("_", [settings.ENV_NAME]) |
| 426 | @pytest.mark.run_mcp_update |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 427 | def test_update_galera(self, salt_actions, reclass_actions, |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 428 | drivetrain_actions, show_step, |
| 429 | update_mysql_to_version, _): |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 430 | """ Upgrade Galera automatically |
| 431 | |
| 432 | Scenario: |
| 433 | 1. Include the Galera upgrade pipeline job to DriveTrain |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 434 | 2. Add fix for mysql 5.7 |
| 435 | 3. Apply the jenkins.client state on the Jenkins nodes |
| 436 | 4. set the openstack_upgrade_enabled parameter to true |
| 437 | 5. Refresh pillars |
| 438 | 6. Add repositories with new Galera packages |
| 439 | 7. Start job from Jenkins |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 440 | """ |
| 441 | salt = salt_actions |
| 442 | reclass = reclass_actions |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 443 | dt = drivetrain_actions |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 444 | # ################### Enable pipeline ################################# |
| 445 | show_step(1) |
| 446 | reclass.add_class( |
| 447 | "system.jenkins.client.job.deploy.update.upgrade_galera", |
| 448 | "cluster/*/cicd/control/leader.yml") |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 449 | # ################### Fix for Mysql 5.7 ############################## |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 450 | show_step(2) |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 451 | if (update_mysql_to_version == '5.7'): |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 452 | mysql_class = "cluster.{env_name}.openstack." \ |
| 453 | "database.mysql_version"\ |
| 454 | .format(env_name=settings.ENV_NAME) |
| 455 | reclass.add_class( |
| 456 | mysql_class, |
| 457 | "cluster/*/openstack/database/master.yml") |
| 458 | reclass.add_class( |
| 459 | mysql_class, |
| 460 | "cluster/*/openstack/database/slave.yml") |
| 461 | mysql_path = "cluster/{env_name}" \ |
| 462 | "/openstack/database/mysql_version.yml"\ |
| 463 | .format(env_name=settings.ENV_NAME) |
| 464 | galera_mysql_version = '5.7' |
| 465 | if update_mysql_to_version != '5.7': |
| 466 | galera_mysql_version = update_mysql_to_version |
| 467 | mysql_version = "parameters:\n" \ |
| 468 | " _param:\n" \ |
| 469 | " galera_mysql_version: {}"\ |
| 470 | .format(galera_mysql_version) |
| 471 | if not os.path.isfile("/srv/salt/reclass/classes/{path}".format( |
| 472 | path=mysql_path)): |
| 473 | reclass.create_yaml_with_context(mysql_version, |
| 474 | mysql_path) |
| 475 | else: |
| 476 | reclass.merge_context(mysql_version, |
| 477 | mysql_path) |
| 478 | reclass_actions.commit( |
| 479 | "[from TCP-QA] Change for mysql new version") |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 480 | |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 481 | show_step(3) |
Vladimir Khlyunev | 1746827 | 2022-06-29 16:19:36 +0400 | [diff] [blame] | 482 | salt.run_state("I@jenkins:client and not I@salt:master", |
| 483 | "saltutil.refresh_pillar") |
| 484 | salt.enforce_state("I@jenkins:client and not I@salt:master", |
| 485 | "jenkins.client.job") |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 486 | else: |
| 487 | LOG.info("Skip mysql upgrade fix") |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 488 | # ############### Enable automatic upgrade ############################ |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 489 | show_step(4) |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 490 | reclass.add_bool_key("parameters._param.openstack_upgrade_enabled", |
| 491 | "True", |
| 492 | "cluster/*/infra/init.yml") |
| 493 | |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 494 | show_step(5) |
Hanna Arhipova | 7cfeb07 | 2019-10-11 15:04:10 +0300 | [diff] [blame] | 495 | salt.run_state("dbs*", "saltutil.refresh_pillar") |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 496 | |
| 497 | # ############# Add repositories with new Galera packages ####### |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 498 | show_step(6) |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 499 | salt.enforce_state("dbs*", "linux.system.repo") |
| 500 | salt.enforce_state("cfg*", "salt.master") |
| 501 | |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 502 | # #################### Login Jenkins on cid01 node ################### |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 503 | show_step(7) |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 504 | |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 505 | job_name = 'deploy-upgrade-galera' |
| 506 | job_parameters = { |
Hanna Arhipova | 874c68f | 2021-03-29 15:57:19 +0300 | [diff] [blame] | 507 | 'INTERACTIVE': 'false', |
| 508 | 'OS_DIST_UPGRADE': 'true', |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 509 | 'OS_UPGRADE': 'true', |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 510 | } |
PGlazov | 554c4fd | 2022-03-23 14:31:41 +0400 | [diff] [blame] | 511 | if (update_mysql_to_version == '5.7'): |
PGlazov | 7efe3b2 | 2022-02-16 17:29:05 +0400 | [diff] [blame] | 512 | job_parameters.update({'UPDATE_TO_MYSQL57': 'true'}) |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 513 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 514 | job_result, job_description = dt.start_job_on_jenkins( |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 515 | job_name=job_name, |
Hanna Arhipova | 17b2c10 | 2019-09-06 16:44:17 +0300 | [diff] [blame] | 516 | job_parameters=job_parameters, |
Pavel Glazov | 2e9cbcd | 2022-12-06 14:52:53 +0400 | [diff] [blame] | 517 | build_timeout=60 * 60) |
Hanna Arhipova | 94a8abe | 2019-08-22 14:11:46 +0300 | [diff] [blame] | 518 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 519 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 520 | |
| 521 | @pytest.fixture |
| 522 | def disable_automatic_failover_neutron_for_test(self, salt_actions): |
| 523 | """ |
| 524 | On each OpenStack controller node, modify the neutron.conf file |
| 525 | Restart the neutron-server service |
| 526 | """ |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 527 | |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 528 | def comment_line(node, file_name, word): |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 529 | """ |
| 530 | Adds '#' before the specific line in specific file |
| 531 | |
| 532 | :param node: string, salt target of node where the file locates |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 533 | :param file_name: string, full path to the file |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 534 | :param word: string, the begin of line which should be commented |
| 535 | :return: None |
| 536 | """ |
| 537 | salt_actions.cmd_run(node, |
| 538 | "sed -i 's/^{word}/#{word}/' {file}". |
| 539 | format(word=word, |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 540 | file=file_name)) |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 541 | |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 542 | def add_line(node, file_name, line): |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 543 | """ |
| 544 | Appends line to the end of file |
| 545 | |
| 546 | :param node: string, salt target of node where the file locates |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 547 | :param file_name: string, full path to the file |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 548 | :param line: string, line that should be added |
| 549 | :return: None |
| 550 | """ |
| 551 | salt_actions.cmd_run(node, "echo {line} >> {file}".format( |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 552 | line=line, |
| 553 | file=file_name)) |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 554 | |
| 555 | neutron_conf = '/etc/neutron/neutron.conf' |
| 556 | neutron_server = "I@neutron:server" |
| 557 | # ######## Create backup for config file ####################### |
| 558 | salt_actions.cmd_run( |
| 559 | neutron_server, |
| 560 | "cp -p {file} {file}.backup".format(file=neutron_conf)) |
| 561 | |
| 562 | # ## Change parameters in neutron.conf' |
| 563 | comment_line(neutron_server, neutron_conf, |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 564 | "allow_automatic_l3agent_failover", ) |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 565 | comment_line(neutron_server, neutron_conf, |
| 566 | "allow_automatic_dhcp_failover") |
| 567 | add_line(neutron_server, neutron_conf, |
| 568 | "allow_automatic_dhcp_failover = false") |
| 569 | add_line(neutron_server, neutron_conf, |
| 570 | "allow_automatic_l3agent_failover = false") |
| 571 | |
| 572 | # ## Apply changed config to the neutron-server service |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 573 | result = salt_actions.cmd_run(neutron_server, |
| 574 | "service neutron-server restart") |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 575 | # TODO: add check that neutron-server is up and running |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 576 | yield result |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 577 | # ## Revert file changes |
| 578 | salt_actions.cmd_run( |
| 579 | neutron_server, |
| 580 | "cp -p {file}.backup {file}".format(file=neutron_conf)) |
| 581 | salt_actions.cmd_run(neutron_server, |
| 582 | "service neutron-server restart") |
| 583 | |
| 584 | @pytest.fixture |
| 585 | def disable_neutron_agents_for_test(self, salt_actions): |
| 586 | """ |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 587 | Disable the neutron services before the test and |
| 588 | enable it after test |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 589 | """ |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 590 | result = salt_actions.cmd_run("I@neutron:server", """ |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 591 | service neutron-dhcp-agent stop && \ |
| 592 | service neutron-l3-agent stop && \ |
| 593 | service neutron-metadata-agent stop && \ |
| 594 | service neutron-openvswitch-agent stop |
| 595 | """) |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 596 | yield result |
| 597 | # |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 598 | salt_actions.cmd_run("I@neutron:server", """ |
| 599 | service neutron-dhcp-agent start && \ |
| 600 | service neutron-l3-agent start && \ |
| 601 | service neutron-metadata-agent start && \ |
| 602 | service neutron-openvswitch-agent start |
| 603 | """) |
| 604 | # TODO: add check that all services are UP and running |
| 605 | |
| 606 | @pytest.mark.grab_versions |
| 607 | @pytest.mark.parametrize("_", [settings.ENV_NAME]) |
| 608 | @pytest.mark.run_mcp_update |
| 609 | def test_update_rabbit(self, salt_actions, reclass_actions, |
| 610 | drivetrain_actions, show_step, _, |
| 611 | disable_automatic_failover_neutron_for_test, |
| 612 | disable_neutron_agents_for_test): |
| 613 | """ Updates RabbitMQ |
| 614 | Scenario: |
| 615 | 1. Include the RabbitMQ upgrade pipeline job to DriveTrain |
| 616 | 2. Add repositories with new RabbitMQ packages |
| 617 | 3. Start Deploy - upgrade RabbitMQ pipeline |
| 618 | |
| 619 | Updating RabbitMq should be completed before the OpenStack updating |
| 620 | process starts |
| 621 | """ |
| 622 | salt = salt_actions |
| 623 | reclass = reclass_actions |
| 624 | dt = drivetrain_actions |
| 625 | |
| 626 | # ####### Include the RabbitMQ upgrade pipeline job to DriveTrain #### |
| 627 | show_step(1) |
| 628 | reclass.add_class( |
| 629 | "system.jenkins.client.job.deploy.update.upgrade_rabbitmq", |
| 630 | "cluster/*/cicd/control/leader.yml") |
| 631 | salt.enforce_state("I@jenkins:client", "jenkins.client") |
| 632 | |
| 633 | reclass.add_bool_key("parameters._param.openstack_upgrade_enabled", |
| 634 | "True", |
| 635 | "cluster/*/infra/init.yml") |
| 636 | salt.run_state("I@rabbitmq:server", "saltutil.refresh_pillar") |
| 637 | |
| 638 | # ########### Add repositories with new RabbitMQ packages ############ |
| 639 | show_step(2) |
| 640 | salt.enforce_state("I@rabbitmq:server", "linux.system.repo") |
| 641 | |
| 642 | # ########### Start Deploy - upgrade RabbitMQ pipeline ############ |
| 643 | show_step(3) |
| 644 | job_parameters = { |
Hanna Arhipova | 874c68f | 2021-03-29 15:57:19 +0300 | [diff] [blame] | 645 | 'INTERACTIVE': 'false', |
| 646 | 'OS_DIST_UPGRADE': 'true', |
| 647 | 'OS_UPGRADE': 'true' |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 648 | } |
| 649 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 650 | job_result, job_description = dt.start_job_on_jenkins( |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 651 | job_name='deploy-upgrade-rabbitmq', |
| 652 | job_parameters=job_parameters, |
PGlazov | 99364cb | 2021-09-20 12:36:18 +0400 | [diff] [blame] | 653 | build_timeout=50 * 60 |
Hanna Arhipova | 1fcaf44 | 2019-09-06 15:30:45 +0300 | [diff] [blame] | 654 | ) |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 655 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 656 | |
| 657 | @pytest.mark.grab_versions |
| 658 | @pytest.mark.parametrize("_", [settings.ENV_NAME]) |
| 659 | @pytest.mark.run_mcp_update |
PGlazov | 215a457 | 2022-03-05 05:09:04 +0400 | [diff] [blame] | 660 | def test_update_ceph(self, salt_actions, drivetrain_actions, |
| 661 | reclass_actions, show_step, _): |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 662 | """ Updates Ceph to the latest minor version |
| 663 | |
| 664 | Scenario: |
| 665 | 1. Add workaround for unhealth Ceph |
| 666 | 2. Start ceph-upgrade job with default parameters |
| 667 | 3. Check Ceph version for all nodes |
PGlazov | 215a457 | 2022-03-05 05:09:04 +0400 | [diff] [blame] | 668 | 4. WA id_reclaim only for nautilus |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 669 | |
| 670 | https://docs.mirantis.com/mcp/master/mcp-operations-guide/update-upgrade/minor-update/ceph-update.html |
| 671 | """ |
| 672 | salt = salt_actions |
PGlazov | 215a457 | 2022-03-05 05:09:04 +0400 | [diff] [blame] | 673 | reclass = reclass_actions |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 674 | dt = drivetrain_actions |
| 675 | |
| 676 | # ###################### Add workaround for unhealth Ceph ############ |
| 677 | show_step(1) |
| 678 | salt.cmd_run("I@ceph:radosgw", |
| 679 | "ceph config set 'mon pg warn max object skew' 20") |
| 680 | # ###################### Start ceph-upgrade pipeline ################# |
| 681 | show_step(2) |
| 682 | job_parameters = {} |
| 683 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 684 | job_result, job_description = dt.start_job_on_jenkins( |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 685 | job_name='ceph-update', |
| 686 | job_parameters=job_parameters) |
| 687 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 688 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 689 | |
| 690 | # ########## Verify Ceph version ##################################### |
| 691 | show_step(3) |
| 692 | |
| 693 | ceph_version_by_nodes = salt.cmd_run( |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 694 | "I@ceph:* and not I@ceph:monitoring and not I@ceph:backup:server", |
| 695 | "ceph version")[0] |
Hanna Arhipova | d35a29b | 2019-09-04 13:24:06 +0300 | [diff] [blame] | 696 | |
| 697 | assert has_only_similar(ceph_version_by_nodes), ceph_version_by_nodes |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 698 | |
PGlazov | 215a457 | 2022-03-05 05:09:04 +0400 | [diff] [blame] | 699 | # ########## WA id_reclaim only for nautilus ######################### |
| 700 | show_step(4) |
| 701 | |
| 702 | ceph_version = salt_actions.get_single_pillar( |
| 703 | "I@salt:master", |
| 704 | "_param:_param:ceph_version") |
| 705 | if ceph_version == "nautilus": |
| 706 | wa_cve_2021_20288_global_id_reclaim(reclass, salt) |
| 707 | else: |
| 708 | LOG.info("Skip WA for luminous") |
| 709 | |
Hanna Arhipova | ac00856 | 2019-10-17 11:54:23 +0300 | [diff] [blame] | 710 | @pytest.mark.grab_versions |
| 711 | @pytest.mark.parametrize("_", [settings.ENV_NAME]) |
| 712 | @pytest.mark.run_mcp_update |
| 713 | def test_update_stacklight(self, _, drivetrain_actions): |
| 714 | """ Update packages for Stacklight |
| 715 | Scenario: |
| 716 | 1. Start Deploy - upgrade Stacklight job |
| 717 | """ |
| 718 | drivetrain = drivetrain_actions |
| 719 | |
| 720 | job_parameters = { |
| 721 | "STAGE_UPGRADE_DOCKER_COMPONENTS": True, |
| 722 | "STAGE_UPGRADE_ES_KIBANA": True, |
Hanna Arhipova | 874c68f | 2021-03-29 15:57:19 +0300 | [diff] [blame] | 723 | "STAGE_UPGRADE_SYSTEM_PART": True, |
| 724 | 'OS_DIST_UPGRADE': 'true', |
| 725 | 'OS_UPGRADE': 'true' |
Hanna Arhipova | ac00856 | 2019-10-17 11:54:23 +0300 | [diff] [blame] | 726 | } |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 727 | job_result, job_description = drivetrain.start_job_on_jenkins( |
Hanna Arhipova | ac00856 | 2019-10-17 11:54:23 +0300 | [diff] [blame] | 728 | job_name="stacklight-upgrade", |
| 729 | job_parameters=job_parameters) |
| 730 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 731 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | ac00856 | 2019-10-17 11:54:23 +0300 | [diff] [blame] | 732 | |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 733 | |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 734 | @pytest.mark.usefixtures("switch_to_proposed_pipelines", |
| 735 | "enable_openstack_update") |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 736 | class TestOpenstackUpdate(object): |
| 737 | |
| 738 | @pytest.mark.grab_versions |
| 739 | @pytest.mark.run_mcp_update |
| 740 | def test__pre_update__enable_pipeline_job(self, |
| 741 | reclass_actions, salt_actions, |
| 742 | show_step): |
| 743 | """ Enable pipeline in the Drivetrain |
| 744 | |
| 745 | Scenario: |
| 746 | 1. Add deploy.update.* classes to the reclass |
| 747 | 2. Start jenkins.client salt state |
| 748 | |
| 749 | """ |
| 750 | salt = salt_actions |
| 751 | reclass = reclass_actions |
| 752 | show_step(1) |
| 753 | reclass.add_class("system.jenkins.client.job.deploy.update.upgrade", |
| 754 | "cluster/*/cicd/control/leader.yml") |
| 755 | |
| 756 | reclass.add_class( |
| 757 | "system.jenkins.client.job.deploy.update.upgrade_ovs_gateway", |
| 758 | "cluster/*/cicd/control/leader.yml") |
| 759 | |
| 760 | reclass.add_class( |
| 761 | "system.jenkins.client.job.deploy.update.upgrade_compute", |
| 762 | "cluster/*/cicd/control/leader.yml") |
| 763 | |
| 764 | show_step(2) |
| 765 | r, errors = salt.enforce_state("I@jenkins:client", "jenkins.client") |
| 766 | assert errors is None |
| 767 | |
| 768 | @pytest.mark.grab_versions |
| 769 | @pytest.mark.parametrize('target', get_control_plane_targets()) |
| 770 | @pytest.mark.run_mcp_update |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 771 | def test__update__control_plane(self, drivetrain_actions, target): |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 772 | """Start 'Deploy - upgrade control VMs' for specific node |
| 773 | """ |
| 774 | job_parameters = { |
| 775 | "TARGET_SERVERS": target, |
Hanna Arhipova | 606d518 | 2019-12-13 13:23:55 +0200 | [diff] [blame] | 776 | "OS_DIST_UPGRADE": True, |
PGlazov | a503925 | 2021-10-06 21:40:01 +0400 | [diff] [blame] | 777 | "UPGRADE_SALTSTACK": True, |
Hanna Arhipova | 606d518 | 2019-12-13 13:23:55 +0200 | [diff] [blame] | 778 | "OS_UPGRADE": True, |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 779 | "INTERACTIVE": False} |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 780 | job_result, job_description = drivetrain_actions.start_job_on_jenkins( |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 781 | job_name="deploy-upgrade-control", |
| 782 | job_parameters=job_parameters) |
| 783 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 784 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 785 | |
| 786 | @pytest.mark.grab_versions |
| 787 | @pytest.mark.run_mcp_update |
Hanna Arhipova | 606d518 | 2019-12-13 13:23:55 +0200 | [diff] [blame] | 788 | def test__update__data_plane(self, drivetrain_actions, salt_actions): |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 789 | """Start 'Deploy - upgrade OVS gateway' |
| 790 | """ |
Hanna Arhipova | 606d518 | 2019-12-13 13:23:55 +0200 | [diff] [blame] | 791 | if not salt_actions.cmd_run("gtw*", "test.ping")[0].keys(): |
| 792 | pytest.skip("This deployment doesn't have gtw* nodes") |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 793 | job_parameters = { |
Hanna Arhipova | 606d518 | 2019-12-13 13:23:55 +0200 | [diff] [blame] | 794 | "OS_DIST_UPGRADE": True, |
| 795 | "OS_UPGRADE": True, |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 796 | "INTERACTIVE": False} |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 797 | job_result, job_description = drivetrain_actions.start_job_on_jenkins( |
Hanna Arhipova | eb3a211 | 2019-09-13 18:45:21 +0300 | [diff] [blame] | 798 | job_name="deploy-upgrade-ovs-gateway", |
| 799 | job_parameters=job_parameters) |
| 800 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 801 | assert job_result == 'SUCCESS', job_description |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 802 | |
| 803 | @pytest.mark.grab_versions |
| 804 | @pytest.mark.run_mcp_update |
| 805 | def test__update__computes(self, drivetrain_actions): |
| 806 | """Start 'Deploy - upgrade computes' |
| 807 | """ |
| 808 | job_parameters = { |
Hanna Arhipova | 606d518 | 2019-12-13 13:23:55 +0200 | [diff] [blame] | 809 | "OS_DIST_UPGRADE": True, |
| 810 | "OS_UPGRADE": True, |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 811 | "INTERACTIVE": False} |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 812 | job_result, job_description = drivetrain_actions.start_job_on_jenkins( |
Hanna Arhipova | 1942996 | 2019-10-17 15:16:49 +0300 | [diff] [blame] | 813 | job_name="deploy-upgrade-compute", |
| 814 | job_parameters=job_parameters) |
| 815 | |
Hanna Arhipova | 508f653 | 2021-01-27 15:52:45 +0200 | [diff] [blame] | 816 | assert job_result == 'SUCCESS', job_description |