blob: 51d29d0aef75f0cd0c1f48c15989040239efed7b [file] [log] [blame]
Hanna Arhipova71ecc272019-08-20 14:54:22 +03001import pytest
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +03002import sys
3import os
Hanna Arhipova71ecc272019-08-20 14:54:22 +03004
5from tcp_tests import logger
6from tcp_tests import settings
Hanna Arhipova71ecc272019-08-20 14:54:22 +03007
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +03008sys.path.append(os.getcwd())
9try:
10 from tcp_tests.fixtures import config_fixtures
11 from tcp_tests.managers import underlay_ssh_manager
12 from tcp_tests.managers import saltmanager as salt_manager
13except ImportError:
14 print("ImportError: Run the application from the tcp-qa directory or "
15 "set the PYTHONPATH environment variable to directory which contains"
16 " ./tcp_tests")
17 sys.exit(1)
Hanna Arhipova71ecc272019-08-20 14:54:22 +030018LOG = logger.logger
19
20
Hanna Arhipovad35a29b2019-09-04 13:24:06 +030021def has_only_similar(values_by_nodes):
22 """
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +030023 :param values_by_nodes: dict
Hanna Arhipovad35a29b2019-09-04 13:24:06 +030024 :return: bool, True if all items in the dict have similar values
25 """
26 values = list(values_by_nodes.values())
27 return all(value == values[0] for value in values)
28
29
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +030030def get_control_plane_targets():
31 config = config_fixtures.config()
32 underlay = underlay_ssh_manager.UnderlaySSHManager(config)
33 saltmanager = salt_manager.SaltManager(config, underlay)
Hanna Arhipovafbcea852019-10-03 13:24:53 +030034 targets = list()
Hanna Arhipova52bb17d2019-10-28 19:15:17 +020035 telemetry_exists = False
36 barbican_exists = False
Hanna Arhipovafbcea852019-10-03 13:24:53 +030037 try:
38 targets += saltmanager.run_state(
39 "I@keystone:server", 'test.ping')[0]['return'][0].keys()
40 targets += saltmanager.run_state(
41 "I@nginx:server and not I@salt:master",
42 "test.ping")[0]['return'][0].keys()
Hanna Arhipova4462dd02019-10-28 19:05:08 +020043 telemetry_exists = saltmanager.get_single_pillar(
44 "I@salt:master",
45 "_param:openstack_telemetry_hostname")
46 barbican_exists = saltmanager.get_single_pillar(
47 "I@salt:master",
48 "_param:barbican_enabled")
Hanna Arhipovafbcea852019-10-03 13:24:53 +030049 except BaseException as err:
50 LOG.warning("Can't retrieve data from Salt. \
51 Maybe cluster is not deployed completely.\
52 Err: {}".format(err))
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +030053
Hanna Arhipova75f21bb2019-10-07 19:22:15 +030054 # check for Manila existence
55 # if saltmanager.get_single_pillar("I@salt:master",
56 # "_param:manila_service_protocol"):
57 # targets.append('share*')
58
59 # check for Tenant Telemetry existence
Hanna Arhipova4462dd02019-10-28 19:05:08 +020060 if telemetry_exists:
Hanna Arhipova75f21bb2019-10-07 19:22:15 +030061 targets.append('mdb*')
62
63 # check for Barbican existence
Hanna Arhipova4462dd02019-10-28 19:05:08 +020064 if barbican_exists:
Hanna Arhipova75f21bb2019-10-07 19:22:15 +030065 targets.append('kmn*')
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +030066 return targets
67
68
Hanna Arhipova19429962019-10-17 15:16:49 +030069@pytest.fixture(scope='class')
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +030070def update_to_tag(request):
71 return request.config.getoption("--update-to-tag")
Hanna Arhipova672efc62021-03-15 09:55:14 +020072
73
74@pytest.fixture(scope='class')
PGlazov7efe3b22022-02-16 17:29:05 +040075def update_mysql_to_version(request):
76 return request.config.getoption("--update-mysql-version")
77
78
79@pytest.fixture(scope='class')
Hanna Arhipova672efc62021-03-15 09:55:14 +020080def switch_to_proposed_pipelines(reclass_actions, salt_actions,
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +030081 update_to_tag):
82 if update_to_tag:
Hanna Arhipova672efc62021-03-15 09:55:14 +020083 return True
84
Hanna Arhipova19429962019-10-17 15:16:49 +030085 reclass = reclass_actions
86 proposed_repo = "http://mirror.mirantis.com/update/proposed/"
87 repo_param = "parameters._param.linux_system_repo_update_url"
Hanna Arhipova7cfeb072019-10-11 15:04:10 +030088
Hanna Arhipova19429962019-10-17 15:16:49 +030089 proposed_pipeline_branch = "release/proposed/2019.2.0"
90 pipeline_branch_param = "parameters._param.jenkins_pipelines_branch"
91 infra_yml = "cluster/*/infra/init.yml"
92
93 LOG.info("Check reclass has release/proposed/2019.2.0 branches")
94 if reclass.get_key(pipeline_branch_param,
95 infra_yml) == proposed_pipeline_branch \
96 and reclass.get_key(repo_param, infra_yml) == proposed_repo:
97 return True
98
99 LOG.info("Switch to release/proposed/2019.2.0 branches")
100 reclass.add_key(pipeline_branch_param, proposed_pipeline_branch, infra_yml)
101
102 reclass.add_key(repo_param, proposed_repo, infra_yml)
103 reclass.add_key(repo_param, proposed_repo, "cluster/*/openstack/init.yml")
104 reclass.add_key(repo_param, proposed_repo, "cluster/*/stacklight/init.yml")
105 reclass.add_key(repo_param, proposed_repo, "cluster/*/ceph/init.yml")
Hanna Arhipova7cfeb072019-10-11 15:04:10 +0300106
107 salt_actions.run_state("*", "saltutil.refresh_pillar")
Hanna Arhipova9ee75902019-10-29 16:33:26 +0200108 salt_actions.enforce_state("*", "salt.minion")
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300109 salt_actions.enforce_state("I@jenkins:client", "jenkins.client")
110
111
Hanna Arhipova745fdbb2020-03-30 14:40:44 +0300112@pytest.fixture
Hanna Arhipovaec201e72020-10-26 17:14:25 +0200113def wa_for_galera_clustercheck_password_prod35705(reclass_actions,
114 salt_actions):
Hanna Arhipova0e816d22021-01-29 00:17:44 +0200115 tgt = "I@galera:master or I@galera:slave"
116 if not salt_actions.get_pillar(tgt,
117 "_param:galera_clustercheck_password")[0]:
118 reclass_actions.add_key(
119 "parameters._param.galera_clustercheck_password",
120 "a"*32,
121 "cluster/*/infra/secrets.yml")
122 salt_actions.run_state(tgt, "saltutil.refresh_pillar")
123 salt_actions.enforce_state(tgt, "galera")
124 salt_actions.enforce_state(tgt, "haproxy")
125 reclass_actions.commit(
126 "[from TCP-QA] Add galera_clustercheck_password")
127 else:
Hanna Arhipova672efc62021-03-15 09:55:14 +0200128
Hanna Arhipova0e816d22021-01-29 00:17:44 +0200129 LOG.info("Skipping WA for Galera Clustercheck Password")
Hanna Arhipova745fdbb2020-03-30 14:40:44 +0300130
131
Hanna Arhipova97fac6e2020-12-03 13:10:56 +0200132@pytest.fixture
PGlazovf5977292021-11-25 19:38:27 +0400133def wa_cve_2021_20288_global_id_reclaim(reclass_actions,
134 salt_actions):
135 tgt = "I@ceph:common"
136 context_file = "cluster/*/ceph/common.yml"
137 if not reclass_actions.check_existence(
Pavel Glazov6ed8d442022-09-20 12:50:18 +0400138 'parameters.ceph.common.config.mon.' +
139 'auth_allow_insecure_global_id_reclaim'):
PGlazov4b013962022-01-31 14:01:42 +0400140 if salt_actions.get_single_pillar(
141 "I@ceph:mgr and *01.*",
142 "_param.ceph_version") == 'luminous':
143 LOG.info('Skip reclaim set')
144 return
PGlazovf5977292021-11-25 19:38:27 +0400145 reclass_actions.add_bool_key(
Pavel Glazov6ed8d442022-09-20 12:50:18 +0400146 'parameters.ceph.common.config.mon.' +
147 'auth_allow_insecure_global_id_reclaim', "False", context_file)
PGlazovf5977292021-11-25 19:38:27 +0400148 salt_actions.run_state(tgt, "state.apply", "ceph.common")
149 salt_actions.cmd_run(tgt, "systemctl restart ceph-mon.target")
150 else:
151 LOG.info("Skipping WA ceph set auth_allow_insecure_global_id_reclaim")
152
153
154@pytest.fixture
Hanna Arhipova97fac6e2020-12-03 13:10:56 +0200155def wa_for_alerta_password_prod35958(reclass_actions,
156 salt_actions):
Pavel Glazovea439c42021-07-25 00:20:20 +0000157
Hanna Arhipova0e816d22021-01-29 00:17:44 +0200158 if not salt_actions.get_pillar("I@prometheus:alerta",
159 "_param:alerta_admin_api_key_generated")[0]:
160 reclass_actions.add_key(
161 "parameters._param.alerta_admin_api_key_generated",
162 "a"*32,
163 "cluster/*/infra/secrets.yml")
164 reclass_actions.add_key(
165 "parameters._param.alerta_admin_key",
166 "${_param:alerta_admin_api_key_generated}",
167 "cluster/*/stacklight/init.yml")
168 reclass_actions.commit("[from TCP-QA] Add alerta_admin_key")
169 salt_actions.run_state(
170 "I@prometheus:alerta or I@prometheus:alertmanager",
171 "saltutil.refresh_pillar")
172 salt_actions.enforce_state(
173 "I@prometheus:alerta", "prometheus.alerta")
174 salt_actions.enforce_state(
175 "I@prometheus:alertmanager", "prometheus.alertmanager")
176 salt_actions.enforce_state(
177 "I@prometheus:alerta or I@prometheus:alertmanager",
178 "docker.client")
179 else:
180 LOG.info("Skipping WA for Alerta API key")
Hanna Arhipova97fac6e2020-12-03 13:10:56 +0200181
182
Vladimir Khlyunev5fe0aaa2022-08-16 23:13:22 +0400183@pytest.fixture
184def wa_redis_server_version_3_prod36960(reclass_actions, salt_actions):
185 version_pillar = "redis:server:version"
186 tgt = "I@{}".format(version_pillar)
187 pillar_file = "cluster/*/openstack/telemetry.yml"
188 redis_target_version = "5.0"
189
190 msg_nodes = salt_actions.get_minions_by_target(tgt)
191 if not msg_nodes:
192 LOG.info("dbs* nodes not found, skipping")
193 return
194 else:
195 msg_node = msg_nodes[0]
196
197 if str(salt_actions.get_single_pillar(
198 msg_node,
199 version_pillar)) == redis_target_version:
200 LOG.info("Redis-server is already '{}', skipping".format(
201 redis_target_version))
202 return
203
204 LOG.info("Setting redis-server version to '{}'".format(
205 redis_target_version))
206 reclass_actions.add_key(
207 "parameters.{}".format(version_pillar.replace(":", ".")),
208 redis_target_version,
209 pillar_file)
210 salt_actions.run_state(" or ".join(msg_nodes),
211 "saltutil.refresh_pillar")
212
213
Hanna Arhipova19429962019-10-17 15:16:49 +0300214@pytest.fixture(scope='class')
215def enable_openstack_update(reclass_actions, salt_actions):
216 param = "parameters._param.openstack_upgrade_enabled"
217 context_file = "cluster/*/infra/init.yml"
218
219 LOG.info("Enable openstack_upgrade_enabled in reclass")
220 reclass_actions.add_bool_key(param, "True", context_file)
221 salt_actions.run_state("*", "saltutil.refresh_pillar")
222 yield True
223 LOG.info("Disable openstack_upgrade_enabled in reclass")
224 reclass_actions.add_bool_key(param, "False", context_file)
225 salt_actions.run_state("*", "saltutil.refresh_pillar")
226
227
Vladimir Khlyunev0ca3ba82022-10-31 12:12:30 +0400228@pytest.fixture(scope='class')
Pavel Glazov6ee164c2022-12-12 06:42:28 +0400229def switch_to_staging_updates_mirantis_repo(update_to_tag,
230 reclass_actions,
231 salt_actions):
Vladimir Khlyunev0ca3ba82022-10-31 12:12:30 +0400232 version_param = "parameters._param.updates_mirantis_version"
233 infra_yml = "cluster/*/infra/init.yml"
Vladimir Khlyunev0ca3ba82022-10-31 12:12:30 +0400234 current_version = reclass_actions.get_key(version_param, infra_yml)
Pavel Glazov6ee164c2022-12-12 06:42:28 +0400235
236 if update_to_tag is not None:
237 reclass_actions.add_key(version_param, update_to_tag, infra_yml)
Vladimir Khlyunev0ca3ba82022-10-31 12:12:30 +0400238 if current_version == 'staging':
239 LOG.info("updates.mirantis.com already set to 'staging', skipping")
240 return
241 if current_version is None:
242 LOG.info("updates.mirantis.com is not enabled, "
243 "skipping /staging repo pillar")
244 return
Pavel Glazov6ee164c2022-12-12 06:42:28 +0400245 if current_version is not None:
246 LOG.info("updates.mirantis.com is other tag "
247 "skipping /staging repo pillar")
248 salt_actions.run_state("*", "saltutil.refresh_pillar")
249 return
Vladimir Khlyunev0ca3ba82022-10-31 12:12:30 +0400250
251 LOG.info("Switch to updates.mirantis.com/staging repo")
252 reclass_actions.add_key(version_param, 'staging', infra_yml)
253
254 salt_actions.run_state("*", "saltutil.refresh_pillar")
255
256
Hanna Arhipova745fdbb2020-03-30 14:40:44 +0300257@pytest.mark.usefixtures("switch_to_proposed_pipelines",
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +0300258 "update_to_tag",
Hanna Arhipova97fac6e2020-12-03 13:10:56 +0200259 "wa_for_galera_clustercheck_password_prod35705",
Vladimir Khlyunev5fe0aaa2022-08-16 23:13:22 +0400260 "wa_for_alerta_password_prod35958",
Vladimir Khlyunev0ca3ba82022-10-31 12:12:30 +0400261 "wa_redis_server_version_3_prod36960",
262 "switch_to_staging_updates_mirantis_repo")
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300263class TestUpdateMcpCluster(object):
264 """
265 Following the steps in
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300266 https://docs.mirantis.com/mcp/master/mcp-operations-guide/update-upgrade/minor-update.html#minor-update
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300267 """
268
269 @pytest.mark.grab_versions
270 @pytest.mark.parametrize("_", [settings.ENV_NAME])
271 @pytest.mark.run_mcp_update
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300272 def test_update_drivetrain(self, salt_actions, drivetrain_actions,
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +0300273 show_step, update_to_tag, _):
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300274 """Updating DriveTrain component to release/proposed/2019.2.0 version
275
276 Scenario:
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300277 1. Add workaround for PROD-32751
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300278 2. Run job git-mirror-downstream-mk-pipelines
279 3. Run job git-mirror-downstream-pipeline-library
dtsapikovcbff2332022-01-19 19:32:22 +0400280 4. Run job pre-upgrade-verify
281 5. If jobs are passed then start 'Deploy - upgrade MCP Drivetrain'
282 6. Wait till salt-master completes all async jobs
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300283
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300284 Duration: ~70 min
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300285 """
286 salt = salt_actions
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300287 dt = drivetrain_actions
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300288
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300289 # #################### Add workaround for PROD-32751 #################
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300290 show_step(1)
291
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300292 # FIXME: workaround for PROD-32751
293 salt.cmd_run("cfg01*", "cd /srv/salt/reclass; git add -u && \
294 git commit --allow-empty -m 'Cluster model update'")
295
296 # ################### Downstream mk-pipelines #########################
297 show_step(2)
298 job_name = 'git-mirror-downstream-mk-pipelines'
299 job_parameters = {
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +0300300 'BRANCHES': '*' or
301 'release/proposed/2019.2.0'
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300302 }
Hanna Arhipova508f6532021-01-27 15:52:45 +0200303 job_result, job_description = dt.start_job_on_jenkins(
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300304 job_name=job_name,
Hanna Arhipovac3c59292020-04-23 16:29:09 +0300305 job_parameters=job_parameters,
306 verbose=True)
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300307
Hanna Arhipova508f6532021-01-27 15:52:45 +0200308 assert job_result == 'SUCCESS', job_description
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300309
310 # ################### Downstream pipeline-library ####################
311 show_step(3)
312 job_name = 'git-mirror-downstream-pipeline-library'
313 job_parameters = {
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +0300314 'BRANCHES': '*' or
315 'release/proposed/2019.2.0'
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300316 }
Hanna Arhipova508f6532021-01-27 15:52:45 +0200317 job_result, job_description = dt.start_job_on_jenkins(
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300318 job_name=job_name,
Hanna Arhipovac3c59292020-04-23 16:29:09 +0300319 job_parameters=job_parameters,
320 verbose=True)
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300321
Hanna Arhipova508f6532021-01-27 15:52:45 +0200322 assert job_result == 'SUCCESS', job_description
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300323
dtsapikovcbff2332022-01-19 19:32:22 +0400324 # ############### Start 'Pre-upgrade verify' job (since 2019.2.17)#####
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300325 show_step(4)
dtsapikovcbff2332022-01-19 19:32:22 +0400326 job_name = 'pre-upgrade-verify'
327 job_parameters = {}
328
Pavel Glazov794b1a92022-10-03 16:33:04 +0400329 if (dt.check_job_exists_on_jenkins(job_name)):
dtsapikovcbff2332022-01-19 19:32:22 +0400330 LOG.info("Pre-upgrade verify job exists on Jenkins!")
331 job_result, job_description = dt.start_job_on_jenkins(
332 job_name=job_name,
333 job_parameters=job_parameters,
334 verbose=True,
335 build_timeout=4 * 60 * 60)
336
337 assert job_result == 'SUCCESS', job_description
338
339 # ################### Start 'Deploy - upgrade MCP Drivetrain' job #####
340 show_step(5)
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300341
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300342 job_name = 'upgrade-mcp-release'
343 job_parameters = {
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +0300344 'GIT_REFSPEC': update_to_tag or 'release/proposed/2019.2.0',
PGlazov1df9e542021-11-16 02:31:49 +0400345 'MK_PIPELINES_REFSPEC':
346 update_to_tag or 'release/proposed/2019.2.0',
Hanna Arhipovae1dd27a2021-07-28 12:40:40 +0300347 'TARGET_MCP_VERSION': update_to_tag or '2019.2.0',
Hanna Arhipova874c68f2021-03-29 15:57:19 +0300348 "DRIVE_TRAIN_PARAMS": {
349 "OS_DIST_UPGRADE": True,
350 "OS_UPGRADE": True,
351 "BATCH_SIZE": 10
352 }
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300353 }
Hanna Arhipova508f6532021-01-27 15:52:45 +0200354 job_result, job_description = dt.start_job_on_jenkins(
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300355 job_name=job_name,
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300356 job_parameters=job_parameters,
Hanna Arhipovac3c59292020-04-23 16:29:09 +0300357 verbose=True,
Hanna Arhipovae7f2e322021-02-25 13:53:18 +0200358 build_timeout=4 * 60 * 60)
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300359
Hanna Arhipova508f6532021-01-27 15:52:45 +0200360 assert job_result == 'SUCCESS', job_description
Hanna Arhipova7a593da2021-06-25 11:16:09 +0300361 # ############### Wait till salt-master completes all async jobs #####
362 show_step(5)
363 salt.wait_jobs_completed(timeout=20*60)
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300364
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300365 @pytest.mark.grab_versions
366 @pytest.mark.parametrize("_", [settings.ENV_NAME])
Hanna Arhipova71ecc272019-08-20 14:54:22 +0300367 @pytest.mark.run_mcp_update
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300368 def test_update_glusterfs(self, salt_actions, reclass_actions,
PGlazov7efe3b22022-02-16 17:29:05 +0400369 drivetrain_actions, show_step,
370 update_mysql_to_version, _):
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300371 """ Upgrade GlusterFS
372 Scenario:
373 1. In infra/init.yml in Reclass, add the glusterfs_version parameter
374 2. Start linux.system.repo state
375 3. Start "update-glusterfs" job
376 4. Check version for GlusterFS servers
377 5. Check version for GlusterFS clients
378
379 """
PGlazov7efe3b22022-02-16 17:29:05 +0400380
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300381 salt = salt_actions
382 reclass = reclass_actions
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300383 dt = drivetrain_actions
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300384
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300385 # ############## Change reclass ######################################
386 show_step(1)
387 reclass.add_key(
388 "parameters._param.linux_system_repo_mcp_glusterfs_version_number",
389 "5",
390 "cluster/*/infra/init.yml"
391 )
392 # ################# Run linux.system state ###########################
393 show_step(2)
394 salt.enforce_state("*", "linux.system.repo")
395
396 # ############## Start deploy-upgrade-galera job #####################
397 show_step(3)
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300398 job_name = 'update-glusterfs'
399
Hanna Arhipova508f6532021-01-27 15:52:45 +0200400 job_result, job_description = dt.start_job_on_jenkins(
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300401 job_name=job_name,
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300402 build_timeout=40 * 60)
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300403
Hanna Arhipova508f6532021-01-27 15:52:45 +0200404 assert job_result == 'SUCCESS', job_description
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300405
406 # ################ Check GlusterFS version for servers ##############
407 show_step(4)
408 gluster_server_versions_by_nodes = salt.cmd_run(
409 "I@glusterfs:server",
410 "glusterd --version|head -n1")[0]
411
Hanna Arhipova19429962019-10-17 15:16:49 +0300412 assert has_only_similar(gluster_server_versions_by_nodes), \
Hanna Arhipovacc3759b2019-08-28 16:01:11 +0300413 gluster_server_versions_by_nodes
414
415 # ################ Check GlusterFS version for clients ##############
416 show_step(5)
417 gluster_client_versions_by_nodes = salt.cmd_run(
418 "I@glusterfs:client",
419 "glusterfs --version|head -n1")[0]
420
421 assert has_only_similar(gluster_client_versions_by_nodes), \
422 gluster_client_versions_by_nodes
423
424 @pytest.mark.grab_versions
425 @pytest.mark.parametrize("_", [settings.ENV_NAME])
426 @pytest.mark.run_mcp_update
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300427 def test_update_galera(self, salt_actions, reclass_actions,
PGlazov7efe3b22022-02-16 17:29:05 +0400428 drivetrain_actions, show_step,
429 update_mysql_to_version, _):
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300430 """ Upgrade Galera automatically
431
432 Scenario:
433 1. Include the Galera upgrade pipeline job to DriveTrain
PGlazov7efe3b22022-02-16 17:29:05 +0400434 2. Add fix for mysql 5.7
435 3. Apply the jenkins.client state on the Jenkins nodes
436 4. set the openstack_upgrade_enabled parameter to true
437 5. Refresh pillars
438 6. Add repositories with new Galera packages
439 7. Start job from Jenkins
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300440 """
441 salt = salt_actions
442 reclass = reclass_actions
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300443 dt = drivetrain_actions
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300444 # ################### Enable pipeline #################################
445 show_step(1)
446 reclass.add_class(
447 "system.jenkins.client.job.deploy.update.upgrade_galera",
448 "cluster/*/cicd/control/leader.yml")
PGlazov7efe3b22022-02-16 17:29:05 +0400449 # ################### Fix for Mysql 5.7 ##############################
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300450 show_step(2)
PGlazov7efe3b22022-02-16 17:29:05 +0400451 if (update_mysql_to_version == '5.7'):
PGlazov7efe3b22022-02-16 17:29:05 +0400452 mysql_class = "cluster.{env_name}.openstack." \
453 "database.mysql_version"\
454 .format(env_name=settings.ENV_NAME)
455 reclass.add_class(
456 mysql_class,
457 "cluster/*/openstack/database/master.yml")
458 reclass.add_class(
459 mysql_class,
460 "cluster/*/openstack/database/slave.yml")
461 mysql_path = "cluster/{env_name}" \
462 "/openstack/database/mysql_version.yml"\
463 .format(env_name=settings.ENV_NAME)
464 galera_mysql_version = '5.7'
465 if update_mysql_to_version != '5.7':
466 galera_mysql_version = update_mysql_to_version
467 mysql_version = "parameters:\n" \
468 " _param:\n" \
469 " galera_mysql_version: {}"\
470 .format(galera_mysql_version)
471 if not os.path.isfile("/srv/salt/reclass/classes/{path}".format(
472 path=mysql_path)):
473 reclass.create_yaml_with_context(mysql_version,
474 mysql_path)
475 else:
476 reclass.merge_context(mysql_version,
477 mysql_path)
478 reclass_actions.commit(
479 "[from TCP-QA] Change for mysql new version")
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300480
PGlazov7efe3b22022-02-16 17:29:05 +0400481 show_step(3)
Vladimir Khlyunev17468272022-06-29 16:19:36 +0400482 salt.run_state("I@jenkins:client and not I@salt:master",
483 "saltutil.refresh_pillar")
484 salt.enforce_state("I@jenkins:client and not I@salt:master",
485 "jenkins.client.job")
PGlazov7efe3b22022-02-16 17:29:05 +0400486 else:
487 LOG.info("Skip mysql upgrade fix")
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300488 # ############### Enable automatic upgrade ############################
PGlazov7efe3b22022-02-16 17:29:05 +0400489 show_step(4)
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300490 reclass.add_bool_key("parameters._param.openstack_upgrade_enabled",
491 "True",
492 "cluster/*/infra/init.yml")
493
PGlazov7efe3b22022-02-16 17:29:05 +0400494 show_step(5)
Hanna Arhipova7cfeb072019-10-11 15:04:10 +0300495 salt.run_state("dbs*", "saltutil.refresh_pillar")
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300496
497 # ############# Add repositories with new Galera packages #######
PGlazov7efe3b22022-02-16 17:29:05 +0400498 show_step(6)
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300499 salt.enforce_state("dbs*", "linux.system.repo")
500 salt.enforce_state("cfg*", "salt.master")
501
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300502 # #################### Login Jenkins on cid01 node ###################
PGlazov7efe3b22022-02-16 17:29:05 +0400503 show_step(7)
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300504
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300505 job_name = 'deploy-upgrade-galera'
506 job_parameters = {
Hanna Arhipova874c68f2021-03-29 15:57:19 +0300507 'INTERACTIVE': 'false',
508 'OS_DIST_UPGRADE': 'true',
PGlazov7efe3b22022-02-16 17:29:05 +0400509 'OS_UPGRADE': 'true',
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300510 }
PGlazov554c4fd2022-03-23 14:31:41 +0400511 if (update_mysql_to_version == '5.7'):
PGlazov7efe3b22022-02-16 17:29:05 +0400512 job_parameters.update({'UPDATE_TO_MYSQL57': 'true'})
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300513
Hanna Arhipova508f6532021-01-27 15:52:45 +0200514 job_result, job_description = dt.start_job_on_jenkins(
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300515 job_name=job_name,
Hanna Arhipova17b2c102019-09-06 16:44:17 +0300516 job_parameters=job_parameters,
Pavel Glazov2e9cbcd2022-12-06 14:52:53 +0400517 build_timeout=60 * 60)
Hanna Arhipova94a8abe2019-08-22 14:11:46 +0300518
Hanna Arhipova508f6532021-01-27 15:52:45 +0200519 assert job_result == 'SUCCESS', job_description
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300520
521 @pytest.fixture
522 def disable_automatic_failover_neutron_for_test(self, salt_actions):
523 """
524 On each OpenStack controller node, modify the neutron.conf file
525 Restart the neutron-server service
526 """
Hanna Arhipova19429962019-10-17 15:16:49 +0300527
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300528 def comment_line(node, file_name, word):
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300529 """
530 Adds '#' before the specific line in specific file
531
532 :param node: string, salt target of node where the file locates
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300533 :param file_name: string, full path to the file
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300534 :param word: string, the begin of line which should be commented
535 :return: None
536 """
537 salt_actions.cmd_run(node,
538 "sed -i 's/^{word}/#{word}/' {file}".
539 format(word=word,
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300540 file=file_name))
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300541
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300542 def add_line(node, file_name, line):
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300543 """
544 Appends line to the end of file
545
546 :param node: string, salt target of node where the file locates
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300547 :param file_name: string, full path to the file
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300548 :param line: string, line that should be added
549 :return: None
550 """
551 salt_actions.cmd_run(node, "echo {line} >> {file}".format(
Hanna Arhipova19429962019-10-17 15:16:49 +0300552 line=line,
553 file=file_name))
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300554
555 neutron_conf = '/etc/neutron/neutron.conf'
556 neutron_server = "I@neutron:server"
557 # ######## Create backup for config file #######################
558 salt_actions.cmd_run(
559 neutron_server,
560 "cp -p {file} {file}.backup".format(file=neutron_conf))
561
562 # ## Change parameters in neutron.conf'
563 comment_line(neutron_server, neutron_conf,
Hanna Arhipova19429962019-10-17 15:16:49 +0300564 "allow_automatic_l3agent_failover", )
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300565 comment_line(neutron_server, neutron_conf,
566 "allow_automatic_dhcp_failover")
567 add_line(neutron_server, neutron_conf,
568 "allow_automatic_dhcp_failover = false")
569 add_line(neutron_server, neutron_conf,
570 "allow_automatic_l3agent_failover = false")
571
572 # ## Apply changed config to the neutron-server service
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300573 result = salt_actions.cmd_run(neutron_server,
574 "service neutron-server restart")
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300575 # TODO: add check that neutron-server is up and running
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300576 yield result
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300577 # ## Revert file changes
578 salt_actions.cmd_run(
579 neutron_server,
580 "cp -p {file}.backup {file}".format(file=neutron_conf))
581 salt_actions.cmd_run(neutron_server,
582 "service neutron-server restart")
583
584 @pytest.fixture
585 def disable_neutron_agents_for_test(self, salt_actions):
586 """
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300587 Disable the neutron services before the test and
588 enable it after test
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300589 """
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300590 result = salt_actions.cmd_run("I@neutron:server", """
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300591 service neutron-dhcp-agent stop && \
592 service neutron-l3-agent stop && \
593 service neutron-metadata-agent stop && \
594 service neutron-openvswitch-agent stop
595 """)
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300596 yield result
597 #
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300598 salt_actions.cmd_run("I@neutron:server", """
599 service neutron-dhcp-agent start && \
600 service neutron-l3-agent start && \
601 service neutron-metadata-agent start && \
602 service neutron-openvswitch-agent start
603 """)
604 # TODO: add check that all services are UP and running
605
606 @pytest.mark.grab_versions
607 @pytest.mark.parametrize("_", [settings.ENV_NAME])
608 @pytest.mark.run_mcp_update
609 def test_update_rabbit(self, salt_actions, reclass_actions,
610 drivetrain_actions, show_step, _,
611 disable_automatic_failover_neutron_for_test,
612 disable_neutron_agents_for_test):
613 """ Updates RabbitMQ
614 Scenario:
615 1. Include the RabbitMQ upgrade pipeline job to DriveTrain
616 2. Add repositories with new RabbitMQ packages
617 3. Start Deploy - upgrade RabbitMQ pipeline
618
619 Updating RabbitMq should be completed before the OpenStack updating
620 process starts
621 """
622 salt = salt_actions
623 reclass = reclass_actions
624 dt = drivetrain_actions
625
626 # ####### Include the RabbitMQ upgrade pipeline job to DriveTrain ####
627 show_step(1)
628 reclass.add_class(
629 "system.jenkins.client.job.deploy.update.upgrade_rabbitmq",
630 "cluster/*/cicd/control/leader.yml")
631 salt.enforce_state("I@jenkins:client", "jenkins.client")
632
633 reclass.add_bool_key("parameters._param.openstack_upgrade_enabled",
634 "True",
635 "cluster/*/infra/init.yml")
636 salt.run_state("I@rabbitmq:server", "saltutil.refresh_pillar")
637
638 # ########### Add repositories with new RabbitMQ packages ############
639 show_step(2)
640 salt.enforce_state("I@rabbitmq:server", "linux.system.repo")
641
642 # ########### Start Deploy - upgrade RabbitMQ pipeline ############
643 show_step(3)
644 job_parameters = {
Hanna Arhipova874c68f2021-03-29 15:57:19 +0300645 'INTERACTIVE': 'false',
646 'OS_DIST_UPGRADE': 'true',
647 'OS_UPGRADE': 'true'
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300648 }
649
Hanna Arhipova508f6532021-01-27 15:52:45 +0200650 job_result, job_description = dt.start_job_on_jenkins(
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300651 job_name='deploy-upgrade-rabbitmq',
652 job_parameters=job_parameters,
PGlazov99364cb2021-09-20 12:36:18 +0400653 build_timeout=50 * 60
Hanna Arhipova1fcaf442019-09-06 15:30:45 +0300654 )
Hanna Arhipova508f6532021-01-27 15:52:45 +0200655 assert job_result == 'SUCCESS', job_description
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300656
657 @pytest.mark.grab_versions
658 @pytest.mark.parametrize("_", [settings.ENV_NAME])
659 @pytest.mark.run_mcp_update
PGlazov215a4572022-03-05 05:09:04 +0400660 def test_update_ceph(self, salt_actions, drivetrain_actions,
661 reclass_actions, show_step, _):
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300662 """ Updates Ceph to the latest minor version
663
664 Scenario:
665 1. Add workaround for unhealth Ceph
666 2. Start ceph-upgrade job with default parameters
667 3. Check Ceph version for all nodes
PGlazov215a4572022-03-05 05:09:04 +0400668 4. WA id_reclaim only for nautilus
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300669
670 https://docs.mirantis.com/mcp/master/mcp-operations-guide/update-upgrade/minor-update/ceph-update.html
671 """
672 salt = salt_actions
PGlazov215a4572022-03-05 05:09:04 +0400673 reclass = reclass_actions
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300674 dt = drivetrain_actions
675
676 # ###################### Add workaround for unhealth Ceph ############
677 show_step(1)
678 salt.cmd_run("I@ceph:radosgw",
679 "ceph config set 'mon pg warn max object skew' 20")
680 # ###################### Start ceph-upgrade pipeline #################
681 show_step(2)
682 job_parameters = {}
683
Hanna Arhipova508f6532021-01-27 15:52:45 +0200684 job_result, job_description = dt.start_job_on_jenkins(
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300685 job_name='ceph-update',
686 job_parameters=job_parameters)
687
Hanna Arhipova508f6532021-01-27 15:52:45 +0200688 assert job_result == 'SUCCESS', job_description
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300689
690 # ########## Verify Ceph version #####################################
691 show_step(3)
692
693 ceph_version_by_nodes = salt.cmd_run(
Hanna Arhipova19429962019-10-17 15:16:49 +0300694 "I@ceph:* and not I@ceph:monitoring and not I@ceph:backup:server",
695 "ceph version")[0]
Hanna Arhipovad35a29b2019-09-04 13:24:06 +0300696
697 assert has_only_similar(ceph_version_by_nodes), ceph_version_by_nodes
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300698
PGlazov215a4572022-03-05 05:09:04 +0400699 # ########## WA id_reclaim only for nautilus #########################
700 show_step(4)
701
702 ceph_version = salt_actions.get_single_pillar(
703 "I@salt:master",
704 "_param:_param:ceph_version")
705 if ceph_version == "nautilus":
706 wa_cve_2021_20288_global_id_reclaim(reclass, salt)
707 else:
708 LOG.info("Skip WA for luminous")
709
Hanna Arhipovaac008562019-10-17 11:54:23 +0300710 @pytest.mark.grab_versions
711 @pytest.mark.parametrize("_", [settings.ENV_NAME])
712 @pytest.mark.run_mcp_update
713 def test_update_stacklight(self, _, drivetrain_actions):
714 """ Update packages for Stacklight
715 Scenario:
716 1. Start Deploy - upgrade Stacklight job
717 """
718 drivetrain = drivetrain_actions
719
720 job_parameters = {
721 "STAGE_UPGRADE_DOCKER_COMPONENTS": True,
722 "STAGE_UPGRADE_ES_KIBANA": True,
Hanna Arhipova874c68f2021-03-29 15:57:19 +0300723 "STAGE_UPGRADE_SYSTEM_PART": True,
724 'OS_DIST_UPGRADE': 'true',
725 'OS_UPGRADE': 'true'
Hanna Arhipovaac008562019-10-17 11:54:23 +0300726 }
Hanna Arhipova508f6532021-01-27 15:52:45 +0200727 job_result, job_description = drivetrain.start_job_on_jenkins(
Hanna Arhipovaac008562019-10-17 11:54:23 +0300728 job_name="stacklight-upgrade",
729 job_parameters=job_parameters)
730
Hanna Arhipova508f6532021-01-27 15:52:45 +0200731 assert job_result == 'SUCCESS', job_description
Hanna Arhipovaac008562019-10-17 11:54:23 +0300732
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300733
Hanna Arhipova19429962019-10-17 15:16:49 +0300734@pytest.mark.usefixtures("switch_to_proposed_pipelines",
735 "enable_openstack_update")
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300736class TestOpenstackUpdate(object):
737
738 @pytest.mark.grab_versions
739 @pytest.mark.run_mcp_update
740 def test__pre_update__enable_pipeline_job(self,
741 reclass_actions, salt_actions,
742 show_step):
743 """ Enable pipeline in the Drivetrain
744
745 Scenario:
746 1. Add deploy.update.* classes to the reclass
747 2. Start jenkins.client salt state
748
749 """
750 salt = salt_actions
751 reclass = reclass_actions
752 show_step(1)
753 reclass.add_class("system.jenkins.client.job.deploy.update.upgrade",
754 "cluster/*/cicd/control/leader.yml")
755
756 reclass.add_class(
757 "system.jenkins.client.job.deploy.update.upgrade_ovs_gateway",
758 "cluster/*/cicd/control/leader.yml")
759
760 reclass.add_class(
761 "system.jenkins.client.job.deploy.update.upgrade_compute",
762 "cluster/*/cicd/control/leader.yml")
763
764 show_step(2)
765 r, errors = salt.enforce_state("I@jenkins:client", "jenkins.client")
766 assert errors is None
767
768 @pytest.mark.grab_versions
769 @pytest.mark.parametrize('target', get_control_plane_targets())
770 @pytest.mark.run_mcp_update
Hanna Arhipova19429962019-10-17 15:16:49 +0300771 def test__update__control_plane(self, drivetrain_actions, target):
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300772 """Start 'Deploy - upgrade control VMs' for specific node
773 """
774 job_parameters = {
775 "TARGET_SERVERS": target,
Hanna Arhipova606d5182019-12-13 13:23:55 +0200776 "OS_DIST_UPGRADE": True,
PGlazova5039252021-10-06 21:40:01 +0400777 "UPGRADE_SALTSTACK": True,
Hanna Arhipova606d5182019-12-13 13:23:55 +0200778 "OS_UPGRADE": True,
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300779 "INTERACTIVE": False}
Hanna Arhipova508f6532021-01-27 15:52:45 +0200780 job_result, job_description = drivetrain_actions.start_job_on_jenkins(
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300781 job_name="deploy-upgrade-control",
782 job_parameters=job_parameters)
783
Hanna Arhipova508f6532021-01-27 15:52:45 +0200784 assert job_result == 'SUCCESS', job_description
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300785
786 @pytest.mark.grab_versions
787 @pytest.mark.run_mcp_update
Hanna Arhipova606d5182019-12-13 13:23:55 +0200788 def test__update__data_plane(self, drivetrain_actions, salt_actions):
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300789 """Start 'Deploy - upgrade OVS gateway'
790 """
Hanna Arhipova606d5182019-12-13 13:23:55 +0200791 if not salt_actions.cmd_run("gtw*", "test.ping")[0].keys():
792 pytest.skip("This deployment doesn't have gtw* nodes")
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300793 job_parameters = {
Hanna Arhipova606d5182019-12-13 13:23:55 +0200794 "OS_DIST_UPGRADE": True,
795 "OS_UPGRADE": True,
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300796 "INTERACTIVE": False}
Hanna Arhipova508f6532021-01-27 15:52:45 +0200797 job_result, job_description = drivetrain_actions.start_job_on_jenkins(
Hanna Arhipovaeb3a2112019-09-13 18:45:21 +0300798 job_name="deploy-upgrade-ovs-gateway",
799 job_parameters=job_parameters)
800
Hanna Arhipova508f6532021-01-27 15:52:45 +0200801 assert job_result == 'SUCCESS', job_description
Hanna Arhipova19429962019-10-17 15:16:49 +0300802
803 @pytest.mark.grab_versions
804 @pytest.mark.run_mcp_update
805 def test__update__computes(self, drivetrain_actions):
806 """Start 'Deploy - upgrade computes'
807 """
808 job_parameters = {
Hanna Arhipova606d5182019-12-13 13:23:55 +0200809 "OS_DIST_UPGRADE": True,
810 "OS_UPGRADE": True,
Hanna Arhipova19429962019-10-17 15:16:49 +0300811 "INTERACTIVE": False}
Hanna Arhipova508f6532021-01-27 15:52:45 +0200812 job_result, job_description = drivetrain_actions.start_job_on_jenkins(
Hanna Arhipova19429962019-10-17 15:16:49 +0300813 job_name="deploy-upgrade-compute",
814 job_parameters=job_parameters)
815
Hanna Arhipova508f6532021-01-27 15:52:45 +0200816 assert job_result == 'SUCCESS', job_description