Prepared fixed for the MOS Victoria-based image, MOS 23.2
The list of the fixes / improvements:
* Dockerfiles: set the FROM image to the Mirantis heat
Victoria image of MOS 23.2. Fixed some packages installation.
* Python cleanup.py script: re-wrote opertions with the
volume groups and volume group types with the specific
API version, resettings volume status, finding the
server groups. Added the functions for non-existing
methods in downstream openstacksdk.
* FIO scripts: Fixed the attaching/detaching the volumes,
some other fixes.
* Refreshed the si-tests, mos-spt, cfg-checker repositories,
archived.
* Changed the tag of the toolset image for Victoria:
toolset:23.2-victoria
Change-Id: I762eaa813c4c864979a2dfb5151dd082aff98cb2
diff --git a/cleanup.py b/cleanup.py
index 247dcbe..6f82823 100644
--- a/cleanup.py
+++ b/cleanup.py
@@ -1,7 +1,11 @@
import argparse
+import json
import os
import re
import sys
+import time
+
+from types import SimpleNamespace
import openstack
@@ -9,6 +13,8 @@
# Send logs to both, a log file and stdout
openstack.enable_logging(debug=False, path='openstack.log', stream=sys.stdout)
+volume_api_version = "3.43"
+
# Connect to cloud
TEST_CLOUD = os.getenv('OS_TEST_CLOUD', 'os-cloud')
cloud = openstack.connect(cloud=TEST_CLOUD)
@@ -41,6 +47,11 @@
return default
+def items_to_object(items):
+ data = json.dumps(items)
+ return json.loads(data, object_hook=lambda d: SimpleNamespace(**d))
+
+
def _filter_test_resources(resources, attribute, pattern=mask_pattern):
filtered_resources = {}
for item in resources:
@@ -64,15 +75,67 @@
log.info(f"... deleting {name} (id={id_}) {type_}")
-def _force_delete_load_balancer(id_):
- log.info(f"... ... force deleting {id_} load balancer")
- lb_ep = load_balancer.get_endpoint()
- lb_uri = f"{lb_ep}/lbaas/loadbalancers/{id_}"
+def _generate_volume_headers():
headers = {'X-Auth-Token': cloud.session.get_token(),
- 'Content-Type': 'application/json'}
- params = {'cascade': 'true', 'force': 'true'}
- cloud.session.request(url=lb_uri, method='DELETE',
- headers=headers, params=params)
+ 'Accept': 'application/json',
+ 'Content-Type': 'application/json',
+ 'OpenStack-API-Version': f'volume {volume_api_version}'}
+ return headers
+
+
+def _get_volume_group_types(all_tenants='true'):
+ # this method is not implemented in Victoria yet in openstacksdk
+ ep = volume.get_endpoint()
+ uri = f"{ep}/group_types"
+ headers = _generate_volume_headers()
+ params = {'all_tenants': all_tenants}
+ response = cloud.session.request(url=uri, method='GET',
+ headers=headers, params=params).json()
+ for g_type in response['group_types']:
+ yield g_type
+
+
+def _delete_volume_group_type(uuid):
+ # this method is not implemented in Victoria yet in openstacksdk
+ ep = volume.get_endpoint()
+ uri = f"{ep}/group_types/{uuid}"
+ headers = _generate_volume_headers()
+ cloud.session.request(url=uri, method='DELETE', headers=headers)
+
+
+def _get_volume_groups(all_tenants='true'):
+ # this method is not implemented in Victoria yet in openstacksdk
+ ep = volume.get_endpoint()
+ uri = f"{ep}/groups/detail"
+ headers = _generate_volume_headers()
+ params = {'all_tenants': all_tenants}
+ response = cloud.session.request(url=uri, method='GET',
+ headers=headers, params=params).json()
+ for group in response['groups']:
+ yield group
+
+
+def _delete_volume_group(uuid, delete_volumes='false'):
+ # this method is not implemented in Victoria yet in openstacksdk
+ ep = volume.get_endpoint()
+ uri = f"{ep}/groups/{uuid}/action"
+ headers = _generate_volume_headers()
+ body = {"delete": {"delete-volumes": delete_volumes}}
+ cloud.session.request(
+ url=uri, method='POST', headers=headers, json=body)
+
+
+def _reset_volume_status(uuid, status='available', attach_status='detached',
+ migration_status='None'):
+ # this method is not implemented in Victoria yet in openstacksdk
+ ep = volume.get_endpoint()
+ uri = f"{ep}/volumes/{uuid}/action"
+ headers = _generate_volume_headers()
+ body = {"os-reset_status": {
+ "status": status, "attach_status": attach_status,
+ "migration_status": migration_status}}
+ cloud.session.request(
+ url=uri, method='POST', headers=headers, json=body)
def cleanup_users():
@@ -225,7 +288,7 @@
if args.dry_run:
return
for id_ in volumes_to_delete:
- volume.reset_volume_status(id_, 'available', 'detached', 'None')
+ _reset_volume_status(id_, 'available', 'detached', 'None')
_log_resource_delete(id_, volumes_to_delete[id_], 'volume')
volume.delete_volume(id_)
vol_obj = volume.get_volume(id_)
@@ -233,14 +296,16 @@
def cleanup_volume_groups():
- groups = volume.groups()
+ groups_in_response = _get_volume_groups()
+ groups = items_to_object([g for g in groups_in_response])
groups_to_delete = _filter_test_resources(groups, 'name')
_log_resources_count(len(groups_to_delete), 'volume group(s)')
if args.dry_run:
return
for id_ in groups_to_delete:
_log_resource_delete(id_, groups_to_delete[id_], 'volume group')
- volume.delete_group(id_)
+ _delete_volume_group(id_)
+ time.sleep(10) # TODO : need to add a proper waiter
def cleanup_volume_backups():
@@ -257,7 +322,8 @@
def cleanup_volume_group_types():
- group_types = volume.group_types()
+ types_in_response = _get_volume_group_types()
+ group_types = items_to_object([g for g in types_in_response])
group_types_to_delete = _filter_test_resources(group_types, 'name')
_log_resources_count(len(group_types_to_delete), 'volume group type(s)')
if args.dry_run:
@@ -265,7 +331,7 @@
for id_ in group_types_to_delete:
_log_resource_delete(
id_, group_types_to_delete[id_], 'volume group type')
- volume.delete_group_type(id_)
+ _delete_volume_group_type(id_)
def cleanup_volume_types():
@@ -357,7 +423,8 @@
load_balancer.delete_load_balancer(id_, cascade=True)
except openstack.exceptions.ConflictException:
# force delete the LB in case it is in some PENDING_* state
- _force_delete_load_balancer(id_)
+ log.info(f"... ... force deleting {id_} load balancer")
+ load_balancer.delete_load_balancer(id_, cascade=True, force=True)
except Exception as e:
log.info(f"... ... could not delete {id_} load balancer: {e}")