cleanup.py

import argparse
import json
import os
import re
import sys
import time

from types import SimpleNamespace

import openstack


# Send logs to both, a log file and stdout
openstack.enable_logging(debug=False, path='openstack.log', stream=sys.stdout)

volume_api_version = "3.43"
manila_api_version = "2.75"

# Connect to cloud
TEST_CLOUD = os.getenv('OS_TEST_CLOUD', 'os-cloud')
cloud = openstack.connect(cloud=TEST_CLOUD)
log = cloud.log

# Get cloud config (clouds.yaml vars)
config_obj = openstack.config.loader.OpenStackConfig()
cloud_config = config_obj.get_one(cloud=TEST_CLOUD)

compute = cloud.compute
identity = cloud.identity
image = cloud.image
network = cloud.network
orchestration = cloud.orchestration
volume = cloud.volume
load_balancer = cloud.load_balancer

# Check if Object Storage is present on the cloud, else skip
object_store_present = any(service.type == 'object-store' for service
                           in list(identity.services()))
if object_store_present:
    object_store = cloud.object_store

# Check if Manila Shared File System is present on the cloud, else skip
manila_present = any(service.type == 'sharev2' for service
                           in list(identity.services()))
if manila_present:
    shared_file_system = cloud.shared_file_system

mask = "cvp|s_rally|rally_|tempest-|tempest_|spt-test|fio"
full_mask = f"^(?!.*(manual|-static-)).*({mask}).*$"
mask_pattern = re.compile(full_mask, re.IGNORECASE)
stack_mask = "api-[0-9]+-[a-z]+"
stack_pattern = re.compile(stack_mask, re.IGNORECASE)


def get_resource_value(resource_key, default):
    try:
        return cloud_config.config['custom_vars'][resource_key]
    except KeyError:
        return default


def items_to_object(items):
    data = json.dumps(items)
    return json.loads(data, object_hook=lambda d: SimpleNamespace(**d))


def _filter_test_resources(resources, attribute, pattern=mask_pattern):
    filtered_resources = {}
    for item in resources:
        # If there is no attribute in object, use just empty string as value
        value = getattr(item, attribute, '')
        # If the attribute value is None, use empty string instead, to be
        # able to run regex search
        if value is None:
            value = ''
        found = pattern.match(value)
        if found:
            filtered_resources[item.id] = getattr(item, attribute)
    return filtered_resources


def _log_resources_count(count, resource, pattern=mask):
    log.info(f"{count} {resource} containing '{pattern}' are found.")


def _log_resources_details_at_dry_run(list_to_delete, resource):
    if len(list_to_delete) > 0:
        log.info(f"... {len(list_to_delete)} {resource} would be deleted:")
        for id_ in list_to_delete:
            log.info(f"... ... {id_} ({list_to_delete[id_]})")


def _log_resource_delete(id_, name, type_):
    log.info(f"... deleting {name} (id={id_}) {type_}")


def _skip_tempest_fixed_net(nets_to_delete: dict) -> dict:
    # the "tempest-fixed-net" comes from MOSK itself and should not be deleted
    keys_to_remove = [
        net_id for net_id, name in nets_to_delete.items()
        if name == "tempest-fixed-net"
    ]
    for net_id in keys_to_remove:
        nets_to_delete.pop(net_id, None)
    return nets_to_delete


def _get_volume_groups(all_tenants='true'):
    ep = volume.get_endpoint()
    uri = f"{ep}/groups/detail"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'Accept': 'application/json',
               'OpenStack-API-Version': f'volume {volume_api_version}'}
    params = {'all_tenants': all_tenants}
    response = cloud.session.request(url=uri, method='GET',
                                     headers=headers, params=params).json()
    for group in response['groups']:
        yield group


def _get_shares(all_tenants='true'):
    ep = shared_file_system.get_endpoint()
    uri = f"{ep}/shares/detail"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'Accept': 'application/json',
               'X-Openstack-Manila-Api-Version': f'{manila_api_version}'}
    params = {'all_tenants': all_tenants}
    response = cloud.session.request(url=uri, method='GET',
                                     headers=headers, params=params).json()
    for share in response['shares']:
        yield share


def _get_share_types(all_tenants='true'):
    ep = shared_file_system.get_endpoint()
    uri = f"{ep}/types"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'Accept': 'application/json',
               'X-Openstack-Manila-Api-Version': f'{manila_api_version}'}
    params = {'all_tenants': all_tenants}
    response = cloud.session.request(url=uri, method='GET',
                                     headers=headers, params=params).json()
    for share_type in response['volume_types']:
        yield share_type


def _get_share_networks(all_tenants='true'):
    ep = shared_file_system.get_endpoint()
    uri = f"{ep}/share-networks/detail"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'Accept': 'application/json',
               'X-Openstack-Manila-Api-Version': f'{manila_api_version}'}
    params = {'all_tenants': all_tenants}
    response = cloud.session.request(url=uri, method='GET',
                                     headers=headers, params=params).json()
    for share_network in response['share_networks']:
        yield share_network


def _delete_volume_group(uuid, delete_volumes='false'):
    ep = volume.get_endpoint()
    uri = f"{ep}/groups/{uuid}/action"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'OpenStack-API-Version': f'volume {volume_api_version}',
               'Content-Type': 'application/json',
               'Accept': 'application/json'}
    body = {"delete": {"delete-volumes": delete_volumes}}
    cloud.session.request(
        url=uri, method='POST', headers=headers, json=body)


def _delete_share_type(uuid):
    ep = shared_file_system.get_endpoint()
    uri = f"{ep}/types/{uuid}"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'X-Openstack-Manila-Api-Version': f'{manila_api_version}',
               'Content-Type': 'application/json',
               'Accept': 'application/json'}
    cloud.session.request(
        url=uri, method='DELETE', headers=headers)


def _delete_share(uuid, force_delete=True):
    ep = shared_file_system.get_endpoint()
    uri = f"{ep}/shares/{uuid}/action"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'X-Openstack-Manila-Api-Version': f'{manila_api_version}',
               'Content-Type': 'application/json',
               'Accept': 'application/json'}
    body = {"force_delete": force_delete}
    cloud.session.request(
        url=uri, method='POST', headers=headers, json=body)


def _delete_share_network(uuid):
    ep = shared_file_system.get_endpoint()
    uri = f"{ep}/share-networks/{uuid}"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'X-Openstack-Manila-Api-Version': f'{manila_api_version}',
               'Content-Type': 'application/json',
               'Accept': 'application/json'}
    cloud.session.request(
        url=uri, method='DELETE', headers=headers)


def _reset_volume_status(uuid, status='available', attach_status='detached',
                         migration_status='None'):
    ep = volume.get_endpoint()
    uri = f"{ep}/volumes/{uuid}/action"
    headers = {'X-Auth-Token': cloud.session.get_token(),
               'Content-Type': 'application/json',
               'Accept': 'application/json'}
    body = {"os-reset_status": {
        "status": status, "attach_status": attach_status,
        "migration_status": migration_status}}
    cloud.session.request(
        url=uri, method='POST', headers=headers, json=body)


def cleanup_users():
    users = identity.users()
    users_to_delete = _filter_test_resources(users, 'name')
    _log_resources_count(len(users_to_delete), 'user(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(users_to_delete, 'user(s)')
        return
    for id_ in users_to_delete:
        _log_resource_delete(id_, users_to_delete[id_], 'user')
        identity.delete_user(id_)


def cleanup_roles():
    roles = identity.roles()
    roles_to_delete = _filter_test_resources(roles, 'name')
    _log_resources_count(len(roles_to_delete), 'role(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(roles_to_delete, 'role(s)')
        return
    for id_ in roles_to_delete:
        _log_resource_delete(id_, roles_to_delete[id_], 'role')
        identity.delete_role(id_)


def cleanup_projects():
    projects = identity.projects()
    projects_to_delete = _filter_test_resources(projects, 'name')
    _log_resources_count(len(projects_to_delete), 'project(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(projects_to_delete, 'project(s)')
        return
    for id_ in projects_to_delete:
        _log_resource_delete(id_, projects_to_delete[id_], 'project')
        identity.delete_project(id_)


def cleanup_regions():
    regions = identity.regions()
    regions_to_delete = _filter_test_resources(regions, 'id')
    _log_resources_count(len(regions_to_delete), 'region(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(regions_to_delete, 'region(s)')
        return
    for id_ in regions_to_delete:
        _log_resource_delete(id_, id_, 'region')
        identity.delete_region(id_)


def cleanup_services():
    services = identity.services()
    services_to_delete = _filter_test_resources(services, 'name')
    _log_resources_count(len(services_to_delete), 'service(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(services_to_delete, 'service(s)')
        return
    for id_ in services_to_delete:
        _log_resource_delete(id_, services_to_delete[id_], 'service')
        identity.delete_service(id_)


def cleanup_stacks(stacks_alt=False):
    stacks = orchestration.stacks()
    stacks_to_delete = _filter_test_resources(stacks, 'name')
    _log_resources_count(len(stacks_to_delete), 'stack(s)')

    # Use additional pattern for searching/deleting test Heat resources,
    # if enabled
    if stacks_alt:
        stacks_alt_to_delete = _filter_test_resources(
            stacks, 'name', stack_pattern)
        _log_resources_count(len(stacks_alt_to_delete), 'stack(s)', stack_mask)
        stacks_to_delete.update(stacks_alt_to_delete)

    if args.dry_run:
        _log_resources_details_at_dry_run(stacks_to_delete, 'stack(s)')
        return

    for id_ in stacks_to_delete:
        _log_resource_delete(id_, stacks_to_delete[id_], 'stack')
        stack_obj = orchestration.get_stack(id_)
        orchestration.delete_stack(id_)
        orchestration.wait_for_delete(stack_obj)


def cleanup_flavors():
    flavors = compute.flavors()
    flavors_to_delete = _filter_test_resources(flavors, 'name')
    _log_resources_count(len(flavors_to_delete), 'flavor(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(flavors_to_delete, 'flavor(s)')
        return
    for id_ in flavors_to_delete:
        _log_resource_delete(id_, flavors_to_delete[id_], 'flavor')
        compute.delete_flavor(id_)


def cleanup_images():
    images = image.images()
    images_to_delete = _filter_test_resources(images, 'name')
    _log_resources_count(len(images_to_delete), 'image(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(images_to_delete, 'image(s)')
        return
    for id_ in images_to_delete:
        _log_resource_delete(id_, images_to_delete[id_], 'image')
        image.delete_image(id_)


def cleanup_keypairs():
    keypairs = compute.keypairs()
    keypairs_to_delete = _filter_test_resources(keypairs, 'name')
    _log_resources_count(len(keypairs_to_delete), 'keypair(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(keypairs_to_delete, 'keypair(s)')
        return
    for id_ in keypairs_to_delete:
        _log_resource_delete(id_, keypairs_to_delete[id_], 'keypair')
        compute.delete_keypair(id_)


def cleanup_servers():
    servers = compute.servers(all_projects=True)
    servers_to_delete = _filter_test_resources(servers, 'name')
    _log_resources_count(len(servers_to_delete), 'server(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(servers_to_delete, 'server(s)')
        return
    for id_ in servers_to_delete:
        if args.servers_active:
            log.info(
                f"... resetting {servers_to_delete[id_]} (id={id_}) server "
                "state to 'active'")
            compute.reset_server_state(id_, 'active')
        _log_resource_delete(id_, servers_to_delete[id_], 'server')
        compute.delete_server(id_)
        srv_obj = compute.get_server(id_)
        compute.wait_for_delete(srv_obj)


def cleanup_shares():
    shares_in_response = _get_shares()
    shares = items_to_object([g for g in shares_in_response])
    shares_to_delete = _filter_test_resources(shares, 'name')
    _log_resources_count(len(shares_to_delete), 'share(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(shares_to_delete, 'share(s)')
        return
    for id_ in shares_to_delete:
        _log_resource_delete(id_, shares_to_delete[id_], 'share')
        # TODO: uncomment
        _delete_share(id_)
        shr_obj = shared_file_system.get_share(id_)
        shared_file_system.wait_for_delete(shr_obj)


def cleanup_share_types():
    share_types_in_response = _get_share_types()
    share_types = items_to_object([g for g in share_types_in_response])
    share_types_to_delete = _filter_test_resources(share_types, 'name')
    _log_resources_count(len(share_types_to_delete), 'share type(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(share_types_to_delete,
                                          'share_type(s)')
        return
    for id_ in share_types_to_delete:
        _log_resource_delete(id_, share_types_to_delete[id_], 'type')
        _delete_share_type(id_)


def cleanup_share_networks():
    share_networks_in_response = _get_share_networks()
    share_networks = items_to_object([g for g in share_networks_in_response])
    share_networks_to_delete = _filter_test_resources(share_networks, 'name')
    _log_resources_count(len(share_networks_to_delete), 'share network(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(share_networks_to_delete,
                                          'share_network(s)')
        return
    for id_ in share_networks_to_delete:
        _log_resource_delete(id_, share_networks_to_delete[id_], 'type')
        _delete_share_network(id_)


def cleanup_snapshots():
    snapshots = volume.snapshots(all_projects=True)
    snapshots_to_delete = _filter_test_resources(snapshots, 'name')
    _log_resources_count(len(snapshots_to_delete), 'snapshot(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(snapshots_to_delete, 'snapshot(s)')
        return
    for id_ in snapshots_to_delete:
        snapshot_obj = volume.get_snapshot(id_)
        volume.reset_snapshot(id_, 'available')
        _log_resource_delete(id_, snapshots_to_delete[id_], 'snapshot')
        volume.delete_snapshot(id_, force=True)
        volume.wait_for_delete(snapshot_obj)


def cleanup_volumes():
    volumes = volume.volumes(all_projects=True)
    volumes_to_delete = _filter_test_resources(volumes, 'name')
    _log_resources_count(len(volumes_to_delete), 'volume(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(volumes_to_delete, 'volume(s)')
        return
    for id_ in volumes_to_delete:
        _reset_volume_status(id_, 'available', 'detached', 'None')
        _log_resource_delete(id_, volumes_to_delete[id_], 'volume')
        volume.delete_volume(id_)
        vol_obj = volume.get_volume(id_)
        volume.wait_for_delete(vol_obj)


def cleanup_volume_groups():
    groups_in_response = _get_volume_groups()
    groups = items_to_object([g for g in groups_in_response])
    groups_to_delete = _filter_test_resources(groups, 'name')
    _log_resources_count(len(groups_to_delete), 'volume group(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(groups_to_delete, 'volume group(s)')
        return
    for id_ in groups_to_delete:
        _log_resource_delete(id_, groups_to_delete[id_], 'volume group')
        _delete_volume_group(id_)
        time.sleep(10)      # TODO(izadorozhna): need to add a proper waiter


def cleanup_volume_backups():
    backups = volume.backups(all_tenants=True)
    backups_to_delete = _filter_test_resources(backups, 'name')
    _log_resources_count(len(backups_to_delete), 'volume backup(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(backups_to_delete,
                                          'volume backup(s)')
        return
    for id_ in backups_to_delete:
        backup_obj = volume.get_backup(id_)
        _log_resource_delete(id_, backups_to_delete[id_], 'volume backup')
        volume.delete_backup(id_)
        volume.wait_for_delete(backup_obj)


def cleanup_volume_group_types():
    group_types = volume.group_types()
    group_types_to_delete = _filter_test_resources(group_types, 'name')
    _log_resources_count(len(group_types_to_delete), 'volume group type(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(group_types_to_delete,
                                          'volume group type(s)')
        return
    for id_ in group_types_to_delete:
        _log_resource_delete(
            id_, group_types_to_delete[id_], 'volume group type')
        volume.delete_group_type(id_)


def cleanup_volume_types():
    volume_types = volume.types()
    volume_types_to_delete = _filter_test_resources(volume_types, 'name')
    _log_resources_count(len(volume_types_to_delete), 'volume type(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(volume_types_to_delete,
                                          'volume type(s)')
        return
    for id_ in volume_types_to_delete:
        _log_resource_delete(id_, volume_types_to_delete[id_], 'volume type')
        volume.delete_type(id_)


def cleanup_sec_groups():
    sec_groups = network.security_groups()
    sec_groups_to_delete = _filter_test_resources(sec_groups, 'name')
    _log_resources_count(len(sec_groups_to_delete), 'security group(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(sec_groups_to_delete,
                                          'security group(s)')
        return
    for id_ in sec_groups_to_delete:
        _log_resource_delete(id_, sec_groups_to_delete[id_], 'security group')
        network.delete_security_group(id_)


def cleanup_containers():
    containers = object_store.containers()
    containers_to_delete = _filter_test_resources(containers, 'name')
    _log_resources_count(len(containers_to_delete), 'container(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(containers_to_delete, 'container(s)')
        return
    for id_ in containers_to_delete:
        _log_resource_delete(id_, containers_to_delete[id_], 'container')
        object_store.delete_container(id_)


def cleanup_routers():
    routers = network.routers()
    routers_to_delete = _filter_test_resources(routers, 'name')
    _log_resources_count(len(routers_to_delete), 'router(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(routers_to_delete, 'router(s)')
        return
    for id_ in routers_to_delete:
        _log_resource_delete(id_, routers_to_delete[id_], 'router')

        # Unset external gateway and remove ports from router
        log.info("... ... removing external gateway from the router")
        network.update_router(id_, external_gateway_info={})
        ports = network.ports(device_id=id_)
        for p in ports:
            if p.device_owner != 'network:router_ha_interface':
                log.info(f"... ... removing port {p.id} from the router")
                network.remove_interface_from_router(id_, port_id=p.id)

        network.delete_router(id_)


def cleanup_networks():
    nets = network.networks()
    nets_to_delete = _filter_test_resources(nets, 'name')
    _skip_tempest_fixed_net(nets_to_delete)
    _log_resources_count(len(nets_to_delete), 'network(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(nets_to_delete, 'network(s)')
        return
    for id_ in nets_to_delete:
        _log_resource_delete(id_, nets_to_delete[id_], 'network')

        ports = network.ports(network_id=id_)
        for p in ports:
            log.info(
                f"... ... removing port {p.id} from the network")
            network.delete_port(p.id)
        subnets = network.subnets(network_id=id_)
        for s in subnets:
            log.info(
                f"... ... removing subnet {s.id} from the network")
            network.delete_subnet(s.id)

        network.delete_network(id_)


def cleanup_load_balancers():
    lbs = load_balancer.load_balancers()
    lbs_to_delete = _filter_test_resources(lbs, 'name')
    _log_resources_count(len(lbs_to_delete), 'load_balancer(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(lbs_to_delete, 'load_balancer(s)')
        return
    for id_ in lbs_to_delete:
        _log_resource_delete(id_, lbs_to_delete[id_], 'load_balancer')
        try:
            load_balancer.delete_load_balancer(id_, cascade=True)
        except openstack.exceptions.ConflictException:
            # force delete the LB in case it is in some PENDING_* state
            log.info(f"... ... force deleting {id_} load balancer")
            load_balancer.delete_load_balancer(id_, cascade=True, force=True)
        except Exception as e:
            log.info(f"... ... could not delete {id_} load balancer: {e}")


def cleanup_floating_ips():
    projects = identity.projects()
    list_projects_to_delete = list(_filter_test_resources(projects, 'name'))
    floating_ips = network.ips()
    fips_to_delete = {}
    for ip in floating_ips:
        # filter only non-associated IPs, only inside target projects
        if (ip.status == 'DOWN') and (ip.fixed_ip_address is None):
            if ip.project_id in list_projects_to_delete:
                fips_to_delete[ip.id] = ip.floating_ip_address
    _log_resources_count(len(fips_to_delete), 'floating ip(s)')
    if args.dry_run:
        _log_resources_details_at_dry_run(fips_to_delete, 'floating ip(s)')
        return
    for id_ in fips_to_delete:
        _log_resource_delete(id_, fips_to_delete[id_], 'floating ip')
        network.delete_ip(id_)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description='OpenStack test resources cleanup script')
    parser.add_argument(
        '-t', dest='dry_run', action='store_true',
        help='Dry run mode, no cleanup is done')
    parser.add_argument(
        '-P', dest='projects', action='store_true',
        help='Force cleanup of projects')
    parser.add_argument(
        '-S', dest='servers_active', action='store_true',
        help='Set servers to ACTIVE before deletion (reqiured by bare metal)')
    parser.add_argument(
        '-f', dest='stacks_alt', action='store_true',
        help='Use additional mask for stack cleanup')

    args = parser.parse_args()

    if args.dry_run:
        log.info("Running in dry-run mode")
    if args.servers_active:
        log.info("Servers will be set to ACTIVE before cleanup")
    if args.projects:
        log.info("Project cleanup is enabled")
    if args.stacks_alt:
        log.info(
            f"Stacks will be cleaned up using additional '{stack_mask}' mask")

    cleanup_stacks(stacks_alt=args.stacks_alt)
    cleanup_load_balancers()

    if manila_present:
        cleanup_shares()
        cleanup_share_types()
        cleanup_share_networks()

    cleanup_servers()
    cleanup_flavors()
    try:  # Skip if cinder-backup service is not enabled
        cleanup_volume_backups()
    except openstack.exceptions.ResourceNotFound:
        pass
    cleanup_snapshots()
    cleanup_volumes()
    cleanup_volume_groups()
    cleanup_volume_group_types()
    cleanup_volume_types()
    cleanup_images()
    cleanup_keypairs()
    cleanup_users()
    cleanup_roles()
    cleanup_services()
    cleanup_regions()
    cleanup_routers()
    cleanup_networks()
    cleanup_sec_groups()
    if object_store_present:
        cleanup_containers()
    cleanup_floating_ips()

    if args.projects:
        cleanup_projects()

    if args.dry_run:
        msg = "DRY-RUN mode, no cleanup is done"
    else:
        msg = "Cleanup is FINISHED"
    log.info(f"\n{'=' * len(msg)}\n{msg}")