Updates to the image for MOSK 25.2.x
* fixed an issue in mos-spt, improve report name and location
* imrpoved cleanup script: show details on dry-run, skip tempest net
* revert to jammy image as base for the toolset dockerfile
* improved ceph collect report location
Related-PROD: PROD-37187
Change-Id: I138d9e550cd244be122ff035bbb67433eb866910
diff --git a/cleanup.sh b/cleanup.sh
index 415d53c..604f3e7 100644
--- a/cleanup.sh
+++ b/cleanup.sh
@@ -1,343 +1,347 @@
-#!/bin/bash
-export OS_INTERFACE='admin'
-mask='cvp\|s_rally\|rally_\|tempest_\|tempest-\|spt-'
-exclude='manual\|-static-'
-stack_alt=false
-stack_regex='api-[0-9]+-[a-z]+'
-dry_run=false
-clean_projects=false
-make_servers_active=false
-serial=false
-batch_size=10
-# granularity values: days,hours,minutes,seconds
-stack_granularity=days
-stack_granularity_value=1
+echo ""
+echo "This script is obsolete. Please use the following script instead: cd /opt/res-files; python3 cleanup.py"
+echo ""
-function show_help {
- printf "Resource cleaning script\nMask is: %s\n\t-h, -?\tShow this help\n" ${mask}
- printf "\t-t\tDry run mode, no cleaning done\n"
- printf "\t-P\tForce cleaning of projects\n"
- printf "\t-s\tUse single thread of 'openstack' client for cleanup\n"
- printf "\t-S\tSet servers to ACTIVE before deletion (bare metal reqiured)\n"
- printf "\t-f\tForce stack cleanup with an additional mask of '%s'\n" ${stack_regex}
- printf "\t-F\tForce purge deleted stacks. Batch size: %s, >%s %s\n" ${batch_size} ${stack_granularity_value} ${stack_granularity}
-}
-
-OPTIND=1 # Reset in case getopts has been used previously in the shell.
-while getopts "h?:tsSPfF" opt; do
- case "$opt" in
- h|\?)
- show_help
- exit 0
- ;;
- t) dry_run=true
- printf "Running in dry-run mode\n"
- ;;
- s) serial=true
- printf "Single threaded mode enabled\n"
- ;;
- S) make_servers_active=true
- printf "Servers will be set to ACTIVE before deletion\n"
- ;;
- P) clean_projects=true
- printf "Project cleanning enabled\n"
- ;;
- f) stack_alt=true
- printf "Cleaning stacks using additional mask '%s'\n" ${stack_regex}
- ;;
- F) purge_deleted_stacks=true
- printf "Purging stacks deleted >$stack_granularity_value $stack_granularity ago enabled, batch size %s\n" $stack_batch_size
- ;;
- esac
-done
-
-shift $((OPTIND-1))
-[ "${1:-}" = "--" ] && shift
-
-### Execute collected commands and flush the temp file
-function _clean_and_flush {
- if [ "$dry_run" = true ] ; then
- return 0
- fi
- if [ -s ${cmds} ]; then
- if [ "${serial}" = false ] ; then
- echo "... processing $(cat ${cmds} | wc -l) commands, worker threads ${batch_size}"
- cat ${cmds} | tr '\n' '\0' | xargs -P ${batch_size} -n 1 -0 echo | openstack
- #cat ${cmds} | openstack
- truncate -s 0 ${cmds}
- else
- echo "... processing $(cat ${cmds} | wc -l) commands"
- cat ${cmds} | tr '\n' '\0' | xargs -P 1 -n 1 -0 echo | openstack
- truncate -s 0 ${cmds}
- fi
- fi
-}
-
-function _clean_and_flush_cinder {
- if [ "$dry_run" = true ] ; then
- return 0
- fi
- if [ -s ${cmds} ]; then
- if [ "${serial}" = false ] ; then
- echo "... processing $(cat ${cmds} | wc -l) commands, worker threads ${batch_size}"
- cat ${cmds} | tr '\n' '\0' | xargs -I{} -P ${batch_size} -n 1 -0 /bin/bash -c 'cinder --os-volume-api-version 3.43 {}'
- #cat ${cmds} | cinder --os-volume-api-version 3.43
- truncate -s 0 ${cmds}
- else
- echo "... processing $(cat ${cmds} | wc -l) commands"
- cat ${cmds} | tr '\n' '\0' | xargs -I{} -P 1 -n 1 -0 /bin/bash -c 'cinder --os-volume-api-version 3.43 {}'
- truncate -s 0 ${cmds}
- fi
- fi
-}
-
-### Users
-function _clean_users {
- users=( $(openstack user list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#users[@]} users containing '${mask}' found"
- printf "%s\n" ${users[@]} | xargs -I{} echo user delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Roles
-function _clean_roles {
- roles=( $(openstack role list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#roles[@]} roles containing '${mask}' found"
- printf "%s\n" ${roles[@]} | xargs -I{} echo role delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Projects
-function _clean_projects {
- projects=( $(openstack project list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#projects[@]} projects containing '${mask}' found"
- printf "%s\n" ${projects[@]} | xargs -I{} echo project delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Servers
-function _clean_servers {
- servers=( $(openstack server list -c ID -c Name -f value --all | grep "${mask}" | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#servers[@]} servers containing '${mask}' found"
- if [ "$make_servers_active" = true ]; then
- printf "%s\n" ${servers[@]} | xargs -I{} echo server set --state active {} >>${cmds}
- fi
- printf "%s\n" ${servers[@]} | xargs -I{} echo server delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Reset snapshot state and delete
-function _clean_snapshots {
- snapshots=( $(openstack volume snapshot list --all -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#snapshots[@]} snapshots containing '${mask}' found"
- printf "%s\n" ${snapshots[@]} | xargs -I{} echo volume snapshot set --state available {} >>${cmds}
- printf "%s\n" ${snapshots[@]} | xargs -I{} echo volume snapshot delete {} >>${cmds}
- _clean_and_flush
-}
-
-function _clean_volumes {
- volumes=( $(openstack volume list --all -c ID -c Name -c Type -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#volumes[@]} volumes containing '${mask}' found"
- printf "%s\n" ${volumes[@]} | xargs -I{} echo volume set --state available {} >>${cmds}
- printf "%s\n" ${volumes[@]} | xargs -I{} echo volume delete {} >>${cmds}
- _clean_and_flush
-}
-
-function _clean_volume_groups {
- groups=( $(cinder --os-volume-api-version 3.43 group-list --all-tenants 1 | grep ${mask} | grep -v ${exclude} | awk '{print $2}') )
- echo "-> ${#groups[@]} groups containing '${mask}' found"
- printf "%s\n" ${groups[@]} | xargs -I{} echo group-delete {} >>${cmds}
- _clean_and_flush_cinder
-}
-
-function _clean_volume_group_types {
- group_types=( $(cinder --os-volume-api-version 3.43 group-type-list | grep ${mask} | grep -v ${exclude} | awk '{print $2}') )
- echo "-> ${#group_types[@]} group types containing '${mask}' found"
- printf "%s\n" ${group_types[@]} | xargs -I{} echo group-type-delete {} >>${cmds}
- _clean_and_flush_cinder
-}
-
-### Volume types
-function _clean_volume_types {
- vtypes=( $(openstack volume type list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#vtypes[@]} volume types containing '${mask}' found"
- printf "%s\n" ${vtypes[@]} | xargs -I{} echo volume type delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Images
-function _clean_images {
- images=( $(openstack image list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#images[@]} images containing '${mask}' found"
- printf "%s\n" ${images[@]} | xargs -I{} echo image delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Sec groups
-function _clean_sec_groups {
- projects=( $(openstack project list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- sgroups=( $(printf "%s\n" ${projects[@]} | xargs -I{} /bin/bash -c "openstack security group list -c ID -c Project -f value | grep {} | cut -d' ' -f1") )
- echo "-> ${#sgroups[@]} security groups for project containing '${mask}' found"
- printf "%s\n" ${sgroups[@]} | xargs -I{} echo security group delete {} >>${cmds}
- _clean_and_flush
-
- # Additional step to cleanup 'hanged' groups
- sgroups_raw=( $(openstack security group list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#sgroups_raw[@]} security groups for '${mask}' found"
- printf "%s\n" ${sgroups_raw[@]} | xargs -I{} echo security group delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Keypairs
-function _clean_keypairs {
- keypairs=( $(openstack keypair list -c Name -f value | grep ${mask} | grep -v ${exclude}) )
- echo "-> ${#keypairs[@]} keypairs containing '${mask}' found"
- printf "%s\n" ${keypairs[@]} | xargs -I{} echo keypair delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Routers and Networks
-function _clean_routers_and_networks {
- routers=( $(openstack router list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d ' ' -f1) )
- if [ ${#routers[@]} -eq 0 ]; then
- echo "-> No routers containing '${mask}' found"
- else
- echo "-> ${#routers[@]} routers containing '${mask}' found"
- echo "... unsetting gateways"
- printf "%s\n" ${routers[@]} | xargs -I{} echo router unset --external-gateway {} >>${cmds}
- _clean_and_flush
-
- echo "... removing ports"
- for router in ${routers[@]}; do
- r_ports=( $(openstack port list --router ${router} -f value -c ID) )
- if [ ${#r_ports[@]} -eq 0 ]; then
- echo "... no ports to unplug for ${router}"
- else
- for r_port in ${r_ports[@]}; do
- echo "... queued removal of port '${r_port}' from router '${router}'"
- echo "router remove port ${router} ${r_port}" >>${cmds}
- done
- fi
- done
- _clean_and_flush
-
- echo "... deleting routers"
- printf "%s\n" ${routers[@]} | xargs -I{} echo router delete {} >>${cmds}
- _clean_and_flush
- fi
-
- networks=( $(openstack network list | grep "${mask}" | grep -v ${exclude} | cut -d' ' -f2) )
- if [ ${#networks[@]} -eq 0 ]; then
- echo "-> No networks containing '${mask}' found"
- else
- ports=()
- subnets=()
- for((idx=0;idx<${#networks[@]};idx++)) do
- ports+=( $(openstack port list --network ${networks[idx]} -c ID -f value) )
- subnets+=( $(openstack subnet list --network ${networks[idx]} -c ID -f value) )
- echo "-> $((${idx}+1)) of ${#networks[@]}, total ${#ports[@]} ports, ${#subnets[@]} subnets"
- done
- printf "%s\n" ${ports[@]} | xargs -I{} echo port delete {} >>${cmds}
- printf "%s\n" ${subnets[@]} | xargs -I{} echo subnet delete {} >>${cmds}
- echo network delete ${networks[@]} >>${cmds}
- echo "-> ${#routers[@]} routers, ${#ports[@]} ports, ${#subnets[@]} subnets, ${#networks[@]} networks"
- fi
- _clean_and_flush
-}
-
-### Regions
-function _clean_regions {
- regions=( $(openstack region list -c Region -f value | grep ${mask} | grep -v ${exclude}) )
- echo "-> ${#regions[@]} regions containing '${mask}' found"
- printf "%s\n" ${regions[@]} | xargs -I{} echo region delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Services
-function _clean_services {
- services=( $(openstack service list -c Name -f value | grep ${mask} | grep -v ${exclude}) )
- echo "-> ${#services[@]} services containing '${mask}' found"
- printf "%s\n" ${services[@]} | xargs -I{} echo service delete {} >>${cmds}
- _clean_and_flush
-}
-
-### Stacks
-function _clean_stacks {
- # By default openstack denies use of global_index for everyone.
- # In case you want to have handy cleanup, consider updating policy.json here:
- # root@ctl0x:~# cat -n /etc/heat/policy.json | grep global_index
- # 48 "stacks:global_index": "rule:deny_everybody",
- # 73 "software_configs:global_index": "rule:deny_everybody",
- # After this you will be able to use --all option
-
- stacks=( $(openstack stack list --nested --hidden -c ID -c "Stack Name" -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#stacks[@]} stacks containing '${mask}' found"
- printf "%s\n" ${stacks[@]} | xargs -I{} echo stack check {} >>${cmds}
- printf "%s\n" ${stacks[@]} | xargs -I{} echo stack delete -y {} >>${cmds}
- _clean_and_flush
-
- if [ "$stack_alt" = true ]; then
- stacks=( $(openstack stack list --nested --hidden -c ID -c "Stack Name" -f value | grep -E ${stack_regex} | cut -d' ' -f1) )
- echo "-> ${#stacks[@]} stacks containing '${stack_regex}' found"
- printf "%s\n" ${stacks[@]} | xargs -I{} echo stack check {} >>${cmds}
- printf "%s\n" ${stacks[@]} | xargs -I{} echo stack delete -y {} >>${cmds}
- _clean_and_flush
- fi
-
- if [ "$purge_deleted_stacks" = true ]; then
- heat-manage purge_deleted -g ${stack_granularity} -b ${batch_size} ${stack_granularity_value} | wc -l | xargs -I{} echo "-> Purged {} stacks"
- fi
-}
-
-### Containers
-function _clean_containers {
- containers=( $(openstack container list --all -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#containers[@]} containers containing '${mask}' found"
- printf "%s\n" ${containers[@]} | xargs -I{} echo container delete {} >>${cmds}
- _clean_and_flush
-}
-
-function _clean_flavors {
- flavors=( $(openstack flavor list --all -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
- echo "-> ${#flavors[@]} flavors containing '${mask}' found"
- printf "%s\n" ${flavors[@]} | xargs -I{} echo flavor delete {} >>${cmds}
- _clean_and_flush
-}
-
-###################
-### Main
-###################
-# temp file for commands
-cmds=$(mktemp)
-trap "rm -f ${cmds}" EXIT
-echo "Using tempfile: '${cmds}'"
-
-# Consider cleaning contrail resources carefully
-# ...and only after that - clean projects
-
-_clean_stacks
-_clean_servers
-_clean_flavors
-_clean_users
-_clean_roles
-_clean_snapshots
-_clean_volumes
-_clean_volume_groups
-_clean_volume_group_types
-_clean_volume_types
-_clean_images
-_clean_sec_groups
-_clean_keypairs
-_clean_routers_and_networks
-_clean_regions
-_clean_services
-_clean_containers
-
-# project cleaning disabled by default
-# Coz cleaning Contrail with no projects is a hard task
-if [ "$clean_projects" = true ]; then
- _clean_projects
-fi
-
-# remove temp file
-rm ${cmds}
+##!/bin/bash
+#export OS_INTERFACE='admin'
+#mask='cvp\|s_rally\|rally_\|tempest_\|tempest-\|spt-'
+#exclude='manual\|-static-'
+#stack_alt=false
+#stack_regex='api-[0-9]+-[a-z]+'
+#dry_run=false
+#clean_projects=false
+#make_servers_active=false
+#serial=false
+#batch_size=10
+## granularity values: days,hours,minutes,seconds
+#stack_granularity=days
+#stack_granularity_value=1
+#
+#function show_help {
+# printf "Resource cleaning script\nMask is: %s\n\t-h, -?\tShow this help\n" ${mask}
+# printf "\t-t\tDry run mode, no cleaning done\n"
+# printf "\t-P\tForce cleaning of projects\n"
+# printf "\t-s\tUse single thread of 'openstack' client for cleanup\n"
+# printf "\t-S\tSet servers to ACTIVE before deletion (bare metal reqiured)\n"
+# printf "\t-f\tForce stack cleanup with an additional mask of '%s'\n" ${stack_regex}
+# printf "\t-F\tForce purge deleted stacks. Batch size: %s, >%s %s\n" ${batch_size} ${stack_granularity_value} ${stack_granularity}
+#}
+#
+#OPTIND=1 # Reset in case getopts has been used previously in the shell.
+#while getopts "h?:tsSPfF" opt; do
+# case "$opt" in
+# h|\?)
+# show_help
+# exit 0
+# ;;
+# t) dry_run=true
+# printf "Running in dry-run mode\n"
+# ;;
+# s) serial=true
+# printf "Single threaded mode enabled\n"
+# ;;
+# S) make_servers_active=true
+# printf "Servers will be set to ACTIVE before deletion\n"
+# ;;
+# P) clean_projects=true
+# printf "Project cleanning enabled\n"
+# ;;
+# f) stack_alt=true
+# printf "Cleaning stacks using additional mask '%s'\n" ${stack_regex}
+# ;;
+# F) purge_deleted_stacks=true
+# printf "Purging stacks deleted >$stack_granularity_value $stack_granularity ago enabled, batch size %s\n" $stack_batch_size
+# ;;
+# esac
+#done
+#
+#shift $((OPTIND-1))
+#[ "${1:-}" = "--" ] && shift
+#
+#### Execute collected commands and flush the temp file
+#function _clean_and_flush {
+# if [ "$dry_run" = true ] ; then
+# return 0
+# fi
+# if [ -s ${cmds} ]; then
+# if [ "${serial}" = false ] ; then
+# echo "... processing $(cat ${cmds} | wc -l) commands, worker threads ${batch_size}"
+# cat ${cmds} | tr '\n' '\0' | xargs -P ${batch_size} -n 1 -0 echo | openstack
+# #cat ${cmds} | openstack
+# truncate -s 0 ${cmds}
+# else
+# echo "... processing $(cat ${cmds} | wc -l) commands"
+# cat ${cmds} | tr '\n' '\0' | xargs -P 1 -n 1 -0 echo | openstack
+# truncate -s 0 ${cmds}
+# fi
+# fi
+#}
+#
+#function _clean_and_flush_cinder {
+# if [ "$dry_run" = true ] ; then
+# return 0
+# fi
+# if [ -s ${cmds} ]; then
+# if [ "${serial}" = false ] ; then
+# echo "... processing $(cat ${cmds} | wc -l) commands, worker threads ${batch_size}"
+# cat ${cmds} | tr '\n' '\0' | xargs -I{} -P ${batch_size} -n 1 -0 /bin/bash -c 'cinder --os-volume-api-version 3.43 {}'
+# #cat ${cmds} | cinder --os-volume-api-version 3.43
+# truncate -s 0 ${cmds}
+# else
+# echo "... processing $(cat ${cmds} | wc -l) commands"
+# cat ${cmds} | tr '\n' '\0' | xargs -I{} -P 1 -n 1 -0 /bin/bash -c 'cinder --os-volume-api-version 3.43 {}'
+# truncate -s 0 ${cmds}
+# fi
+# fi
+#}
+#
+#### Users
+#function _clean_users {
+# users=( $(openstack user list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#users[@]} users containing '${mask}' found"
+# printf "%s\n" ${users[@]} | xargs -I{} echo user delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Roles
+#function _clean_roles {
+# roles=( $(openstack role list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#roles[@]} roles containing '${mask}' found"
+# printf "%s\n" ${roles[@]} | xargs -I{} echo role delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Projects
+#function _clean_projects {
+# projects=( $(openstack project list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#projects[@]} projects containing '${mask}' found"
+# printf "%s\n" ${projects[@]} | xargs -I{} echo project delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Servers
+#function _clean_servers {
+# servers=( $(openstack server list -c ID -c Name -f value --all | grep "${mask}" | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#servers[@]} servers containing '${mask}' found"
+# if [ "$make_servers_active" = true ]; then
+# printf "%s\n" ${servers[@]} | xargs -I{} echo server set --state active {} >>${cmds}
+# fi
+# printf "%s\n" ${servers[@]} | xargs -I{} echo server delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Reset snapshot state and delete
+#function _clean_snapshots {
+# snapshots=( $(openstack volume snapshot list --all -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#snapshots[@]} snapshots containing '${mask}' found"
+# printf "%s\n" ${snapshots[@]} | xargs -I{} echo volume snapshot set --state available {} >>${cmds}
+# printf "%s\n" ${snapshots[@]} | xargs -I{} echo volume snapshot delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#function _clean_volumes {
+# volumes=( $(openstack volume list --all -c ID -c Name -c Type -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#volumes[@]} volumes containing '${mask}' found"
+# printf "%s\n" ${volumes[@]} | xargs -I{} echo volume set --state available {} >>${cmds}
+# printf "%s\n" ${volumes[@]} | xargs -I{} echo volume delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#function _clean_volume_groups {
+# groups=( $(cinder --os-volume-api-version 3.43 group-list --all-tenants 1 | grep ${mask} | grep -v ${exclude} | awk '{print $2}') )
+# echo "-> ${#groups[@]} groups containing '${mask}' found"
+# printf "%s\n" ${groups[@]} | xargs -I{} echo group-delete {} >>${cmds}
+# _clean_and_flush_cinder
+#}
+#
+#function _clean_volume_group_types {
+# group_types=( $(cinder --os-volume-api-version 3.43 group-type-list | grep ${mask} | grep -v ${exclude} | awk '{print $2}') )
+# echo "-> ${#group_types[@]} group types containing '${mask}' found"
+# printf "%s\n" ${group_types[@]} | xargs -I{} echo group-type-delete {} >>${cmds}
+# _clean_and_flush_cinder
+#}
+#
+#### Volume types
+#function _clean_volume_types {
+# vtypes=( $(openstack volume type list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#vtypes[@]} volume types containing '${mask}' found"
+# printf "%s\n" ${vtypes[@]} | xargs -I{} echo volume type delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Images
+#function _clean_images {
+# images=( $(openstack image list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#images[@]} images containing '${mask}' found"
+# printf "%s\n" ${images[@]} | xargs -I{} echo image delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Sec groups
+#function _clean_sec_groups {
+# projects=( $(openstack project list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# sgroups=( $(printf "%s\n" ${projects[@]} | xargs -I{} /bin/bash -c "openstack security group list -c ID -c Project -f value | grep {} | cut -d' ' -f1") )
+# echo "-> ${#sgroups[@]} security groups for project containing '${mask}' found"
+# printf "%s\n" ${sgroups[@]} | xargs -I{} echo security group delete {} >>${cmds}
+# _clean_and_flush
+#
+# # Additional step to cleanup 'hanged' groups
+# sgroups_raw=( $(openstack security group list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#sgroups_raw[@]} security groups for '${mask}' found"
+# printf "%s\n" ${sgroups_raw[@]} | xargs -I{} echo security group delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Keypairs
+#function _clean_keypairs {
+# keypairs=( $(openstack keypair list -c Name -f value | grep ${mask} | grep -v ${exclude}) )
+# echo "-> ${#keypairs[@]} keypairs containing '${mask}' found"
+# printf "%s\n" ${keypairs[@]} | xargs -I{} echo keypair delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Routers and Networks
+#function _clean_routers_and_networks {
+# routers=( $(openstack router list -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d ' ' -f1) )
+# if [ ${#routers[@]} -eq 0 ]; then
+# echo "-> No routers containing '${mask}' found"
+# else
+# echo "-> ${#routers[@]} routers containing '${mask}' found"
+# echo "... unsetting gateways"
+# printf "%s\n" ${routers[@]} | xargs -I{} echo router unset --external-gateway {} >>${cmds}
+# _clean_and_flush
+#
+# echo "... removing ports"
+# for router in ${routers[@]}; do
+# r_ports=( $(openstack port list --router ${router} -f value -c ID) )
+# if [ ${#r_ports[@]} -eq 0 ]; then
+# echo "... no ports to unplug for ${router}"
+# else
+# for r_port in ${r_ports[@]}; do
+# echo "... queued removal of port '${r_port}' from router '${router}'"
+# echo "router remove port ${router} ${r_port}" >>${cmds}
+# done
+# fi
+# done
+# _clean_and_flush
+#
+# echo "... deleting routers"
+# printf "%s\n" ${routers[@]} | xargs -I{} echo router delete {} >>${cmds}
+# _clean_and_flush
+# fi
+#
+# networks=( $(openstack network list | grep "${mask}" | grep -v ${exclude} | cut -d' ' -f2) )
+# if [ ${#networks[@]} -eq 0 ]; then
+# echo "-> No networks containing '${mask}' found"
+# else
+# ports=()
+# subnets=()
+# for((idx=0;idx<${#networks[@]};idx++)) do
+# ports+=( $(openstack port list --network ${networks[idx]} -c ID -f value) )
+# subnets+=( $(openstack subnet list --network ${networks[idx]} -c ID -f value) )
+# echo "-> $((${idx}+1)) of ${#networks[@]}, total ${#ports[@]} ports, ${#subnets[@]} subnets"
+# done
+# printf "%s\n" ${ports[@]} | xargs -I{} echo port delete {} >>${cmds}
+# printf "%s\n" ${subnets[@]} | xargs -I{} echo subnet delete {} >>${cmds}
+# echo network delete ${networks[@]} >>${cmds}
+# echo "-> ${#routers[@]} routers, ${#ports[@]} ports, ${#subnets[@]} subnets, ${#networks[@]} networks"
+# fi
+# _clean_and_flush
+#}
+#
+#### Regions
+#function _clean_regions {
+# regions=( $(openstack region list -c Region -f value | grep ${mask} | grep -v ${exclude}) )
+# echo "-> ${#regions[@]} regions containing '${mask}' found"
+# printf "%s\n" ${regions[@]} | xargs -I{} echo region delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Services
+#function _clean_services {
+# services=( $(openstack service list -c Name -f value | grep ${mask} | grep -v ${exclude}) )
+# echo "-> ${#services[@]} services containing '${mask}' found"
+# printf "%s\n" ${services[@]} | xargs -I{} echo service delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#### Stacks
+#function _clean_stacks {
+# # By default openstack denies use of global_index for everyone.
+# # In case you want to have handy cleanup, consider updating policy.json here:
+# # root@ctl0x:~# cat -n /etc/heat/policy.json | grep global_index
+# # 48 "stacks:global_index": "rule:deny_everybody",
+# # 73 "software_configs:global_index": "rule:deny_everybody",
+# # After this you will be able to use --all option
+#
+# stacks=( $(openstack stack list --nested --hidden -c ID -c "Stack Name" -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#stacks[@]} stacks containing '${mask}' found"
+# printf "%s\n" ${stacks[@]} | xargs -I{} echo stack check {} >>${cmds}
+# printf "%s\n" ${stacks[@]} | xargs -I{} echo stack delete -y {} >>${cmds}
+# _clean_and_flush
+#
+# if [ "$stack_alt" = true ]; then
+# stacks=( $(openstack stack list --nested --hidden -c ID -c "Stack Name" -f value | grep -E ${stack_regex} | cut -d' ' -f1) )
+# echo "-> ${#stacks[@]} stacks containing '${stack_regex}' found"
+# printf "%s\n" ${stacks[@]} | xargs -I{} echo stack check {} >>${cmds}
+# printf "%s\n" ${stacks[@]} | xargs -I{} echo stack delete -y {} >>${cmds}
+# _clean_and_flush
+# fi
+#
+# if [ "$purge_deleted_stacks" = true ]; then
+# heat-manage purge_deleted -g ${stack_granularity} -b ${batch_size} ${stack_granularity_value} | wc -l | xargs -I{} echo "-> Purged {} stacks"
+# fi
+#}
+#
+#### Containers
+#function _clean_containers {
+# containers=( $(openstack container list --all -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#containers[@]} containers containing '${mask}' found"
+# printf "%s\n" ${containers[@]} | xargs -I{} echo container delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+#function _clean_flavors {
+# flavors=( $(openstack flavor list --all -c ID -c Name -f value | grep ${mask} | grep -v ${exclude} | cut -d' ' -f1) )
+# echo "-> ${#flavors[@]} flavors containing '${mask}' found"
+# printf "%s\n" ${flavors[@]} | xargs -I{} echo flavor delete {} >>${cmds}
+# _clean_and_flush
+#}
+#
+####################
+#### Main
+####################
+## temp file for commands
+#cmds=$(mktemp)
+#trap "rm -f ${cmds}" EXIT
+#echo "Using tempfile: '${cmds}'"
+#
+## Consider cleaning contrail resources carefully
+## ...and only after that - clean projects
+#
+#_clean_stacks
+#_clean_servers
+#_clean_flavors
+#_clean_users
+#_clean_roles
+#_clean_snapshots
+#_clean_volumes
+#_clean_volume_groups
+#_clean_volume_group_types
+#_clean_volume_types
+#_clean_images
+#_clean_sec_groups
+#_clean_keypairs
+#_clean_routers_and_networks
+#_clean_regions
+#_clean_services
+#_clean_containers
+#
+## project cleaning disabled by default
+## Coz cleaning Contrail with no projects is a hard task
+#if [ "$clean_projects" = true ]; then
+# _clean_projects
+#fi
+#
+## remove temp file
+#rm ${cmds}