Add workaround for issue with deletion of TF security-group
In case of TungstenFabric we can encounter a issue when security
group is still referred to virtual-machine-interface and TF need
some time to sync changes after deletion of VM.
Also add new config option timeout_409 to achieve control
and flexibility for different environments by setting the timeout
to wait for the successful deletion of Security Group
in case we get 409 HTTP status code (Conflict).
This is squashed 4 commits from Train, additional change-ids are
I832dd8252635860a40b42a9f8c3f8de87f6ed845
Iac9c6f73c6d5b8dadd924b39bbcc9065d41f4fd2
I5d6c55c1f9e581a6d4f4c03161de44757cd8dc7e
Related-PROD: PRODX-4874
Change-Id: Iab1b9bfdbe8413c73bdf47ca20859132e55a3a10
(cherry picked from commit 403756274fc633f7350acb3c8140eb8316ebd8ba)
(cherry picked from commit 107c4b72e12fd08836734843ffcaab1a98a897e1)
diff --git a/tempest/common/credentials_factory.py b/tempest/common/credentials_factory.py
index 2d486a7..449bb90 100644
--- a/tempest/common/credentials_factory.py
+++ b/tempest/common/credentials_factory.py
@@ -87,7 +87,8 @@
('create_networks', (CONF.auth.create_isolated_networks and not
CONF.network.shared_physical_network)),
('resource_prefix', 'tempest'),
- ('identity_admin_endpoint_type', endpoint_type)
+ ('identity_admin_endpoint_type', endpoint_type),
+ ('networking_timeout_409', CONF.network.timeout_409)
]))
diff --git a/tempest/config.py b/tempest/config.py
index d0812cd..7ace984 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -821,6 +821,10 @@
default=False,
help="The environment does not support network separation "
"between tenants."),
+ cfg.IntOpt('timeout_409',
+ default=120,
+ help="Total time in seconds to keep retrying a request that "
+ "returns HTTP 409 (Conflict)."),
]
network_feature_group = cfg.OptGroup(name='network-feature-enabled',
diff --git a/tempest/lib/common/dynamic_creds.py b/tempest/lib/common/dynamic_creds.py
index f334c36..7196945 100644
--- a/tempest/lib/common/dynamic_creds.py
+++ b/tempest/lib/common/dynamic_creds.py
@@ -13,6 +13,7 @@
# under the License.
import ipaddress
+import time
import netaddr
from oslo_log import log as logging
@@ -73,7 +74,8 @@
neutron_available=False, create_networks=True,
project_network_cidr=None, project_network_mask_bits=None,
public_network_id=None, resource_prefix=None,
- identity_admin_endpoint_type='public', identity_uri=None):
+ identity_admin_endpoint_type='public', identity_uri=None,
+ networking_timeout_409=120):
super(DynamicCredentialProvider, self).__init__(
identity_version=identity_version, identity_uri=identity_uri,
admin_role=admin_role, name=name,
@@ -118,6 +120,7 @@
self.roles_admin_client,
self.domains_admin_client,
self.creds_domain_name)
+ self.networking_timeout_409 = networking_timeout_409
def _get_admin_clients(self, endpoint_type):
"""Returns a tuple with instances of the following admin clients
@@ -515,11 +518,24 @@
name="default")
secgroups_to_delete = resp_body['security_groups']
for secgroup in secgroups_to_delete:
- try:
- nsg_client.delete_security_group(secgroup['id'])
- except lib_exc.NotFound:
- LOG.warning('Security group %s, id %s not found for clean-up',
- secgroup['name'], secgroup['id'])
+ # Workaround for PRODX-4003
+ start_time = time.time()
+ while True:
+ try:
+ nsg_client.delete_security_group(secgroup['id'])
+ break
+ except lib_exc.NotFound:
+ LOG.warning('Security group %s, id %s not found for '
+ 'clean-up', secgroup['name'], secgroup['id'])
+ break
+ except lib_exc.Conflict:
+ LOG.warning('Conflict with state of security group %s, '
+ 'id %s.', secgroup['name'], secgroup['id'])
+ if (time.time() - self.networking_timeout_409) > \
+ start_time:
+ raise
+ else:
+ time.sleep(5)
def _clear_isolated_net_resources(self):
client = self.routers_admin_client