Retry subnet/network deletes on 409 Conflict
Neutron can be slow to cleanup ports from subnets/networks.
This patch adds retries when deleting subnets and networks in the
tempest teardown/clean up phase after tests.
Also, there were cases where addClassResourceCleanup was being used
inside test cases instead of addCleanup. This patch corrects those to
use addCleanup.
Story: 2004826
Task: 29000
Change-Id: Ia29541d1c89f3559a3ce22b1a27c6bcf079ce2cc
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py b/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py
index 7b769e1..b2b1d18 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py
@@ -103,7 +103,7 @@
}
hm = self.mem_healthmonitor_client.create_healthmonitor(**hm_kwargs)
- self.addClassResourceCleanup(
+ self.addCleanup(
self.mem_healthmonitor_client.cleanup_healthmonitor,
hm[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py b/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py
index 98d3bc6..9e09f35 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py
@@ -115,7 +115,7 @@
}
l7policy = self.mem_l7policy_client.create_l7policy(**l7policy_kwargs)
- self.addClassResourceCleanup(
+ self.addCleanup(
self.mem_l7policy_client.cleanup_l7policy,
l7policy[const.ID],
lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py b/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py
index 3e14a74..114ea3e 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py
@@ -112,7 +112,7 @@
}
l7rule = self.mem_l7rule_client.create_l7rule(**l7rule_kwargs)
- self.addClassResourceCleanup(
+ self.addCleanup(
self.mem_l7rule_client.cleanup_l7rule,
l7rule[const.ID], l7policy_id=self.l7policy_id,
lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_listener.py b/octavia_tempest_plugin/tests/scenario/v2/test_listener.py
index 685c200..7720d27 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_listener.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_listener.py
@@ -132,7 +132,7 @@
})
listener = self.mem_listener_client.create_listener(**listener_kwargs)
- self.addClassResourceCleanup(
+ self.addCleanup(
self.mem_listener_client.cleanup_listener,
listener[const.ID],
lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py b/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py
index cfc3ee5..79f1876 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py
@@ -59,7 +59,7 @@
self._setup_lb_network_kwargs(lb_kwargs, ip_version)
lb = self.mem_lb_client.create_loadbalancer(**lb_kwargs)
- self.addClassResourceCleanup(
+ self.addCleanup(
self.mem_lb_client.cleanup_loadbalancer,
lb[const.ID])
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_member.py b/octavia_tempest_plugin/tests/scenario/v2/test_member.py
index b30d651..6bd18de 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_member.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_member.py
@@ -123,7 +123,7 @@
const.ID]
member = self.mem_member_client.create_member(**member_kwargs)
- self.addClassResourceCleanup(
+ self.addCleanup(
self.mem_member_client.cleanup_member,
member[const.ID], pool_id=self.pool_id,
lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_pool.py b/octavia_tempest_plugin/tests/scenario/v2/test_pool.py
index 1ddc8b2..cd26687 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_pool.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_pool.py
@@ -108,7 +108,7 @@
pool_kwargs[const.LOADBALANCER_ID] = self.lb_id
pool = self.mem_pool_client.create_pool(**pool_kwargs)
- self.addClassResourceCleanup(
+ self.addCleanup(
self.mem_pool_client.cleanup_pool,
pool[const.ID],
lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/test_base.py b/octavia_tempest_plugin/tests/test_base.py
index a48ba0b..b935409 100644
--- a/octavia_tempest_plugin/tests/test_base.py
+++ b/octavia_tempest_plugin/tests/test_base.py
@@ -30,6 +30,7 @@
from tempest.lib.common.utils.linux import remote_client
from tempest.lib import exceptions
from tempest import test
+import tenacity
from octavia_tempest_plugin import clients
from octavia_tempest_plugin.common import constants as const
@@ -39,6 +40,11 @@
CONF = config.CONF
LOG = logging.getLogger(__name__)
+RETRY_ATTEMPTS = 15
+RETRY_INITIAL_DELAY = 1
+RETRY_BACKOFF = 1
+RETRY_MAX = 5
+
class LoadBalancerBaseTest(test.BaseTestCase):
"""Base class for load balancer tests."""
@@ -204,6 +210,42 @@
cls.lb_member_2_ipv6_subnet[const.ID]))
@classmethod
+ # Neutron can be slow to clean up ports from the subnets/networks.
+ # Retry this delete a few times if we get a "Conflict" error to give
+ # neutron time to fully cleanup the ports.
+ @tenacity.retry(
+ retry=tenacity.retry_if_exception_type(exceptions.Conflict),
+ wait=tenacity.wait_incrementing(
+ RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
+ stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
+ def _logging_delete_network(cls, net_id):
+ try:
+ cls.lb_mem_net_client.delete_network(net_id)
+ except Exception:
+ LOG.error('Unable to delete network {}. Active ports:'.format(
+ net_id))
+ LOG.error(cls.lb_mem_ports_client.list_ports())
+ raise
+
+ @classmethod
+ # Neutron can be slow to clean up ports from the subnets/networks.
+ # Retry this delete a few times if we get a "Conflict" error to give
+ # neutron time to fully cleanup the ports.
+ @tenacity.retry(
+ retry=tenacity.retry_if_exception_type(exceptions.Conflict),
+ wait=tenacity.wait_incrementing(
+ RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
+ stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
+ def _logging_delete_subnet(cls, subnet_id):
+ try:
+ cls.lb_mem_subnet_client.delete_subnet(subnet_id)
+ except Exception:
+ LOG.error('Unable to delete subnet {}. Active ports:'.format(
+ subnet_id))
+ LOG.error(cls.lb_mem_ports_client.list_ports())
+ raise
+
+ @classmethod
def _create_networks(cls):
"""Creates networks, subnets, and routers used in tests.
@@ -230,7 +272,7 @@
LOG.info('lb_member_vip_net: {}'.format(cls.lb_member_vip_net))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_net_client.delete_network,
+ cls._logging_delete_network,
cls.lb_mem_net_client.show_network,
cls.lb_member_vip_net['id'])
@@ -245,7 +287,7 @@
LOG.info('lb_member_vip_subnet: {}'.format(cls.lb_member_vip_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_subnet_client.delete_subnet,
+ cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_vip_subnet['id'])
@@ -270,7 +312,7 @@
cls.lb_member_vip_ipv6_subnet = result['subnet']
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_subnet_client.delete_subnet,
+ cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_vip_ipv6_subnet['id'])
LOG.info('lb_member_vip_ipv6_subnet: {}'.format(
@@ -289,7 +331,7 @@
LOG.info('lb_member_1_net: {}'.format(cls.lb_member_1_net))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_net_client.delete_network,
+ cls._logging_delete_network,
cls.lb_mem_net_client.show_network,
cls.lb_member_1_net['id'])
@@ -304,7 +346,7 @@
LOG.info('lb_member_1_subnet: {}'.format(cls.lb_member_1_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_subnet_client.delete_subnet,
+ cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_1_subnet['id'])
@@ -325,7 +367,7 @@
cls.lb_member_1_ipv6_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_subnet_client.delete_subnet,
+ cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_1_ipv6_subnet['id'])
@@ -342,7 +384,7 @@
LOG.info('lb_member_2_net: {}'.format(cls.lb_member_2_net))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_net_client.delete_network,
+ cls._logging_delete_network,
cls.lb_mem_net_client.show_network,
cls.lb_member_2_net['id'])
@@ -357,7 +399,7 @@
LOG.info('lb_member_2_subnet: {}'.format(cls.lb_member_2_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_subnet_client.delete_subnet,
+ cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_2_subnet['id'])
@@ -378,7 +420,7 @@
cls.lb_member_2_ipv6_subnet))
cls.addClassResourceCleanup(
waiters.wait_for_not_found,
- cls.lb_mem_subnet_client.delete_subnet,
+ cls._logging_delete_subnet,
cls.lb_mem_subnet_client.show_subnet,
cls.lb_member_2_ipv6_subnet['id'])