Retry subnet/network deletes on 409 Conflict

Neutron can be slow to cleanup ports from subnets/networks.
This patch adds retries when deleting subnets and networks in the
tempest teardown/clean up phase after tests.

Also, there were cases where addClassResourceCleanup was being used
inside test cases instead of addCleanup. This patch corrects those to
use addCleanup.

Story: 2004826
Task: 29000

Change-Id: Ia29541d1c89f3559a3ce22b1a27c6bcf079ce2cc
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py b/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py
index 7b769e1..b2b1d18 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_healthmonitor.py
@@ -103,7 +103,7 @@
         }
 
         hm = self.mem_healthmonitor_client.create_healthmonitor(**hm_kwargs)
-        self.addClassResourceCleanup(
+        self.addCleanup(
             self.mem_healthmonitor_client.cleanup_healthmonitor,
             hm[const.ID], lb_client=self.mem_lb_client, lb_id=self.lb_id)
 
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py b/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py
index 98d3bc6..9e09f35 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_l7policy.py
@@ -115,7 +115,7 @@
         }
 
         l7policy = self.mem_l7policy_client.create_l7policy(**l7policy_kwargs)
-        self.addClassResourceCleanup(
+        self.addCleanup(
             self.mem_l7policy_client.cleanup_l7policy,
             l7policy[const.ID],
             lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py b/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py
index 3e14a74..114ea3e 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_l7rule.py
@@ -112,7 +112,7 @@
         }
 
         l7rule = self.mem_l7rule_client.create_l7rule(**l7rule_kwargs)
-        self.addClassResourceCleanup(
+        self.addCleanup(
             self.mem_l7rule_client.cleanup_l7rule,
             l7rule[const.ID], l7policy_id=self.l7policy_id,
             lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_listener.py b/octavia_tempest_plugin/tests/scenario/v2/test_listener.py
index 685c200..7720d27 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_listener.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_listener.py
@@ -132,7 +132,7 @@
             })
 
         listener = self.mem_listener_client.create_listener(**listener_kwargs)
-        self.addClassResourceCleanup(
+        self.addCleanup(
             self.mem_listener_client.cleanup_listener,
             listener[const.ID],
             lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py b/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py
index cfc3ee5..79f1876 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_load_balancer.py
@@ -59,7 +59,7 @@
         self._setup_lb_network_kwargs(lb_kwargs, ip_version)
 
         lb = self.mem_lb_client.create_loadbalancer(**lb_kwargs)
-        self.addClassResourceCleanup(
+        self.addCleanup(
             self.mem_lb_client.cleanup_loadbalancer,
             lb[const.ID])
 
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_member.py b/octavia_tempest_plugin/tests/scenario/v2/test_member.py
index b30d651..6bd18de 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_member.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_member.py
@@ -123,7 +123,7 @@
                 const.ID]
 
         member = self.mem_member_client.create_member(**member_kwargs)
-        self.addClassResourceCleanup(
+        self.addCleanup(
             self.mem_member_client.cleanup_member,
             member[const.ID], pool_id=self.pool_id,
             lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/scenario/v2/test_pool.py b/octavia_tempest_plugin/tests/scenario/v2/test_pool.py
index 1ddc8b2..cd26687 100644
--- a/octavia_tempest_plugin/tests/scenario/v2/test_pool.py
+++ b/octavia_tempest_plugin/tests/scenario/v2/test_pool.py
@@ -108,7 +108,7 @@
             pool_kwargs[const.LOADBALANCER_ID] = self.lb_id
 
         pool = self.mem_pool_client.create_pool(**pool_kwargs)
-        self.addClassResourceCleanup(
+        self.addCleanup(
             self.mem_pool_client.cleanup_pool,
             pool[const.ID],
             lb_client=self.mem_lb_client, lb_id=self.lb_id)
diff --git a/octavia_tempest_plugin/tests/test_base.py b/octavia_tempest_plugin/tests/test_base.py
index a48ba0b..b935409 100644
--- a/octavia_tempest_plugin/tests/test_base.py
+++ b/octavia_tempest_plugin/tests/test_base.py
@@ -30,6 +30,7 @@
 from tempest.lib.common.utils.linux import remote_client
 from tempest.lib import exceptions
 from tempest import test
+import tenacity
 
 from octavia_tempest_plugin import clients
 from octavia_tempest_plugin.common import constants as const
@@ -39,6 +40,11 @@
 CONF = config.CONF
 LOG = logging.getLogger(__name__)
 
+RETRY_ATTEMPTS = 15
+RETRY_INITIAL_DELAY = 1
+RETRY_BACKOFF = 1
+RETRY_MAX = 5
+
 
 class LoadBalancerBaseTest(test.BaseTestCase):
     """Base class for load balancer tests."""
@@ -204,6 +210,42 @@
                     cls.lb_member_2_ipv6_subnet[const.ID]))
 
     @classmethod
+    # Neutron can be slow to clean up ports from the subnets/networks.
+    # Retry this delete a few times if we get a "Conflict" error to give
+    # neutron time to fully cleanup the ports.
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(exceptions.Conflict),
+        wait=tenacity.wait_incrementing(
+            RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
+        stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
+    def _logging_delete_network(cls, net_id):
+        try:
+            cls.lb_mem_net_client.delete_network(net_id)
+        except Exception:
+            LOG.error('Unable to delete network {}. Active ports:'.format(
+                net_id))
+            LOG.error(cls.lb_mem_ports_client.list_ports())
+            raise
+
+    @classmethod
+    # Neutron can be slow to clean up ports from the subnets/networks.
+    # Retry this delete a few times if we get a "Conflict" error to give
+    # neutron time to fully cleanup the ports.
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(exceptions.Conflict),
+        wait=tenacity.wait_incrementing(
+            RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
+        stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
+    def _logging_delete_subnet(cls, subnet_id):
+        try:
+            cls.lb_mem_subnet_client.delete_subnet(subnet_id)
+        except Exception:
+            LOG.error('Unable to delete subnet {}. Active ports:'.format(
+                subnet_id))
+            LOG.error(cls.lb_mem_ports_client.list_ports())
+            raise
+
+    @classmethod
     def _create_networks(cls):
         """Creates networks, subnets, and routers used in tests.
 
@@ -230,7 +272,7 @@
         LOG.info('lb_member_vip_net: {}'.format(cls.lb_member_vip_net))
         cls.addClassResourceCleanup(
             waiters.wait_for_not_found,
-            cls.lb_mem_net_client.delete_network,
+            cls._logging_delete_network,
             cls.lb_mem_net_client.show_network,
             cls.lb_member_vip_net['id'])
 
@@ -245,7 +287,7 @@
         LOG.info('lb_member_vip_subnet: {}'.format(cls.lb_member_vip_subnet))
         cls.addClassResourceCleanup(
             waiters.wait_for_not_found,
-            cls.lb_mem_subnet_client.delete_subnet,
+            cls._logging_delete_subnet,
             cls.lb_mem_subnet_client.show_subnet,
             cls.lb_member_vip_subnet['id'])
 
@@ -270,7 +312,7 @@
                 cls.lb_member_vip_ipv6_subnet = result['subnet']
                 cls.addClassResourceCleanup(
                     waiters.wait_for_not_found,
-                    cls.lb_mem_subnet_client.delete_subnet,
+                    cls._logging_delete_subnet,
                     cls.lb_mem_subnet_client.show_subnet,
                     cls.lb_member_vip_ipv6_subnet['id'])
             LOG.info('lb_member_vip_ipv6_subnet: {}'.format(
@@ -289,7 +331,7 @@
         LOG.info('lb_member_1_net: {}'.format(cls.lb_member_1_net))
         cls.addClassResourceCleanup(
             waiters.wait_for_not_found,
-            cls.lb_mem_net_client.delete_network,
+            cls._logging_delete_network,
             cls.lb_mem_net_client.show_network,
             cls.lb_member_1_net['id'])
 
@@ -304,7 +346,7 @@
         LOG.info('lb_member_1_subnet: {}'.format(cls.lb_member_1_subnet))
         cls.addClassResourceCleanup(
             waiters.wait_for_not_found,
-            cls.lb_mem_subnet_client.delete_subnet,
+            cls._logging_delete_subnet,
             cls.lb_mem_subnet_client.show_subnet,
             cls.lb_member_1_subnet['id'])
 
@@ -325,7 +367,7 @@
                 cls.lb_member_1_ipv6_subnet))
             cls.addClassResourceCleanup(
                 waiters.wait_for_not_found,
-                cls.lb_mem_subnet_client.delete_subnet,
+                cls._logging_delete_subnet,
                 cls.lb_mem_subnet_client.show_subnet,
                 cls.lb_member_1_ipv6_subnet['id'])
 
@@ -342,7 +384,7 @@
         LOG.info('lb_member_2_net: {}'.format(cls.lb_member_2_net))
         cls.addClassResourceCleanup(
             waiters.wait_for_not_found,
-            cls.lb_mem_net_client.delete_network,
+            cls._logging_delete_network,
             cls.lb_mem_net_client.show_network,
             cls.lb_member_2_net['id'])
 
@@ -357,7 +399,7 @@
         LOG.info('lb_member_2_subnet: {}'.format(cls.lb_member_2_subnet))
         cls.addClassResourceCleanup(
             waiters.wait_for_not_found,
-            cls.lb_mem_subnet_client.delete_subnet,
+            cls._logging_delete_subnet,
             cls.lb_mem_subnet_client.show_subnet,
             cls.lb_member_2_subnet['id'])
 
@@ -378,7 +420,7 @@
                 cls.lb_member_2_ipv6_subnet))
             cls.addClassResourceCleanup(
                 waiters.wait_for_not_found,
-                cls.lb_mem_subnet_client.delete_subnet,
+                cls._logging_delete_subnet,
                 cls.lb_mem_subnet_client.show_subnet,
                 cls.lb_member_2_ipv6_subnet['id'])