commit 8d409f6a5a7ac3bdeee7466c87d6c914423ecfb7
author Vasyl Saienko <vsaienko@mirantis.com> Tue Oct 18 08:58:56 2022 +0000
committer Vasyl Saienko <vsaienko@mirantis.com> Tue Oct 18 08:58:59 2022 +0000
tree fd8b18945188cce3da9c2ded12349dfd72907876
parent c4def745c83cd79bb5b3c4ae7c475f5b18eb5468

Retry router removal on conflict

Manila injects share network to tenant router underhood,
the interface removal is asynchronous in manila. This is
only one example when conflict migh happen. Since rest assumes
we can retry on Conflicts, do it here.

Related-Prod: PRODX-27590
Change-Id: I3c284673abf13e2c01ebca047c353de1207caf12

requirements.txt

diff --git a/requirements.txt b/requirements.txt
index c4c7fcc..6908527 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -21,3 +21,4 @@
 PrettyTable>=0.7.1 # BSD
 urllib3>=1.21.1 # MIT
 debtcollector>=1.2.0 # Apache-2.0
+tenacity>=4.4.0 # Apache-2.0

tempest/lib/common/constants.py

diff --git a/tempest/lib/common/constants.py b/tempest/lib/common/constants.py
new file mode 100644
index 0000000..57fdd93
--- /dev/null
+++ b/tempest/lib/common/constants.py
@@ -0,0 +1,5 @@
+# Retry constants
+RETRY_ATTEMPTS = 30
+RETRY_INITIAL_DELAY = 1
+RETRY_BACKOFF = 3
+RETRY_MAX = 10

tempest/lib/common/dynamic_creds.py

diff --git a/tempest/lib/common/dynamic_creds.py b/tempest/lib/common/dynamic_creds.py
index 19b76a3..cbac5a6 100644
--- a/tempest/lib/common/dynamic_creds.py
+++ b/tempest/lib/common/dynamic_creds.py
@@ -16,7 +16,9 @@
 
 import netaddr
 from oslo_log import log as logging
+import tenacity
 
+import tempest.lib.common.constants as const
 from tempest.lib.common import cred_client
 from tempest.lib.common import cred_provider
 from tempest.lib.common.utils import data_utils
@@ -498,6 +500,11 @@
             del self._creds[creds_name]
         return self.get_credentials(roles, scope=scope)
 
+    @tenacity.retry(
+        retry=tenacity.retry_if_exception_type(lib_exc.Conflict),
+        wait=tenacity.wait_incrementing(
+            const.RETRY_INITIAL_DELAY, const.RETRY_BACKOFF, const.RETRY_MAX),
+        stop=tenacity.stop_after_attempt(const.RETRY_ATTEMPTS))
     def _clear_isolated_router(self, router_id, router_name):
         client = self.routers_admin_client
         try:

tempest/lib/common/waiters.py

diff --git a/tempest/lib/common/waiters.py b/tempest/lib/common/waiters.py
index 1f0bdc9..7bedd0d 100644
--- a/tempest/lib/common/waiters.py
+++ b/tempest/lib/common/waiters.py
@@ -24,6 +24,7 @@
             ports = ports_client.list_ports(
                 device_id=router_id,
                 fixed_ips=f"subnet_id={subnet_id}")['ports']
+
             if len(ports) == 0:
                 return
             time.sleep(interval)