Add retries for base client
Octavia API doesn't allow to change fields of LB when it is in
some internal states. In this case 409 Conflict exception is raised.
The patch adds retry logic to update/delete/create methods in base
clients to retry on 409 responce.
Related-Prod: PRODX-14186
Change-Id: Icb22b12ff30dc8f6a48506c00eb00705fa1de470
(cherry picked from commit 7848930006586c35d1df4fe381de2023881e0455)
(cherry picked from commit d57ca631531a04f17156feb1136bdae46b4f1f40)
diff --git a/octavia_tempest_plugin/common/constants.py b/octavia_tempest_plugin/common/constants.py
index 8ef8d94..d73be5a 100644
--- a/octavia_tempest_plugin/common/constants.py
+++ b/octavia_tempest_plugin/common/constants.py
@@ -243,6 +243,12 @@
COMPUTE_ZONE = 'compute_zone'
MANAGEMENT_NETWORK = 'management_network'
+# Retry constants
+RETRY_ATTEMPTS = 15
+RETRY_INITIAL_DELAY = 1
+RETRY_BACKOFF = 1
+RETRY_MAX = 5
+
# API valid fields
SHOW_LOAD_BALANCER_RESPONSE_FIELDS = (
ADMIN_STATE_UP, CREATED_AT, DESCRIPTION, FLAVOR_ID, ID, LISTENERS, NAME,
diff --git a/octavia_tempest_plugin/services/load_balancer/v2/base_client.py b/octavia_tempest_plugin/services/load_balancer/v2/base_client.py
index 96ea33c..bf08c5a 100644
--- a/octavia_tempest_plugin/services/load_balancer/v2/base_client.py
+++ b/octavia_tempest_plugin/services/load_balancer/v2/base_client.py
@@ -19,6 +19,7 @@
from tempest.lib.common import rest_client
from tempest.lib.common.utils import test_utils
from tempest.lib import exceptions
+import tenacity
from octavia_tempest_plugin.common import constants as const
from octavia_tempest_plugin.tests import waiters
@@ -55,6 +56,11 @@
if not hasattr(self, cleanup_func_name):
setattr(self, cleanup_func_name, self._cleanup_obj)
+ @tenacity.retry(
+ retry=tenacity.retry_if_exception_type(exceptions.Conflict),
+ wait=tenacity.wait_incrementing(
+ const.RETRY_INITIAL_DELAY, const.RETRY_BACKOFF, const.RETRY_MAX),
+ stop=tenacity.stop_after_attempt(const.RETRY_ATTEMPTS))
def _create_object(self, parent_id=None, return_object_only=True,
**kwargs):
"""Create an object.
@@ -211,6 +217,11 @@
else:
return jsonutils.loads(body.decode('utf-8'))
+ @tenacity.retry(
+ retry=tenacity.retry_if_exception_type(exceptions.Conflict),
+ wait=tenacity.wait_incrementing(
+ const.RETRY_INITIAL_DELAY, const.RETRY_BACKOFF, const.RETRY_MAX),
+ stop=tenacity.stop_after_attempt(const.RETRY_ATTEMPTS))
def _update_object(self, obj_id, parent_id=None, return_object_only=True,
**kwargs):
"""Update an object.
@@ -264,6 +275,11 @@
else:
return jsonutils.loads(body.decode('utf-8'))
+ @tenacity.retry(
+ retry=tenacity.retry_if_exception_type(exceptions.Conflict),
+ wait=tenacity.wait_incrementing(
+ const.RETRY_INITIAL_DELAY, const.RETRY_BACKOFF, const.RETRY_MAX),
+ stop=tenacity.stop_after_attempt(const.RETRY_ATTEMPTS))
def _delete_obj(self, obj_id, parent_id=None, ignore_errors=False,
cascade=False):
"""Delete an object.
diff --git a/octavia_tempest_plugin/tests/test_base.py b/octavia_tempest_plugin/tests/test_base.py
index 716f359..4955cb3 100644
--- a/octavia_tempest_plugin/tests/test_base.py
+++ b/octavia_tempest_plugin/tests/test_base.py
@@ -41,11 +41,6 @@
CONF = config.CONF
LOG = logging.getLogger(__name__)
-RETRY_ATTEMPTS = 15
-RETRY_INITIAL_DELAY = 1
-RETRY_BACKOFF = 1
-RETRY_MAX = 5
-
class LoadBalancerBaseTest(validators.ValidatorsMixin,
RBAC_tests.RBACTestsMixin, test.BaseTestCase):
@@ -325,8 +320,8 @@
@tenacity.retry(
retry=tenacity.retry_if_exception_type(exceptions.Conflict),
wait=tenacity.wait_incrementing(
- RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
- stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
+ const.RETRY_INITIAL_DELAY, const.RETRY_BACKOFF, const.RETRY_MAX),
+ stop=tenacity.stop_after_attempt(const.RETRY_ATTEMPTS))
def _logging_delete_network(cls, net_id):
try:
cls.lb_mem_net_client.delete_network(net_id)
@@ -343,8 +338,8 @@
@tenacity.retry(
retry=tenacity.retry_if_exception_type(exceptions.Conflict),
wait=tenacity.wait_incrementing(
- RETRY_INITIAL_DELAY, RETRY_BACKOFF, RETRY_MAX),
- stop=tenacity.stop_after_attempt(RETRY_ATTEMPTS))
+ const.RETRY_INITIAL_DELAY, const.RETRY_BACKOFF, const.RETRY_MAX),
+ stop=tenacity.stop_after_attempt(const.RETRY_ATTEMPTS))
def _logging_delete_subnet(cls, subnet_id):
try:
cls.lb_mem_subnet_client.delete_subnet(subnet_id)