object_storage: retry object creation on Conflict
In some cases (such as when using Ceph RADOSGW with a load-balancer)
the container (bucket) creation will be received by different backend
to the object creation triggering a subtle edge case.
The rgw handling the object creation request has (cached) knowledge
the bucket doesn't exist (it didn't a few seconds ago) and will (in
rgw) create it, at which point triggering a "409 BucketAlreadyExists"
code.
Detecting this and retrying (with a small delay) should reduce
instances of this failing.
Change-Id: Id88fb93dd1a48d046917e9168d623e53497ec83e
diff --git a/tempest/api/object_storage/base.py b/tempest/api/object_storage/base.py
index 478a834..77e26ef 100644
--- a/tempest/api/object_storage/base.py
+++ b/tempest/api/object_storage/base.py
@@ -13,6 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
+import time
+
from tempest.common import custom_matchers
from tempest import config
from tempest.lib.common.utils import data_utils
@@ -119,12 +121,20 @@
object_name = data_utils.rand_name(name='TestObject')
if data is None:
data = data_utils.random_bytes()
- cls.object_client.create_object(container_name,
- object_name,
- data,
- metadata=metadata)
- return object_name, data
+ err = Exception()
+ for _ in range(5):
+ try:
+ cls.object_client.create_object(container_name,
+ object_name,
+ data,
+ metadata=metadata)
+ return object_name, data
+ # after bucket creation we might see Conflict
+ except lib_exc.Conflict as e:
+ err = e
+ time.sleep(2)
+ raise err
@classmethod
def delete_containers(cls, container_client=None, object_client=None):