Add wait_for_resource_deletion for swift api clients

Currently today we dont have any way in swift to verify that resources were deleted
before moving to the next command
In current code there was hardcoded sleep for 2 seconds instead of checking if
resource really deleted.

Added to the current cleanup :
Implement is_resource_deleted for object_client and container_client
After remove action we wait/ verify till resource really deleted
Remove hardcoded sleep for 2 seconds
Remove ignore for not found in reomval , if we hit on it means something
wrong in our code.

Change-Id: I32f37f8e874a3510bb1af6db45a1b9a8d2fed543
diff --git a/tempest/api/object_storage/base.py b/tempest/api/object_storage/base.py
index e8f3f8b..478a834 100644
--- a/tempest/api/object_storage/base.py
+++ b/tempest/api/object_storage/base.py
@@ -13,12 +13,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import time
-
 from tempest.common import custom_matchers
 from tempest import config
 from tempest.lib.common.utils import data_utils
-from tempest.lib.common.utils import test_utils
 from tempest.lib import exceptions as lib_exc
 import tempest.test
 
@@ -50,12 +47,11 @@
             _, objlist = container_client.list_container_objects(cont, params)
             # delete every object in the container
             for obj in objlist:
-                test_utils.call_and_ignore_notfound_exc(
-                    object_client.delete_object, cont, obj['name'])
-            # sleep 2 seconds to sync the deletion of the objects
-            # in HA deployment
-            time.sleep(2)
+                object_client.delete_object(cont, obj['name'])
+                object_client.wait_for_resource_deletion(obj['name'], cont)
+            # Verify resource deletion
             container_client.delete_container(cont)
+            container_client.wait_for_resource_deletion(cont)
         except lib_exc.NotFound:
             pass
 
diff --git a/tempest/lib/common/rest_client.py b/tempest/lib/common/rest_client.py
index a987e03..0fa78fa 100644
--- a/tempest/lib/common/rest_client.py
+++ b/tempest/lib/common/rest_client.py
@@ -890,7 +890,7 @@
             return True
         return 'exceed' in resp_body.get('message', 'blabla')
 
-    def wait_for_resource_deletion(self, id):
+    def wait_for_resource_deletion(self, id, *args, **kwargs):
         """Waits for a resource to be deleted
 
         This method will loop over is_resource_deleted until either
@@ -903,7 +903,7 @@
         """
         start_time = int(time.time())
         while True:
-            if self.is_resource_deleted(id):
+            if self.is_resource_deleted(id, *args, **kwargs):
                 return
             if int(time.time()) - start_time >= self.build_timeout:
                 message = ('Failed to delete %(resource_type)s %(id)s within '
diff --git a/tempest/lib/services/object_storage/container_client.py b/tempest/lib/services/object_storage/container_client.py
index 027fb1f..da92adc 100644
--- a/tempest/lib/services/object_storage/container_client.py
+++ b/tempest/lib/services/object_storage/container_client.py
@@ -20,10 +20,18 @@
 from six.moves.urllib import parse as urllib
 
 from tempest.lib.common import rest_client
+from tempest.lib import exceptions
 
 
 class ContainerClient(rest_client.RestClient):
 
+    def is_resource_deleted(self, container):
+        try:
+            self.list_container_metadata(container)
+        except exceptions.NotFound:
+            return True
+        return False
+
     def update_container(self, container_name, **headers):
         """Creates or Updates a container
 
diff --git a/tempest/lib/services/object_storage/object_client.py b/tempest/lib/services/object_storage/object_client.py
index 1d38153..dd7d517 100644
--- a/tempest/lib/services/object_storage/object_client.py
+++ b/tempest/lib/services/object_storage/object_client.py
@@ -23,6 +23,13 @@
 
 class ObjectClient(rest_client.RestClient):
 
+    def is_resource_deleted(self, object_name, container):
+        try:
+            self.get_object(container, object_name)
+        except exceptions.NotFound:
+            return True
+        return False
+
     def create_object(self, container, object_name, data,
                       params=None, metadata=None, headers=None,
                       chunked=False):