fix race condition between addCleanup and lockutils
it turns out that addCleanup is actually running much later than
anticipated, so by that time we're actually outside of the
synchronized region that lockutils has created. This means that we
have a potential race in tearing down our resources. This potential
race has become a 100% race if you turn on additional workers in
devstack, which jog0 exposed today.
Convert the lockutils usage to a fixture based approach which will
keep the lock around through the cleanup phase.
Fixes bug #1221469
Co-author: Joe Gordon <joe.gordon0@gmail.com>
Change-Id: I2cb73cb64cc04077cf1595a880b124f31aa13f89
diff --git a/tempest/api/compute/admin/test_aggregates.py b/tempest/api/compute/admin/test_aggregates.py
index 303bc0c..0bb0460 100644
--- a/tempest/api/compute/admin/test_aggregates.py
+++ b/tempest/api/compute/admin/test_aggregates.py
@@ -15,6 +15,8 @@
# License for the specific language governing permissions and limitations
# under the License.
+import fixtures
+
from tempest.api.compute import base
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
@@ -22,6 +24,16 @@
from tempest.test import attr
+class LockFixture(fixtures.Fixture):
+ def __init__(self, name):
+ self.mgr = lockutils.lock(name, 'tempest-', True)
+
+ def setUp(self):
+ super(LockFixture, self).setUp()
+ self.addCleanup(self.mgr.__exit__, None, None, None)
+ self.mgr.__enter__()
+
+
class AggregatesAdminTestJSON(base.BaseComputeAdminTest):
"""
@@ -146,9 +158,9 @@
self.client.get_aggregate, -1)
@attr(type='gate')
- @lockutils.synchronized('availability_zone', 'tempest-', True)
def test_aggregate_add_remove_host(self):
# Add an host to the given aggregate and remove.
+ self.useFixture(LockFixture('availability_zone'))
aggregate_name = rand_name(self.aggregate_name_prefix)
resp, aggregate = self.client.create_aggregate(aggregate_name)
self.addCleanup(self.client.delete_aggregate, aggregate['id'])
@@ -168,9 +180,9 @@
self.assertNotIn(self.host, body['hosts'])
@attr(type='gate')
- @lockutils.synchronized('availability_zone', 'tempest-', True)
def test_aggregate_add_host_list(self):
# Add an host to the given aggregate and list.
+ self.useFixture(LockFixture('availability_zone'))
aggregate_name = rand_name(self.aggregate_name_prefix)
resp, aggregate = self.client.create_aggregate(aggregate_name)
self.addCleanup(self.client.delete_aggregate, aggregate['id'])
@@ -186,9 +198,9 @@
self.assertIn(self.host, agg['hosts'])
@attr(type='gate')
- @lockutils.synchronized('availability_zone', 'tempest-', True)
def test_aggregate_add_host_get_details(self):
# Add an host to the given aggregate and get details.
+ self.useFixture(LockFixture('availability_zone'))
aggregate_name = rand_name(self.aggregate_name_prefix)
resp, aggregate = self.client.create_aggregate(aggregate_name)
self.addCleanup(self.client.delete_aggregate, aggregate['id'])
@@ -201,9 +213,9 @@
self.assertIn(self.host, body['hosts'])
@attr(type='gate')
- @lockutils.synchronized('availability_zone', 'tempest-', True)
def test_aggregate_add_host_create_server_with_az(self):
# Add an host to the given aggregate and create a server.
+ self.useFixture(LockFixture('availability_zone'))
aggregate_name = rand_name(self.aggregate_name_prefix)
az_name = rand_name(self.az_name_prefix)
resp, aggregate = self.client.create_aggregate(aggregate_name, az_name)
@@ -248,9 +260,9 @@
aggregate['id'], self.host)
@attr(type=['negative', 'gate'])
- @lockutils.synchronized('availability_zone', 'tempest-', True)
def test_aggregate_remove_host_as_user(self):
# Regular user is not allowed to remove a host from an aggregate.
+ self.useFixture(LockFixture('availability_zone'))
aggregate_name = rand_name(self.aggregate_name_prefix)
resp, aggregate = self.client.create_aggregate(aggregate_name)
self.addCleanup(self.client.delete_aggregate, aggregate['id'])