Make unit tests not depend on random hash seed
This commit fixes the places in the tempest unit tests which were
dependent on the python hashseed being 0. They were mostly issues of
expectations, either explicit or inadvertently implicit, on dict or
list ordering.
Change-Id: Iab72ed4d52765de744804e6202ee4bf0664988c5
diff --git a/tempest/common/custom_matchers.py b/tempest/common/custom_matchers.py
index 7348a7d..93391b6 100644
--- a/tempest/common/custom_matchers.py
+++ b/tempest/common/custom_matchers.py
@@ -13,7 +13,6 @@
# under the License.
import re
-from unittest import util
from testtools import helpers
@@ -204,17 +203,28 @@
self.intersect = set(self.expected) & set(self.actual)
self.symmetric_diff = set(self.expected) ^ set(self.actual)
+ def _format_dict(self, dict_to_format):
+ # Ensure the error string dict is printed in a set order
+ # NOTE(mtreinish): needed to ensure a deterministic error msg for
+ # testing. Otherwise the error message will be dependent on the
+ # dict ordering.
+ dict_string = "{"
+ for key in sorted(dict_to_format):
+ dict_string += "'%s': %s, " % (key, dict_to_format[key])
+ dict_string = dict_string[:-2] + '}'
+ return dict_string
+
def describe(self):
msg = ""
if self.symmetric_diff:
only_expected = helpers.dict_subtract(self.expected, self.actual)
only_actual = helpers.dict_subtract(self.actual, self.expected)
if only_expected:
- msg += "Only in expected:\n %s\n" % \
- util.safe_repr(only_expected)
+ msg += "Only in expected:\n %s\n" % self._format_dict(
+ only_expected)
if only_actual:
- msg += "Only in actual:\n %s\n" % \
- util.safe_repr(only_actual)
+ msg += "Only in actual:\n %s\n" % self._format_dict(
+ only_actual)
diff_set = set(o for o in self.intersect if
self.expected[o] != self.actual[o])
if diff_set:
diff --git a/tempest/tests/cmd/test_verify_tempest_config.py b/tempest/tests/cmd/test_verify_tempest_config.py
index d0140dd..a28684e 100644
--- a/tempest/tests/cmd/test_verify_tempest_config.py
+++ b/tempest/tests/cmd/test_verify_tempest_config.py
@@ -238,8 +238,8 @@
'neutron', {})
self.assertIn('neutron', results)
self.assertIn('extensions', results['neutron'])
- self.assertEqual(['fake1', 'fake2', 'not_fake'],
- results['neutron']['extensions'])
+ self.assertEqual(sorted(['fake1', 'fake2', 'not_fake']),
+ sorted(results['neutron']['extensions']))
def test_verify_extensions_cinder(self):
def fake_list_extensions():
@@ -277,8 +277,8 @@
'cinder', {})
self.assertIn('cinder', results)
self.assertIn('extensions', results['cinder'])
- self.assertEqual(['fake1', 'fake2', 'not_fake'],
- results['cinder']['extensions'])
+ self.assertEqual(sorted(['fake1', 'fake2', 'not_fake']),
+ sorted(results['cinder']['extensions']))
def test_verify_extensions_nova(self):
def fake_list_extensions():
@@ -316,8 +316,8 @@
'nova', {})
self.assertIn('nova', results)
self.assertIn('extensions', results['nova'])
- self.assertEqual(['fake1', 'fake2', 'not_fake'],
- results['nova']['extensions'])
+ self.assertEqual(sorted(['fake1', 'fake2', 'not_fake']),
+ sorted(results['nova']['extensions']))
def test_verify_extensions_nova_v3(self):
def fake_list_extensions():
@@ -355,8 +355,8 @@
'nova_v3', {})
self.assertIn('nova_v3', results)
self.assertIn('extensions', results['nova_v3'])
- self.assertEqual(['fake1', 'fake2', 'not_fake'],
- results['nova_v3']['extensions'])
+ self.assertEqual(sorted(['fake1', 'fake2', 'not_fake']),
+ sorted(results['nova_v3']['extensions']))
def test_verify_extensions_swift(self):
def fake_list_extensions():
@@ -395,5 +395,5 @@
'swift', {})
self.assertIn('swift', results)
self.assertIn('extensions', results['swift'])
- self.assertEqual(['not_fake', 'fake1', 'fake2'],
- results['swift']['extensions'])
+ self.assertEqual(sorted(['not_fake', 'fake1', 'fake2']),
+ sorted(results['swift']['extensions']))
diff --git a/tempest/tests/test_credentials.py b/tempest/tests/test_credentials.py
index 9da5f92..ea576c4 100644
--- a/tempest/tests/test_credentials.py
+++ b/tempest/tests/test_credentials.py
@@ -128,12 +128,22 @@
creds = self._get_credentials()
self.assertTrue(creds.is_valid())
- def test_is_not_valid(self):
+ def _test_is_not_valid(self, ignore_key):
creds = self._get_credentials()
for attr in self.attributes.keys():
+ if attr == ignore_key:
+ continue
+ temp_attr = getattr(creds, attr)
delattr(creds, attr)
self.assertFalse(creds.is_valid(),
"Credentials should be invalid without %s" % attr)
+ setattr(creds, attr, temp_attr)
+
+ def test_is_not_valid(self):
+ # NOTE(mtreinish): A KeystoneV2 credential object is valid without
+ # a tenant_name. So skip that check. See tempest.auth for the valid
+ # credential requirements
+ self._test_is_not_valid('tenant_name')
def test_default(self):
self.useFixture(fixtures.LockFixture('auth_version'))
@@ -205,6 +215,12 @@
config_value = 'fake_' + attr
self.assertEqual(getattr(creds, attr), config_value)
+ def test_is_not_valid(self):
+ # NOTE(mtreinish) For a Keystone V3 credential object a project name
+ # is not required to be valid, so we skip that check. See tempest.auth
+ # for the valid credential requirements
+ self._test_is_not_valid('project_name')
+
def test_synced_attributes(self):
attributes = self.attributes
# Create V3 credentials with tenant instead of project, and user_domain