Sync latest versions of oslo incubator
This commit pulls in the lastest version of oslo incubator modules.
Change-Id: Ib591897958d6e12c251375d30342218b7d2cdd05
diff --git a/tempest/openstack/common/excutils.py b/tempest/openstack/common/excutils.py
index 81aad14..db37660 100644
--- a/tempest/openstack/common/excutils.py
+++ b/tempest/openstack/common/excutils.py
@@ -24,6 +24,8 @@
import time
import traceback
+import six
+
from tempest.openstack.common.gettextutils import _ # noqa
@@ -65,7 +67,7 @@
self.tb))
return False
if self.reraise:
- raise self.type_, self.value, self.tb
+ six.reraise(self.type_, self.value, self.tb)
def forever_retry_uncaught_exceptions(infunc):
@@ -77,7 +79,8 @@
try:
return infunc(*args, **kwargs)
except Exception as exc:
- if exc.message == last_exc_message:
+ this_exc_message = unicode(exc)
+ if this_exc_message == last_exc_message:
exc_count += 1
else:
exc_count = 1
@@ -85,12 +88,12 @@
# the exception message changes
cur_time = int(time.time())
if (cur_time - last_log_time > 60 or
- exc.message != last_exc_message):
+ this_exc_message != last_exc_message):
logging.exception(
_('Unexpected exception occurred %d time(s)... '
'retrying.') % exc_count)
last_log_time = cur_time
- last_exc_message = exc.message
+ last_exc_message = this_exc_message
exc_count = 0
# This should be a very rare event. In case it isn't, do
# a sleep.
diff --git a/tempest/openstack/common/fileutils.py b/tempest/openstack/common/fileutils.py
index d2e3d3e..6cf68ba 100644
--- a/tempest/openstack/common/fileutils.py
+++ b/tempest/openstack/common/fileutils.py
@@ -69,33 +69,34 @@
return (reloaded, cache_info['data'])
-def delete_if_exists(path):
+def delete_if_exists(path, remove=os.unlink):
"""Delete a file, but ignore file not found error.
:param path: File to delete
+ :param remove: Optional function to remove passed path
"""
try:
- os.unlink(path)
+ remove(path)
except OSError as e:
- if e.errno == errno.ENOENT:
- return
- else:
+ if e.errno != errno.ENOENT:
raise
@contextlib.contextmanager
-def remove_path_on_error(path):
+def remove_path_on_error(path, remove=delete_if_exists):
"""Protect code that wants to operate on PATH atomically.
Any exception will cause PATH to be removed.
:param path: File to work with
+ :param remove: Optional function to remove passed path
"""
+
try:
yield
except Exception:
with excutils.save_and_reraise_exception():
- delete_if_exists(path)
+ remove(path)
def file_open(*args, **kwargs):
diff --git a/tempest/openstack/common/gettextutils.py b/tempest/openstack/common/gettextutils.py
index 8594937..cbf570a 100644
--- a/tempest/openstack/common/gettextutils.py
+++ b/tempest/openstack/common/gettextutils.py
@@ -1,8 +1,8 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2012 Red Hat, Inc.
-# All Rights Reserved.
# Copyright 2013 IBM Corp.
+# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@@ -26,22 +26,44 @@
import copy
import gettext
-import logging.handlers
+import logging
import os
import re
-import UserString
+try:
+ import UserString as _userString
+except ImportError:
+ import collections as _userString
+from babel import localedata
import six
_localedir = os.environ.get('tempest'.upper() + '_LOCALEDIR')
_t = gettext.translation('tempest', localedir=_localedir, fallback=True)
+_AVAILABLE_LANGUAGES = {}
+USE_LAZY = False
+
+
+def enable_lazy():
+ """Convenience function for configuring _() to use lazy gettext
+
+ Call this at the start of execution to enable the gettextutils._
+ function to use lazy gettext functionality. This is useful if
+ your project is importing _ directly instead of using the
+ gettextutils.install() way of importing the _ function.
+ """
+ global USE_LAZY
+ USE_LAZY = True
+
def _(msg):
- return _t.ugettext(msg)
+ if USE_LAZY:
+ return Message(msg, 'tempest')
+ else:
+ return _t.ugettext(msg)
-def install(domain):
+def install(domain, lazy=False):
"""Install a _() function using the given translation domain.
Given a translation domain, install a _() function using gettext's
@@ -51,44 +73,48 @@
overriding the default localedir (e.g. /usr/share/locale) using
a translation-domain-specific environment variable (e.g.
NOVA_LOCALEDIR).
+
+ :param domain: the translation domain
+ :param lazy: indicates whether or not to install the lazy _() function.
+ The lazy _() introduces a way to do deferred translation
+ of messages by installing a _ that builds Message objects,
+ instead of strings, which can then be lazily translated into
+ any available locale.
"""
- gettext.install(domain,
- localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
- unicode=True)
+ if lazy:
+ # NOTE(mrodden): Lazy gettext functionality.
+ #
+ # The following introduces a deferred way to do translations on
+ # messages in OpenStack. We override the standard _() function
+ # and % (format string) operation to build Message objects that can
+ # later be translated when we have more information.
+ #
+ # Also included below is an example LocaleHandler that translates
+ # Messages to an associated locale, effectively allowing many logs,
+ # each with their own locale.
+
+ def _lazy_gettext(msg):
+ """Create and return a Message object.
+
+ Lazy gettext function for a given domain, it is a factory method
+ for a project/module to get a lazy gettext function for its own
+ translation domain (i.e. nova, glance, cinder, etc.)
+
+ Message encapsulates a string so that we can translate
+ it later when needed.
+ """
+ return Message(msg, domain)
+
+ import __builtin__
+ __builtin__.__dict__['_'] = _lazy_gettext
+ else:
+ localedir = '%s_LOCALEDIR' % domain.upper()
+ gettext.install(domain,
+ localedir=os.environ.get(localedir),
+ unicode=True)
-"""
-Lazy gettext functionality.
-
-The following is an attempt to introduce a deferred way
-to do translations on messages in OpenStack. We attempt to
-override the standard _() function and % (format string) operation
-to build Message objects that can later be translated when we have
-more information. Also included is an example LogHandler that
-translates Messages to an associated locale, effectively allowing
-many logs, each with their own locale.
-"""
-
-
-def get_lazy_gettext(domain):
- """Assemble and return a lazy gettext function for a given domain.
-
- Factory method for a project/module to get a lazy gettext function
- for its own translation domain (i.e. nova, glance, cinder, etc.)
- """
-
- def _lazy_gettext(msg):
- """Create and return a Message object.
-
- Message encapsulates a string so that we can translate it later when
- needed.
- """
- return Message(msg, domain)
-
- return _lazy_gettext
-
-
-class Message(UserString.UserString, object):
+class Message(_userString.UserString, object):
"""Class used to encapsulate translatable messages."""
def __init__(self, msg, domain):
# _msg is the gettext msgid and should never change
@@ -130,7 +156,7 @@
# look for %(blah) fields in string;
# ignore %% and deal with the
# case where % is first character on the line
- keys = re.findall('(?:[^%]|^)%\((\w*)\)[a-z]', full_msg)
+ keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
# if we don't find any %(blah) blocks but have a %s
if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
@@ -229,7 +255,47 @@
if name in ops:
return getattr(self.data, name)
else:
- return UserString.UserString.__getattribute__(self, name)
+ return _userString.UserString.__getattribute__(self, name)
+
+
+def get_available_languages(domain):
+ """Lists the available languages for the given translation domain.
+
+ :param domain: the domain to get languages for
+ """
+ if domain in _AVAILABLE_LANGUAGES:
+ return copy.copy(_AVAILABLE_LANGUAGES[domain])
+
+ localedir = '%s_LOCALEDIR' % domain.upper()
+ find = lambda x: gettext.find(domain,
+ localedir=os.environ.get(localedir),
+ languages=[x])
+
+ # NOTE(mrodden): en_US should always be available (and first in case
+ # order matters) since our in-line message strings are en_US
+ language_list = ['en_US']
+ # NOTE(luisg): Babel <1.0 used a function called list(), which was
+ # renamed to locale_identifiers() in >=1.0, the requirements master list
+ # requires >=0.9.6, uncapped, so defensively work with both. We can remove
+ # this check when the master list updates to >=1.0, and all projects udpate
+ list_identifiers = (getattr(localedata, 'list', None) or
+ getattr(localedata, 'locale_identifiers'))
+ locale_identifiers = list_identifiers()
+ for i in locale_identifiers:
+ if find(i) is not None:
+ language_list.append(i)
+ _AVAILABLE_LANGUAGES[domain] = language_list
+ return copy.copy(language_list)
+
+
+def get_localized_message(message, user_locale):
+ """Gets a localized version of the given message in the given locale."""
+ if isinstance(message, Message):
+ if user_locale:
+ message.locale = user_locale
+ return unicode(message)
+ else:
+ return message
class LocaleHandler(logging.Handler):
diff --git a/tempest/openstack/common/jsonutils.py b/tempest/openstack/common/jsonutils.py
index bd43e59..c568a06 100644
--- a/tempest/openstack/common/jsonutils.py
+++ b/tempest/openstack/common/jsonutils.py
@@ -38,14 +38,18 @@
import inspect
import itertools
import json
-import types
-import xmlrpclib
+try:
+ import xmlrpclib
+except ImportError:
+ # NOTE(jd): xmlrpclib is not shipped with Python 3
+ xmlrpclib = None
-import netaddr
import six
+from tempest.openstack.common import importutils
from tempest.openstack.common import timeutils
+netaddr = importutils.try_import("netaddr")
_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
inspect.isfunction, inspect.isgeneratorfunction,
@@ -53,7 +57,8 @@
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
inspect.isabstract]
-_simple_types = (types.NoneType, int, basestring, bool, float, long)
+_simple_types = (six.string_types + six.integer_types
+ + (type(None), bool, float))
def to_primitive(value, convert_instances=False, convert_datetime=True,
@@ -125,7 +130,7 @@
# It's not clear why xmlrpclib created their own DateTime type, but
# for our purposes, make it a datetime type which is explicitly
# handled
- if isinstance(value, xmlrpclib.DateTime):
+ if xmlrpclib and isinstance(value, xmlrpclib.DateTime):
value = datetime.datetime(*tuple(value.timetuple())[:6])
if convert_datetime and isinstance(value, datetime.datetime):
@@ -138,7 +143,7 @@
# Likely an instance of something. Watch for cycles.
# Ignore class member vars.
return recursive(value.__dict__, level=level + 1)
- elif isinstance(value, netaddr.IPAddress):
+ elif netaddr and isinstance(value, netaddr.IPAddress):
return six.text_type(value)
else:
if any(test(value) for test in _nasty_type_tests):
diff --git a/tempest/openstack/common/local.py b/tempest/openstack/common/local.py
index f1bfc82..e82f17d 100644
--- a/tempest/openstack/common/local.py
+++ b/tempest/openstack/common/local.py
@@ -15,16 +15,15 @@
# License for the specific language governing permissions and limitations
# under the License.
-"""Greenthread local storage of variables using weak references"""
+"""Local storage of variables using weak references"""
+import threading
import weakref
-from eventlet import corolocal
-
-class WeakLocal(corolocal.local):
+class WeakLocal(threading.local):
def __getattribute__(self, attr):
- rval = corolocal.local.__getattribute__(self, attr)
+ rval = super(WeakLocal, self).__getattribute__(attr)
if rval:
# NOTE(mikal): this bit is confusing. What is stored is a weak
# reference, not the value itself. We therefore need to lookup
@@ -34,7 +33,7 @@
def __setattr__(self, attr, value):
value = weakref.ref(value)
- return corolocal.local.__setattr__(self, attr, value)
+ return super(WeakLocal, self).__setattr__(attr, value)
# NOTE(mikal): the name "store" should be deprecated in the future
@@ -45,4 +44,4 @@
# "strong" store will hold a reference to the object so that it never falls out
# of scope.
weak_store = WeakLocal()
-strong_store = corolocal.local
+strong_store = threading.local()
diff --git a/tempest/openstack/common/lockutils.py b/tempest/openstack/common/lockutils.py
index 3ff1a7a..0abd1a7 100644
--- a/tempest/openstack/common/lockutils.py
+++ b/tempest/openstack/common/lockutils.py
@@ -20,10 +20,10 @@
import errno
import functools
import os
+import threading
import time
import weakref
-from eventlet import semaphore
from oslo.config import cfg
from tempest.openstack.common import fileutils
@@ -137,7 +137,8 @@
def lock(name, lock_file_prefix=None, external=False, lock_path=None):
"""Context based lock
- This function yields a `semaphore.Semaphore` instance unless external is
+ This function yields a `threading.Semaphore` instance (if we don't use
+ eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
True, in which case, it'll yield an InterProcessLock instance.
:param lock_file_prefix: The lock_file_prefix argument is used to provide
@@ -155,7 +156,7 @@
# NOTE(soren): If we ever go natively threaded, this will be racy.
# See http://stackoverflow.com/questions/5390569/dyn
# amically-allocating-and-destroying-mutexes
- sem = _semaphores.get(name, semaphore.Semaphore())
+ sem = _semaphores.get(name, threading.Semaphore())
if name not in _semaphores:
# this check is not racy - we're already holding ref locally
# so GC won't remove the item and there was no IO switch
diff --git a/tempest/openstack/common/timeutils.py b/tempest/openstack/common/timeutils.py
index bd60489..60f02bc 100644
--- a/tempest/openstack/common/timeutils.py
+++ b/tempest/openstack/common/timeutils.py
@@ -21,6 +21,7 @@
import calendar
import datetime
+import time
import iso8601
import six
@@ -49,9 +50,9 @@
try:
return iso8601.parse_date(timestr)
except iso8601.ParseError as e:
- raise ValueError(e.message)
+ raise ValueError(unicode(e))
except TypeError as e:
- raise ValueError(e.message)
+ raise ValueError(unicode(e))
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
@@ -90,6 +91,11 @@
def utcnow_ts():
"""Timestamp version of our utcnow function."""
+ if utcnow.override_time is None:
+ # NOTE(kgriffs): This is several times faster
+ # than going through calendar.timegm(...)
+ return int(time.time())
+
return calendar.timegm(utcnow().timetuple())
diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py
index f428c1e..0999e2c 100644
--- a/tools/install_venv_common.py
+++ b/tools/install_venv_common.py
@@ -114,9 +114,10 @@
print('Installing dependencies with pip (this can take a while)...')
# First things first, make sure our venv has the latest pip and
- # setuptools.
- self.pip_install('pip>=1.3')
+ # setuptools and pbr
+ self.pip_install('pip>=1.4')
self.pip_install('setuptools')
+ self.pip_install('pbr')
self.pip_install('-r', self.requirements)
self.pip_install('-r', self.test_requirements)
@@ -201,12 +202,13 @@
RHEL: https://bugzilla.redhat.com/958868
"""
- # Install "patch" program if it's not there
- if not self.check_pkg('patch'):
- self.die("Please install 'patch'.")
+ if os.path.exists('contrib/redhat-eventlet.patch'):
+ # Install "patch" program if it's not there
+ if not self.check_pkg('patch'):
+ self.die("Please install 'patch'.")
- # Apply the eventlet patch
- self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
- 'site-packages',
- 'eventlet/green/subprocess.py'),
- 'contrib/redhat-eventlet.patch')
+ # Apply the eventlet patch
+ self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
+ 'site-packages',
+ 'eventlet/green/subprocess.py'),
+ 'contrib/redhat-eventlet.patch')