Sync latest versions of oslo incubator

This commit pulls in the lastest version of oslo incubator modules.

Change-Id: Ib591897958d6e12c251375d30342218b7d2cdd05
diff --git a/tempest/openstack/common/excutils.py b/tempest/openstack/common/excutils.py
index 81aad14..db37660 100644
--- a/tempest/openstack/common/excutils.py
+++ b/tempest/openstack/common/excutils.py
@@ -24,6 +24,8 @@
 import time
 import traceback
 
+import six
+
 from tempest.openstack.common.gettextutils import _  # noqa
 
 
@@ -65,7 +67,7 @@
                                                      self.tb))
             return False
         if self.reraise:
-            raise self.type_, self.value, self.tb
+            six.reraise(self.type_, self.value, self.tb)
 
 
 def forever_retry_uncaught_exceptions(infunc):
@@ -77,7 +79,8 @@
             try:
                 return infunc(*args, **kwargs)
             except Exception as exc:
-                if exc.message == last_exc_message:
+                this_exc_message = unicode(exc)
+                if this_exc_message == last_exc_message:
                     exc_count += 1
                 else:
                     exc_count = 1
@@ -85,12 +88,12 @@
                 # the exception message changes
                 cur_time = int(time.time())
                 if (cur_time - last_log_time > 60 or
-                        exc.message != last_exc_message):
+                        this_exc_message != last_exc_message):
                     logging.exception(
                         _('Unexpected exception occurred %d time(s)... '
                           'retrying.') % exc_count)
                     last_log_time = cur_time
-                    last_exc_message = exc.message
+                    last_exc_message = this_exc_message
                     exc_count = 0
                 # This should be a very rare event. In case it isn't, do
                 # a sleep.
diff --git a/tempest/openstack/common/fileutils.py b/tempest/openstack/common/fileutils.py
index d2e3d3e..6cf68ba 100644
--- a/tempest/openstack/common/fileutils.py
+++ b/tempest/openstack/common/fileutils.py
@@ -69,33 +69,34 @@
     return (reloaded, cache_info['data'])
 
 
-def delete_if_exists(path):
+def delete_if_exists(path, remove=os.unlink):
     """Delete a file, but ignore file not found error.
 
     :param path: File to delete
+    :param remove: Optional function to remove passed path
     """
 
     try:
-        os.unlink(path)
+        remove(path)
     except OSError as e:
-        if e.errno == errno.ENOENT:
-            return
-        else:
+        if e.errno != errno.ENOENT:
             raise
 
 
 @contextlib.contextmanager
-def remove_path_on_error(path):
+def remove_path_on_error(path, remove=delete_if_exists):
     """Protect code that wants to operate on PATH atomically.
     Any exception will cause PATH to be removed.
 
     :param path: File to work with
+    :param remove: Optional function to remove passed path
     """
+
     try:
         yield
     except Exception:
         with excutils.save_and_reraise_exception():
-            delete_if_exists(path)
+            remove(path)
 
 
 def file_open(*args, **kwargs):
diff --git a/tempest/openstack/common/gettextutils.py b/tempest/openstack/common/gettextutils.py
index 8594937..cbf570a 100644
--- a/tempest/openstack/common/gettextutils.py
+++ b/tempest/openstack/common/gettextutils.py
@@ -1,8 +1,8 @@
 # vim: tabstop=4 shiftwidth=4 softtabstop=4
 
 # Copyright 2012 Red Hat, Inc.
-# All Rights Reserved.
 # Copyright 2013 IBM Corp.
+# All Rights Reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -26,22 +26,44 @@
 
 import copy
 import gettext
-import logging.handlers
+import logging
 import os
 import re
-import UserString
+try:
+    import UserString as _userString
+except ImportError:
+    import collections as _userString
 
+from babel import localedata
 import six
 
 _localedir = os.environ.get('tempest'.upper() + '_LOCALEDIR')
 _t = gettext.translation('tempest', localedir=_localedir, fallback=True)
 
+_AVAILABLE_LANGUAGES = {}
+USE_LAZY = False
+
+
+def enable_lazy():
+    """Convenience function for configuring _() to use lazy gettext
+
+    Call this at the start of execution to enable the gettextutils._
+    function to use lazy gettext functionality. This is useful if
+    your project is importing _ directly instead of using the
+    gettextutils.install() way of importing the _ function.
+    """
+    global USE_LAZY
+    USE_LAZY = True
+
 
 def _(msg):
-    return _t.ugettext(msg)
+    if USE_LAZY:
+        return Message(msg, 'tempest')
+    else:
+        return _t.ugettext(msg)
 
 
-def install(domain):
+def install(domain, lazy=False):
     """Install a _() function using the given translation domain.
 
     Given a translation domain, install a _() function using gettext's
@@ -51,44 +73,48 @@
     overriding the default localedir (e.g. /usr/share/locale) using
     a translation-domain-specific environment variable (e.g.
     NOVA_LOCALEDIR).
+
+    :param domain: the translation domain
+    :param lazy: indicates whether or not to install the lazy _() function.
+                 The lazy _() introduces a way to do deferred translation
+                 of messages by installing a _ that builds Message objects,
+                 instead of strings, which can then be lazily translated into
+                 any available locale.
     """
-    gettext.install(domain,
-                    localedir=os.environ.get(domain.upper() + '_LOCALEDIR'),
-                    unicode=True)
+    if lazy:
+        # NOTE(mrodden): Lazy gettext functionality.
+        #
+        # The following introduces a deferred way to do translations on
+        # messages in OpenStack. We override the standard _() function
+        # and % (format string) operation to build Message objects that can
+        # later be translated when we have more information.
+        #
+        # Also included below is an example LocaleHandler that translates
+        # Messages to an associated locale, effectively allowing many logs,
+        # each with their own locale.
+
+        def _lazy_gettext(msg):
+            """Create and return a Message object.
+
+            Lazy gettext function for a given domain, it is a factory method
+            for a project/module to get a lazy gettext function for its own
+            translation domain (i.e. nova, glance, cinder, etc.)
+
+            Message encapsulates a string so that we can translate
+            it later when needed.
+            """
+            return Message(msg, domain)
+
+        import __builtin__
+        __builtin__.__dict__['_'] = _lazy_gettext
+    else:
+        localedir = '%s_LOCALEDIR' % domain.upper()
+        gettext.install(domain,
+                        localedir=os.environ.get(localedir),
+                        unicode=True)
 
 
-"""
-Lazy gettext functionality.
-
-The following is an attempt to introduce a deferred way
-to do translations on messages in OpenStack. We attempt to
-override the standard _() function and % (format string) operation
-to build Message objects that can later be translated when we have
-more information. Also included is an example LogHandler that
-translates Messages to an associated locale, effectively allowing
-many logs, each with their own locale.
-"""
-
-
-def get_lazy_gettext(domain):
-    """Assemble and return a lazy gettext function for a given domain.
-
-    Factory method for a project/module to get a lazy gettext function
-    for its own translation domain (i.e. nova, glance, cinder, etc.)
-    """
-
-    def _lazy_gettext(msg):
-        """Create and return a Message object.
-
-        Message encapsulates a string so that we can translate it later when
-        needed.
-        """
-        return Message(msg, domain)
-
-    return _lazy_gettext
-
-
-class Message(UserString.UserString, object):
+class Message(_userString.UserString, object):
     """Class used to encapsulate translatable messages."""
     def __init__(self, msg, domain):
         # _msg is the gettext msgid and should never change
@@ -130,7 +156,7 @@
         # look for %(blah) fields in string;
         # ignore %% and deal with the
         # case where % is first character on the line
-        keys = re.findall('(?:[^%]|^)%\((\w*)\)[a-z]', full_msg)
+        keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
 
         # if we don't find any %(blah) blocks but have a %s
         if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
@@ -229,7 +255,47 @@
         if name in ops:
             return getattr(self.data, name)
         else:
-            return UserString.UserString.__getattribute__(self, name)
+            return _userString.UserString.__getattribute__(self, name)
+
+
+def get_available_languages(domain):
+    """Lists the available languages for the given translation domain.
+
+    :param domain: the domain to get languages for
+    """
+    if domain in _AVAILABLE_LANGUAGES:
+        return copy.copy(_AVAILABLE_LANGUAGES[domain])
+
+    localedir = '%s_LOCALEDIR' % domain.upper()
+    find = lambda x: gettext.find(domain,
+                                  localedir=os.environ.get(localedir),
+                                  languages=[x])
+
+    # NOTE(mrodden): en_US should always be available (and first in case
+    # order matters) since our in-line message strings are en_US
+    language_list = ['en_US']
+    # NOTE(luisg): Babel <1.0 used a function called list(), which was
+    # renamed to locale_identifiers() in >=1.0, the requirements master list
+    # requires >=0.9.6, uncapped, so defensively work with both. We can remove
+    # this check when the master list updates to >=1.0, and all projects udpate
+    list_identifiers = (getattr(localedata, 'list', None) or
+                        getattr(localedata, 'locale_identifiers'))
+    locale_identifiers = list_identifiers()
+    for i in locale_identifiers:
+        if find(i) is not None:
+            language_list.append(i)
+    _AVAILABLE_LANGUAGES[domain] = language_list
+    return copy.copy(language_list)
+
+
+def get_localized_message(message, user_locale):
+    """Gets a localized version of the given message in the given locale."""
+    if isinstance(message, Message):
+        if user_locale:
+            message.locale = user_locale
+        return unicode(message)
+    else:
+        return message
 
 
 class LocaleHandler(logging.Handler):
diff --git a/tempest/openstack/common/jsonutils.py b/tempest/openstack/common/jsonutils.py
index bd43e59..c568a06 100644
--- a/tempest/openstack/common/jsonutils.py
+++ b/tempest/openstack/common/jsonutils.py
@@ -38,14 +38,18 @@
 import inspect
 import itertools
 import json
-import types
-import xmlrpclib
+try:
+    import xmlrpclib
+except ImportError:
+    # NOTE(jd): xmlrpclib is not shipped with Python 3
+    xmlrpclib = None
 
-import netaddr
 import six
 
+from tempest.openstack.common import importutils
 from tempest.openstack.common import timeutils
 
+netaddr = importutils.try_import("netaddr")
 
 _nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
                      inspect.isfunction, inspect.isgeneratorfunction,
@@ -53,7 +57,8 @@
                      inspect.iscode, inspect.isbuiltin, inspect.isroutine,
                      inspect.isabstract]
 
-_simple_types = (types.NoneType, int, basestring, bool, float, long)
+_simple_types = (six.string_types + six.integer_types
+                 + (type(None), bool, float))
 
 
 def to_primitive(value, convert_instances=False, convert_datetime=True,
@@ -125,7 +130,7 @@
         # It's not clear why xmlrpclib created their own DateTime type, but
         # for our purposes, make it a datetime type which is explicitly
         # handled
-        if isinstance(value, xmlrpclib.DateTime):
+        if xmlrpclib and isinstance(value, xmlrpclib.DateTime):
             value = datetime.datetime(*tuple(value.timetuple())[:6])
 
         if convert_datetime and isinstance(value, datetime.datetime):
@@ -138,7 +143,7 @@
             # Likely an instance of something. Watch for cycles.
             # Ignore class member vars.
             return recursive(value.__dict__, level=level + 1)
-        elif isinstance(value, netaddr.IPAddress):
+        elif netaddr and isinstance(value, netaddr.IPAddress):
             return six.text_type(value)
         else:
             if any(test(value) for test in _nasty_type_tests):
diff --git a/tempest/openstack/common/local.py b/tempest/openstack/common/local.py
index f1bfc82..e82f17d 100644
--- a/tempest/openstack/common/local.py
+++ b/tempest/openstack/common/local.py
@@ -15,16 +15,15 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-"""Greenthread local storage of variables using weak references"""
+"""Local storage of variables using weak references"""
 
+import threading
 import weakref
 
-from eventlet import corolocal
 
-
-class WeakLocal(corolocal.local):
+class WeakLocal(threading.local):
     def __getattribute__(self, attr):
-        rval = corolocal.local.__getattribute__(self, attr)
+        rval = super(WeakLocal, self).__getattribute__(attr)
         if rval:
             # NOTE(mikal): this bit is confusing. What is stored is a weak
             # reference, not the value itself. We therefore need to lookup
@@ -34,7 +33,7 @@
 
     def __setattr__(self, attr, value):
         value = weakref.ref(value)
-        return corolocal.local.__setattr__(self, attr, value)
+        return super(WeakLocal, self).__setattr__(attr, value)
 
 
 # NOTE(mikal): the name "store" should be deprecated in the future
@@ -45,4 +44,4 @@
 # "strong" store will hold a reference to the object so that it never falls out
 # of scope.
 weak_store = WeakLocal()
-strong_store = corolocal.local
+strong_store = threading.local()
diff --git a/tempest/openstack/common/lockutils.py b/tempest/openstack/common/lockutils.py
index 3ff1a7a..0abd1a7 100644
--- a/tempest/openstack/common/lockutils.py
+++ b/tempest/openstack/common/lockutils.py
@@ -20,10 +20,10 @@
 import errno
 import functools
 import os
+import threading
 import time
 import weakref
 
-from eventlet import semaphore
 from oslo.config import cfg
 
 from tempest.openstack.common import fileutils
@@ -137,7 +137,8 @@
 def lock(name, lock_file_prefix=None, external=False, lock_path=None):
     """Context based lock
 
-    This function yields a `semaphore.Semaphore` instance unless external is
+    This function yields a `threading.Semaphore` instance (if we don't use
+    eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
     True, in which case, it'll yield an InterProcessLock instance.
 
     :param lock_file_prefix: The lock_file_prefix argument is used to provide
@@ -155,7 +156,7 @@
     # NOTE(soren): If we ever go natively threaded, this will be racy.
     #              See http://stackoverflow.com/questions/5390569/dyn
     #              amically-allocating-and-destroying-mutexes
-    sem = _semaphores.get(name, semaphore.Semaphore())
+    sem = _semaphores.get(name, threading.Semaphore())
     if name not in _semaphores:
         # this check is not racy - we're already holding ref locally
         # so GC won't remove the item and there was no IO switch
diff --git a/tempest/openstack/common/timeutils.py b/tempest/openstack/common/timeutils.py
index bd60489..60f02bc 100644
--- a/tempest/openstack/common/timeutils.py
+++ b/tempest/openstack/common/timeutils.py
@@ -21,6 +21,7 @@
 
 import calendar
 import datetime
+import time
 
 import iso8601
 import six
@@ -49,9 +50,9 @@
     try:
         return iso8601.parse_date(timestr)
     except iso8601.ParseError as e:
-        raise ValueError(e.message)
+        raise ValueError(unicode(e))
     except TypeError as e:
-        raise ValueError(e.message)
+        raise ValueError(unicode(e))
 
 
 def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
@@ -90,6 +91,11 @@
 
 def utcnow_ts():
     """Timestamp version of our utcnow function."""
+    if utcnow.override_time is None:
+        # NOTE(kgriffs): This is several times faster
+        # than going through calendar.timegm(...)
+        return int(time.time())
+
     return calendar.timegm(utcnow().timetuple())
 
 
diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py
index f428c1e..0999e2c 100644
--- a/tools/install_venv_common.py
+++ b/tools/install_venv_common.py
@@ -114,9 +114,10 @@
         print('Installing dependencies with pip (this can take a while)...')
 
         # First things first, make sure our venv has the latest pip and
-        # setuptools.
-        self.pip_install('pip>=1.3')
+        # setuptools and pbr
+        self.pip_install('pip>=1.4')
         self.pip_install('setuptools')
+        self.pip_install('pbr')
 
         self.pip_install('-r', self.requirements)
         self.pip_install('-r', self.test_requirements)
@@ -201,12 +202,13 @@
         RHEL: https://bugzilla.redhat.com/958868
         """
 
-        # Install "patch" program if it's not there
-        if not self.check_pkg('patch'):
-            self.die("Please install 'patch'.")
+        if os.path.exists('contrib/redhat-eventlet.patch'):
+            # Install "patch" program if it's not there
+            if not self.check_pkg('patch'):
+                self.die("Please install 'patch'.")
 
-        # Apply the eventlet patch
-        self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
-                                      'site-packages',
-                                      'eventlet/green/subprocess.py'),
-                         'contrib/redhat-eventlet.patch')
+            # Apply the eventlet patch
+            self.apply_patch(os.path.join(self.venv, 'lib', self.py_version,
+                                          'site-packages',
+                                          'eventlet/green/subprocess.py'),
+                             'contrib/redhat-eventlet.patch')