Merge "Use 2 network clients from tempest-lib"
diff --git a/.gitignore b/.gitignore
index efba45e..d58b162 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,9 @@
 dist
 build
 .testrepository
+.idea
+.project
+.pydevproject
 .coverage*
 !.coveragerc
 cover/
diff --git a/.mailmap b/.mailmap
index 5c37a5e..a43c0b9 100644
--- a/.mailmap
+++ b/.mailmap
@@ -1,9 +1,25 @@
-Ravikumar Venkatesan <ravikumar.venkatesan@hp.com> ravikumar-venkatesan <ravikumar.venkatesan@hp.com>
-Ravikumar Venkatesan <ravikumar.venkatesan@hp.com> ravikumar venkatesan <ravikumar.venkatesan@hp.com>
-Rohit Karajgi <rohit.karajgi@nttdata.com> Rohit Karajgi <rohit.karajgi@vertex.co.in>
-Jay Pipes <jaypipes@gmail.com> Jay Pipes <jpipes@librebox.gateway.2wire.net>
-Joe Gordon <joe.gordon0@gmail.com> <jogo@cloudscaling.com>
 <brian.waldon@rackspace.com> <bcwaldon@gmail.com>
-Daryl Walleck <daryl.walleck@rackspace.com> dwalleck <daryl.walleck@rackspace.com>
 <jeblair@hp.com> <corvus@inaugust.com>
 <jeblair@hp.com> <james.blair@rackspace.com>
+Adam Gandelman <adamg@ubuntu.com> Adam Gandelman <adamg@canonical.com>
+Andrea Frittoli (andreaf) <andrea.frittoli@hpe.com> Andrea Frittoli (andreaf) <andrea.frittoli@hp.com>
+Andrea Frittoli (andreaf) <andrea.frittoli@hpe.com> Andrea Frittoli <andrea.frittoli@hp.com>
+Daryl Walleck <daryl.walleck@rackspace.com> dwalleck <daryl.walleck@rackspace.com>
+David Kranz <dkranz@redhat.com> David Kranz <david.kranz@qrclab.com>
+Ghanshyam <ghanshyam.mann@nectechnologies.in> Ghanshyam Mann <ghanshyam.mann@nectechnologies.in>
+Ghanshyam <ghanshyam.mann@nectechnologies.in> ghanshyam <ghanshyam.mann@nectechnologies.in>
+Jay Pipes <jaypipes@gmail.com> Jay Pipes <jpipes@librebox.gateway.2wire.net>
+Joe Gordon <joe.gordon0@gmail.com> <jogo@cloudscaling.com>
+Ken'ichi Ohmichi <ken-oomichi@wx.jp.nec.com> Ken'ichi Ohmichi <oomichi@mxs.nes.nec.co.jp>
+Marc Koderer <marc@koderer.com> Marc Koderer <m.koderer@telekom.de>
+Masayuki Igawa <masayuki.igawa@gmail.com> Masayuki Igawa <igawa@mxs.nes.nec.co.jp>
+Masayuki Igawa <masayuki.igawa@gmail.com> Masayuki Igawa <mas-igawa@ut.jp.nec.com>
+Matthew Treinish <mtreinish@kortar.org> Matthew Treinish <treinish@linux.vnet.ibm.com>
+Nayna Patel <nayna.patel@hp.com> nayna-patel <nayna.patel@hp.com>
+ravikumar-venkatesan <ravikumar.venkatesan@hp.com> Ravikumar Venkatesan <ravikumar.venkatesan@hp.com>
+ravikumar-venkatesan <ravikumar.venkatesan@hp.com> ravikumar venkatesan <ravikumar.venkatesan@hp.com>
+Rohit Karajgi <rohit.karajgi@nttdata.com> Rohit Karajgi <rohit.karajgi@vertex.co.in>
+Sean Dague <sean@dague.net> Sean Dague <sdague@linux.vnet.ibm.com>
+Sean Dague <sean@dague.net> Sean Dague <sean.dague@samsung.com>
+Yuiko Takada <takada-yuiko@mxn.nes.nec.co.jp> YuikoTakada <takada-yuiko@mxn.nes.nec.co.jp>
+Zhi Kun Liu <zhikunli@cn.ibm.com> Liu, Zhi Kun <zhikunli@cn.ibm.com>
diff --git a/requirements.txt b/requirements.txt
index 7a6ed97..66e5696 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,12 +2,12 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 pbr>=1.6 # Apache-2.0
-cliff>=1.15.0 # Apache-2.0
+cliff!=1.16.0,>=1.15.0 # Apache-2.0
 anyjson>=0.3.3 # BSD
 httplib2>=0.7.5 # MIT
 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
 testtools>=1.4.0 # MIT
-paramiko>=1.13.0 # LGPL
+paramiko>=1.16.0 # LGPL
 netaddr!=0.7.16,>=0.7.12 # BSD
 testrepository>=0.0.18 # Apache-2.0/BSD
 pyOpenSSL>=0.14 # Apache-2.0
diff --git a/tempest/api/identity/admin/v3/test_endpoints.py b/tempest/api/identity/admin/v3/test_endpoints.py
index 2ac832e..2538829 100644
--- a/tempest/api/identity/admin/v3/test_endpoints.py
+++ b/tempest/api/identity/admin/v3/test_endpoints.py
@@ -71,7 +71,7 @@
                          ', '.join(str(e) for e in missing_endpoints))
 
     @test.idempotent_id('0e2446d2-c1fd-461b-a729-b9e73e3e3b37')
-    def test_create_list_delete_endpoint(self):
+    def test_create_list_show_delete_endpoint(self):
         region = data_utils.rand_name('region')
         url = data_utils.rand_url()
         interface = 'public'
@@ -79,16 +79,30 @@
                                                interface=interface,
                                                url=url, region=region,
                                                enabled=True)['endpoint']
+
         # Asserting Create Endpoint response body
         self.assertIn('id', endpoint)
         self.assertEqual(region, endpoint['region'])
         self.assertEqual(url, endpoint['url'])
+
         # Checking if created endpoint is present in the list of endpoints
         fetched_endpoints = self.client.list_endpoints()['endpoints']
         fetched_endpoints_id = [e['id'] for e in fetched_endpoints]
         self.assertIn(endpoint['id'], fetched_endpoints_id)
+
+        # Show endpoint
+        fetched_endpoint = (
+            self.client.show_endpoint(endpoint['id'])['endpoint'])
+        # Asserting if the attributes of endpoint are the same
+        self.assertEqual(self.service_id, fetched_endpoint['service_id'])
+        self.assertEqual(interface, fetched_endpoint['interface'])
+        self.assertEqual(url, fetched_endpoint['url'])
+        self.assertEqual(region, fetched_endpoint['region'])
+        self.assertEqual(True, fetched_endpoint['enabled'])
+
         # Deleting the endpoint created in this method
         self.client.delete_endpoint(endpoint['id'])
+
         # Checking whether endpoint is deleted successfully
         fetched_endpoints = self.client.list_endpoints()['endpoints']
         fetched_endpoints_id = [e['id'] for e in fetched_endpoints]
diff --git a/tempest/api/identity/base.py b/tempest/api/identity/base.py
index 91d620c..d31569b 100644
--- a/tempest/api/identity/base.py
+++ b/tempest/api/identity/base.py
@@ -75,7 +75,6 @@
         cls.non_admin_client = cls.os.identity_public_client
         cls.non_admin_token_client = cls.os.token_client
         cls.non_admin_tenants_client = cls.os.tenants_public_client
-        cls.non_admin_roles_client = cls.os.roles_public_client
         cls.non_admin_users_client = cls.os.users_public_client
 
     @classmethod
diff --git a/tempest/api/network/test_ports.py b/tempest/api/network/test_ports.py
index a8ca266..d7b220b 100644
--- a/tempest/api/network/test_ports.py
+++ b/tempest/api/network/test_ports.py
@@ -22,6 +22,7 @@
 from tempest.common import custom_matchers
 from tempest.common.utils import data_utils
 from tempest import config
+from tempest import exceptions
 from tempest import test
 
 CONF = config.CONF
@@ -87,14 +88,16 @@
 
     @classmethod
     def _get_ipaddress_from_tempest_conf(cls):
-        """Return first subnet gateway for configured CIDR """
+        """Return subnet with mask bits for configured CIDR """
         if cls._ip_version == 4:
             cidr = netaddr.IPNetwork(CONF.network.tenant_network_cidr)
+            cidr.prefixlen = CONF.network.tenant_network_mask_bits
 
         elif cls._ip_version == 6:
             cidr = netaddr.IPNetwork(CONF.network.tenant_network_v6_cidr)
+            cidr.prefixlen = CONF.network.tenant_network_v6_mask_bits
 
-        return netaddr.IPAddress(cidr)
+        return cidr
 
     @test.attr(type='smoke')
     @test.idempotent_id('0435f278-40ae-48cb-a404-b8a087bc09b1')
@@ -102,9 +105,15 @@
         network = self.create_network()
         net_id = network['id']
         address = self._get_ipaddress_from_tempest_conf()
-        allocation_pools = {'allocation_pools': [{'start': str(address + 4),
-                                                  'end': str(address + 6)}]}
-        subnet = self.create_subnet(network, **allocation_pools)
+        if ((address.version == 4 and address.prefixlen >= 30) or
+           (address.version == 6 and address.prefixlen >= 126)):
+            msg = ("Subnet %s isn't large enough for the test" % address.cidr)
+            raise exceptions.InvalidConfiguration(message=msg)
+        allocation_pools = {'allocation_pools': [{'start': str(address[2]),
+                                                  'end': str(address[-2])}]}
+        subnet = self.create_subnet(network, cidr=address,
+                                    mask_bits=address.prefixlen,
+                                    **allocation_pools)
         self.addCleanup(self.subnets_client.delete_subnet, subnet['id'])
         body = self.ports_client.create_port(network_id=net_id)
         self.addCleanup(self.ports_client.delete_port, body['port']['id'])
diff --git a/tempest/clients.py b/tempest/clients.py
index c84b1a4..d617b0f 100644
--- a/tempest/clients.py
+++ b/tempest/clients.py
@@ -501,9 +501,10 @@
             'region': CONF.identity.region
         }
         params.update(self.default_params_with_timeout_values)
+
+        # Clients below use the admin endpoint type of Keystone API v2
         params_v2_admin = params.copy()
         params_v2_admin['endpoint_type'] = CONF.identity.v2_admin_endpoint_type
-        # Client uses admin endpoint type of Keystone API v2
         self.endpoints_v2_client = EndpointsV2Client(self.auth_provider,
                                                      **params_v2_admin)
         self.identity_client = IdentityClient(self.auth_provider,
@@ -516,21 +517,21 @@
                                         **params_v2_admin)
         self.services_v2_client = ServicesV2Client(self.auth_provider,
                                                    **params_v2_admin)
+
+        # Clients below use the public endpoint type of Keystone API v2
         params_v2_public = params.copy()
         params_v2_public['endpoint_type'] = (
             CONF.identity.v2_public_endpoint_type)
-        # Client uses public endpoint type of Keystone API v2
         self.identity_public_client = IdentityClient(self.auth_provider,
                                                      **params_v2_public)
         self.tenants_public_client = TenantsClient(self.auth_provider,
                                                    **params_v2_public)
-        self.roles_public_client = RolesClient(self.auth_provider,
-                                               **params_v2_public)
         self.users_public_client = UsersClient(self.auth_provider,
                                                **params_v2_public)
+
+        # Clients below use the endpoint type of Keystone API v3
         params_v3 = params.copy()
         params_v3['endpoint_type'] = CONF.identity.v3_endpoint_type
-        # Clients below use the endpoint type of Keystone API v3
         self.identity_v3_client = IdentityV3Client(self.auth_provider,
                                                    **params_v3)
         self.endpoints_client = EndPointV3Client(self.auth_provider,
@@ -543,6 +544,7 @@
         self.credentials_client = CredentialsV3Client(self.auth_provider,
                                                       **params_v3)
         self.groups_client = GroupsV3Client(self.auth_provider, **params_v3)
+
         # Token clients do not use the catalog. They only need default_params.
         # They read auth_url, so they should only be set if the corresponding
         # API version is marked as enabled
diff --git a/tempest/common/utils/__init__.py b/tempest/common/utils/__init__.py
index 81b8110..aad6373 100644
--- a/tempest/common/utils/__init__.py
+++ b/tempest/common/utils/__init__.py
@@ -27,8 +27,6 @@
 
 class DataUtils(object):
     def __getattr__(self, attr):
-        if attr in self.__dict__:
-            return self.__dict__[attr]
 
         if attr == 'rand_name':
             # NOTE(flwang): This is a proxy to generate a random name that
diff --git a/tempest/common/utils/linux/remote_client.py b/tempest/common/utils/linux/remote_client.py
index 2e233c5..8f5faef 100644
--- a/tempest/common/utils/linux/remote_client.py
+++ b/tempest/common/utils/linux/remote_client.py
@@ -16,6 +16,7 @@
 
 from oslo_log import log as logging
 from tempest_lib.common import ssh
+import tempest_lib.exceptions
 
 from tempest import config
 from tempest import exceptions
@@ -178,4 +179,10 @@
 
     def make_fs(self, dev_name, fs='ext4'):
         cmd_mkfs = 'sudo /usr/sbin/mke2fs -t %s /dev/%s' % (fs, dev_name)
-        self.exec_command(cmd_mkfs)
+        try:
+            self.exec_command(cmd_mkfs)
+        except tempest_lib.exceptions.SSHExecCommandFailed:
+            LOG.error("Couldn't mke2fs")
+            cmd_why = 'sudo ls -lR /dev'
+            LOG.info("Contents of /dev: %s" % self.exec_command(cmd_why))
+            raise
diff --git a/tempest/scenario/manager.py b/tempest/scenario/manager.py
index 72424ad..1962286 100644
--- a/tempest/scenario/manager.py
+++ b/tempest/scenario/manager.py
@@ -377,7 +377,7 @@
             linux_client.validate_authentication()
         except Exception as e:
             message = ('Initializing SSH connection to %(ip)s failed. '
-                       'Error: %(error)s' % {'ip_address': ip_address,
+                       'Error: %(error)s' % {'ip': ip_address,
                                              'error': e})
             caller = misc_utils.find_test_caller()
             if caller:
diff --git a/tempest/services/identity/v3/json/endpoints_client.py b/tempest/services/identity/v3/json/endpoints_client.py
index ede5edb..8ab7464 100644
--- a/tempest/services/identity/v3/json/endpoints_client.py
+++ b/tempest/services/identity/v3/json/endpoints_client.py
@@ -61,3 +61,10 @@
         resp_header, resp_body = self.delete('endpoints/%s' % endpoint_id)
         self.expected_success(204, resp_header.status)
         return service_client.ResponseBody(resp_header, resp_body)
+
+    def show_endpoint(self, endpoint_id):
+        """Get endpoint."""
+        resp_header, resp_body = self.get('endpoints/%s' % endpoint_id)
+        self.expected_success(200, resp_header.status)
+        resp_body = json.loads(resp_body)
+        return service_client.ResponseBody(resp_header, resp_body)
diff --git a/tempest/test.py b/tempest/test.py
index dfed947..9c04ea1 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -293,7 +293,7 @@
                     LOG.exception("teardown of %s failed: %s" % (name, te))
                 if not etype:
                     etype, value, trace = sys_exec_info
-        # If exceptions were raised during teardown, an not before, re-raise
+        # If exceptions were raised during teardown, and not before, re-raise
         # the first one
         if re_raise and etype is not None:
             try:
@@ -600,7 +600,7 @@
         networks_client = cls.get_client_manager().compute_networks_client
         cred_provider = cls._get_credentials_provider()
         # In case of nova network, isolated tenants are not able to list the
-        # network configured in fixed_network_name, even if the can use it
+        # network configured in fixed_network_name, even if they can use it
         # for their servers, so using an admin network client to validate
         # the network name
         if (not CONF.service_available.neutron and
@@ -790,9 +790,9 @@
 
     @classmethod
     def set_resource(cls, name, resource):
-        """Register a resoruce for a test
+        """Register a resource for a test
 
-        This function can be used in setUpClass context to register a resoruce
+        This function can be used in setUpClass context to register a resource
         for a test.
 
         :param name: The name of the kind of resource such as "flavor", "role",
diff --git a/tools/install_venv_common.py b/tools/install_venv_common.py
index d1643f8..e279159 100644
--- a/tools/install_venv_common.py
+++ b/tools/install_venv_common.py
@@ -113,7 +113,7 @@
 
         # First things first, make sure our venv has the latest pip and
         # setuptools and pbr
-        self.pip_install('pip<8,>=1.4')
+        self.pip_install('pip>=1.4')
         self.pip_install('setuptools')
         self.pip_install('pbr')