Merge "Change containers to list if string"
diff --git a/roles/run-tempest/README.rst b/roles/run-tempest/README.rst
index 001586e..b5defb7 100644
--- a/roles/run-tempest/README.rst
+++ b/roles/run-tempest/README.rst
@@ -16,9 +16,19 @@
    :default: ''
 
    A regular expression used to select the tests.
+
    It works only when used with some specific tox environments
    ('all', 'all-plugin'.)
 
+   Multi-line and commented regexs can be achieved by doing this:
+
+       ::
+           vars:
+             tempest_test_regex: |
+               (?x)    # Ignore comments and whitespaces
+               # Line with only a comment.
+               (tempest\.(api|scenario|thirdparty)).*$    # Run only api scenario and third party
+
 .. zuul:rolevar:: tox_venvlist
    :default: smoke
 
diff --git a/tempest/api/compute/servers/test_server_actions.py b/tempest/api/compute/servers/test_server_actions.py
index e2be249..5c3e9f0 100644
--- a/tempest/api/compute/servers/test_server_actions.py
+++ b/tempest/api/compute/servers/test_server_actions.py
@@ -582,6 +582,12 @@
         compute.shelve_server(self.client, self.server_id,
                               force_shelve_offload=True)
 
+        def _unshelve_server():
+            server_info = self.client.show_server(self.server_id)['server']
+            if 'SHELVED' in server_info['status']:
+                self.client.unshelve_server(self.server_id)
+        self.addOnException(_unshelve_server)
+
         server = self.client.show_server(self.server_id)['server']
         image_name = server['name'] + '-shelved'
         params = {'name': image_name}
diff --git a/tempest/api/compute/servers/test_servers_negative.py b/tempest/api/compute/servers/test_servers_negative.py
index d067bb3..9b545af 100644
--- a/tempest/api/compute/servers/test_servers_negative.py
+++ b/tempest/api/compute/servers/test_servers_negative.py
@@ -477,6 +477,12 @@
         # shelve a shelved server.
         compute.shelve_server(self.client, self.server_id)
 
+        def _unshelve_server():
+            server_info = self.client.show_server(self.server_id)['server']
+            if 'SHELVED' in server_info['status']:
+                self.client.unshelve_server(self.server_id)
+        self.addOnException(_unshelve_server)
+
         server = self.client.show_server(self.server_id)['server']
         image_name = server['name'] + '-shelved'
         params = {'name': image_name}
diff --git a/tempest/api/network/admin/test_external_network_extension.py b/tempest/api/network/admin/test_external_network_extension.py
index 4d41e33..49a9cdb 100644
--- a/tempest/api/network/admin/test_external_network_extension.py
+++ b/tempest/api/network/admin/test_external_network_extension.py
@@ -130,5 +130,3 @@
         subnet_list = self.admin_subnets_client.list_subnets()
         self.assertNotIn(subnet['id'],
                          (s['id'] for s in subnet_list))
-        # Removes subnet from the cleanup list
-        self.subnets.remove(subnet)
diff --git a/tempest/api/network/base.py b/tempest/api/network/base.py
index bdfda0a..8670165 100644
--- a/tempest/api/network/base.py
+++ b/tempest/api/network/base.py
@@ -88,7 +88,6 @@
     @classmethod
     def resource_setup(cls):
         super(BaseNetworkTest, cls).resource_setup()
-        cls.networks = []
         cls.subnets = []
         cls.ports = []
         cls.routers = []
@@ -101,27 +100,6 @@
             cls.mask_bits = CONF.network.project_network_v6_mask_bits
 
     @classmethod
-    def resource_cleanup(cls):
-        if CONF.service_available.neutron:
-            # Clean up ports
-            for port in cls.ports:
-                test_utils.call_and_ignore_notfound_exc(
-                    cls.ports_client.delete_port, port['id'])
-            # Clean up routers
-            for router in cls.routers:
-                test_utils.call_and_ignore_notfound_exc(
-                    cls.delete_router, router)
-            # Clean up subnets
-            for subnet in cls.subnets:
-                test_utils.call_and_ignore_notfound_exc(
-                    cls.subnets_client.delete_subnet, subnet['id'])
-            # Clean up networks
-            for network in cls.networks:
-                test_utils.call_and_ignore_notfound_exc(
-                    cls.networks_client.delete_network, network['id'])
-        super(BaseNetworkTest, cls).resource_cleanup()
-
-    @classmethod
     def create_network(cls, network_name=None, **kwargs):
         """Wrapper utility that returns a test network."""
         network_name = network_name or data_utils.rand_name(
@@ -129,7 +107,9 @@
 
         body = cls.networks_client.create_network(name=network_name, **kwargs)
         network = body['network']
-        cls.networks.append(network)
+        cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+                                    cls.networks_client.delete_network,
+                                    network['id'])
         return network
 
     @classmethod
@@ -172,6 +152,9 @@
             message = 'Available CIDR for subnet creation could not be found'
             raise exceptions.BuildErrorException(message)
         subnet = body['subnet']
+        cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+                                    cls.subnets_client.delete_subnet,
+                                    subnet['id'])
         cls.subnets.append(subnet)
         return subnet
 
@@ -181,6 +164,8 @@
         body = cls.ports_client.create_port(network_id=network['id'],
                                             **kwargs)
         port = body['port']
+        cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+                                    cls.ports_client.delete_port, port['id'])
         cls.ports.append(port)
         return port
 
@@ -207,6 +192,8 @@
             name=router_name, external_gateway_info=ext_gw_info,
             admin_state_up=admin_state_up, **kwargs)
         router = body['router']
+        cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+                                    cls.delete_router, router)
         cls.routers.append(router)
         return router
 
diff --git a/tempest/api/network/test_networks.py b/tempest/api/network/test_networks.py
index 1c59556..7345fd1 100644
--- a/tempest/api/network/test_networks.py
+++ b/tempest/api/network/test_networks.py
@@ -104,15 +104,6 @@
         self.assertThat(actual, custom_matchers.MatchesDictExceptForKeys(
                         expected, exclude_keys))
 
-    def _delete_network(self, network):
-        # Deleting network also deletes its subnets if exists
-        self.networks_client.delete_network(network['id'])
-        if network in self.networks:
-            self.networks.remove(network)
-        for subnet in self.subnets:
-            if subnet['network_id'] == network['id']:
-                self.subnets.remove(subnet)
-
     def _create_verify_delete_subnet(self, cidr=None, mask_bits=None,
                                      **kwargs):
         network = self.create_network()
@@ -132,8 +123,6 @@
 
         self._compare_resource_attrs(subnet, compare_args)
         self.networks_client.delete_network(net_id)
-        self.networks.pop()
-        self.subnets.pop()
 
 
 class NetworksTest(BaseNetworkTestResources):
@@ -171,7 +160,7 @@
     def test_create_update_delete_network_subnet(self):
         # Create a network
         network = self.create_network()
-        self.addCleanup(self._delete_network, network)
+        self.addCleanup(self.networks_client.delete_network, network['id'])
         net_id = network['id']
         self.assertEqual('ACTIVE', network['status'])
         # Verify network update
@@ -280,7 +269,7 @@
         network = self.create_network()
         net_id = network['id']
         self.addCleanup(test_utils.call_and_ignore_notfound_exc,
-                        self._delete_network, network)
+                        self.networks_client.delete_network, network['id'])
 
         # Find a cidr that is not in use yet and create a subnet with it
         subnet = self.create_subnet(network)
@@ -324,7 +313,7 @@
     @decorators.idempotent_id('3d3852eb-3009-49ec-97ac-5ce83b73010a')
     def test_update_subnet_gw_dns_host_routes_dhcp(self):
         network = self.create_network()
-        self.addCleanup(self._delete_network, network)
+        self.addCleanup(self.networks_client.delete_network, network['id'])
 
         subnet = self.create_subnet(
             network, **self.subnet_dict(['gateway', 'host_routes',
@@ -622,7 +611,6 @@
         port = self.create_port(slaac_network)
         self.assertIsNotNone(port['fixed_ips'][0]['ip_address'])
         self.subnets_client.delete_subnet(subnet_slaac['id'])
-        self.subnets.pop()
         subnets = self.subnets_client.list_subnets()
         subnet_ids = [subnet['id'] for subnet in subnets['subnets']]
         self.assertNotIn(subnet_slaac['id'], subnet_ids,
diff --git a/tempest/api/network/test_routers_negative.py b/tempest/api/network/test_routers_negative.py
index c9ce55c..ddd7d3a 100644
--- a/tempest/api/network/test_routers_negative.py
+++ b/tempest/api/network/test_routers_negative.py
@@ -84,6 +84,8 @@
     def test_router_remove_interface_in_use_returns_409(self):
         self.routers_client.add_router_interface(self.router['id'],
                                                  subnet_id=self.subnet['id'])
+        self.addCleanup(self.routers_client.remove_router_interface,
+                        self.router['id'], subnet_id=self.subnet['id'])
         self.assertRaises(lib_exc.Conflict,
                           self.routers_client.delete_router,
                           self.router['id'])
diff --git a/tempest/api/object_storage/test_container_sync.py b/tempest/api/object_storage/test_container_sync.py
index 042d288..2e979bc 100644
--- a/tempest/api/object_storage/test_container_sync.py
+++ b/tempest/api/object_storage/test_container_sync.py
@@ -33,8 +33,6 @@
 
 
 class ContainerSyncTest(base.BaseObjectTest):
-    clients = {}
-
     credentials = [['operator', CONF.object_storage.operator_role],
                    ['operator_alt', CONF.object_storage.operator_role]]
 
@@ -54,6 +52,7 @@
         super(ContainerSyncTest, cls).resource_setup()
         cls.containers = []
         cls.objects = []
+        cls.clients = {}
 
         # Default container-server config only allows localhost
         cls.local_ip = '127.0.0.1'
diff --git a/tempest/api/volume/test_volumes_list.py b/tempest/api/volume/test_volumes_list.py
index b5f98ea..d5358ab 100644
--- a/tempest/api/volume/test_volumes_list.py
+++ b/tempest/api/volume/test_volumes_list.py
@@ -26,15 +26,28 @@
 
 
 class VolumesListTestJSON(base.BaseVolumeTest):
-    # NOTE: This test creates a number of 1G volumes. To run successfully,
-    # ensure that the backing file for the volume group that Nova uses
+    # NOTE: This test creates a number of 1G volumes. To run it successfully,
+    # ensure that the backing file for the volume group that Cinder uses
     # has space for at least 3 1G volumes!
     # If you are running a Devstack environment, ensure that the
     # VOLUME_BACKING_FILE_SIZE is at least 4G in your localrc
 
     VOLUME_FIELDS = ('id', 'name')
 
-    def assertVolumesIn(self, fetched_list, expected_list, fields=None):
+    @classmethod
+    def _remove_volatile_fields(cls, fetched_list):
+        """Remove fields that should not be compared.
+
+        This method makes sure that Tempest does not compare e.g.
+        the volume's "updated_at" field that may change for any reason
+        internal to the operation of Cinder.
+        """
+        for volume in fetched_list:
+            for field in ('updated_at',):
+                if field in volume:
+                    del volume[field]
+
+    def _assert_volumes_in(self, fetched_list, expected_list, fields=None):
         """Check out the list.
 
         This function is aim at check out whether all of the volumes in
@@ -45,6 +58,8 @@
             expected_list = map(fieldsgetter, expected_list)
             fetched_list = [fieldsgetter(item) for item in fetched_list]
 
+        # Hopefully the expected_list has already been cleaned.
+        self._remove_volatile_fields(fetched_list)
         missing_vols = [v for v in expected_list if v not in fetched_list]
         if not missing_vols:
             return
@@ -72,6 +87,7 @@
             volume = cls.volumes_client.show_volume(volume['id'])['volume']
             cls.volume_list.append(volume)
             cls.volume_id_list.append(volume['id'])
+        cls._remove_volatile_fields(cls.volume_list)
 
     def _list_by_param_value_and_assert(self, params, with_detail=False):
         """list or list_details with given params and validates result"""
@@ -103,15 +119,15 @@
         # Get a list of Volumes
         # Fetch all volumes
         fetched_list = self.volumes_client.list_volumes()['volumes']
-        self.assertVolumesIn(fetched_list, self.volume_list,
-                             fields=self.VOLUME_FIELDS)
+        self._assert_volumes_in(fetched_list, self.volume_list,
+                                fields=self.VOLUME_FIELDS)
 
     @decorators.idempotent_id('adcbb5a7-5ad8-4b61-bd10-5380e111a877')
     def test_volume_list_with_details(self):
         # Get a list of Volumes with details
         # Fetch all Volumes
         fetched_list = self.volumes_client.list_volumes(detail=True)['volumes']
-        self.assertVolumesIn(fetched_list, self.volume_list)
+        self._assert_volumes_in(fetched_list, self.volume_list)
 
     @decorators.idempotent_id('a28e8da4-0b56-472f-87a8-0f4d3f819c02')
     def test_volume_list_by_name(self):
@@ -137,8 +153,8 @@
         fetched_list = self.volumes_client.list_volumes(
             params=params)['volumes']
         self._list_by_param_value_and_assert(params)
-        self.assertVolumesIn(fetched_list, self.volume_list,
-                             fields=self.VOLUME_FIELDS)
+        self._assert_volumes_in(fetched_list, self.volume_list,
+                                fields=self.VOLUME_FIELDS)
 
     @decorators.idempotent_id('2943f712-71ec-482a-bf49-d5ca06216b9f')
     def test_volumes_list_details_by_status(self):
@@ -147,7 +163,7 @@
             detail=True, params=params)['volumes']
         for volume in fetched_list:
             self.assertEqual('available', volume['status'])
-        self.assertVolumesIn(fetched_list, self.volume_list)
+        self._assert_volumes_in(fetched_list, self.volume_list)
 
     @decorators.idempotent_id('2016a942-3020-40d7-95ce-7613bf8407ce')
     def test_volumes_list_by_bootable(self):
@@ -160,8 +176,8 @@
         fetched_list = self.volumes_client.list_volumes(
             params=params)['volumes']
         self._list_by_param_value_and_assert(params)
-        self.assertVolumesIn(fetched_list, self.volume_list,
-                             fields=self.VOLUME_FIELDS)
+        self._assert_volumes_in(fetched_list, self.volume_list,
+                                fields=self.VOLUME_FIELDS)
 
     @decorators.idempotent_id('2016a939-72ec-482a-bf49-d5ca06216b9f')
     def test_volumes_list_details_by_bootable(self):
@@ -170,7 +186,7 @@
             detail=True, params=params)['volumes']
         for volume in fetched_list:
             self.assertEqual('false', volume['bootable'])
-        self.assertVolumesIn(fetched_list, self.volume_list)
+        self._assert_volumes_in(fetched_list, self.volume_list)
 
     @decorators.idempotent_id('c0cfa863-3020-40d7-b587-e35f597d5d87')
     def test_volumes_list_by_availability_zone(self):
@@ -180,8 +196,8 @@
         fetched_list = self.volumes_client.list_volumes(
             params=params)['volumes']
         self._list_by_param_value_and_assert(params)
-        self.assertVolumesIn(fetched_list, self.volume_list,
-                             fields=self.VOLUME_FIELDS)
+        self._assert_volumes_in(fetched_list, self.volume_list,
+                                fields=self.VOLUME_FIELDS)
 
     @decorators.idempotent_id('e1b80d13-94f0-4ba2-a40e-386af29f8db1')
     def test_volumes_list_details_by_availability_zone(self):
@@ -192,7 +208,7 @@
             detail=True, params=params)['volumes']
         for volume in fetched_list:
             self.assertEqual(zone, volume['availability_zone'])
-        self.assertVolumesIn(fetched_list, self.volume_list)
+        self._assert_volumes_in(fetched_list, self.volume_list)
 
     @decorators.idempotent_id('b5ebea1b-0603-40a0-bb41-15fcd0a53214')
     def test_volume_list_with_param_metadata(self):
diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py
index 0d847bd..6435717 100644
--- a/tempest/cmd/run.py
+++ b/tempest/cmd/run.py
@@ -149,7 +149,7 @@
         discover_path = os.path.join(top_level_path, 'test_discover')
         file_contents = init.TESTR_CONF % (top_level_path, discover_path)
         with open('.testr.conf', 'w+') as testr_conf_file:
-                testr_conf_file.write(file_contents)
+            testr_conf_file.write(file_contents)
 
     def take_action(self, parsed_args):
         returncode = 0
diff --git a/tempest/lib/services/network/metering_label_rules_client.py b/tempest/lib/services/network/metering_label_rules_client.py
index 36cf8e3..9542e8f 100644
--- a/tempest/lib/services/network/metering_label_rules_client.py
+++ b/tempest/lib/services/network/metering_label_rules_client.py
@@ -16,6 +16,12 @@
 class MeteringLabelRulesClient(base.BaseNetworkClient):
 
     def create_metering_label_rule(self, **kwargs):
+        """Create metering label rule.
+
+        For a full list of available parameters, please refer to the official
+        API reference:
+        https://developer.openstack.org/api-ref/network/v2/index.html#create-metering-label-rule
+        """
         uri = '/metering/metering-label-rules'
         post_data = {'metering_label_rule': kwargs}
         return self.create_resource(uri, post_data)
@@ -29,5 +35,11 @@
         return self.delete_resource(uri)
 
     def list_metering_label_rules(self, **filters):
+        """List metering label rules.
+
+        For a full list of available parameters, please refer to the official
+        API reference:
+        https://developer.openstack.org/api-ref/network/v2/index.html#list-metering-label-rules
+        """
         uri = '/metering/metering-label-rules'
         return self.list_resources(uri, **filters)
diff --git a/tempest/lib/services/network/quotas_client.py b/tempest/lib/services/network/quotas_client.py
index 752b253..fdd3d6b 100644
--- a/tempest/lib/services/network/quotas_client.py
+++ b/tempest/lib/services/network/quotas_client.py
@@ -18,6 +18,12 @@
 class QuotasClient(base.BaseNetworkClient):
 
     def update_quotas(self, tenant_id, **kwargs):
+        """Update quota for a project.
+
+        For a full list of available parameters, please refer to the official
+        API reference:
+        https://developer.openstack.org/api-ref/network/v2/index.html#update-quota-for-a-project
+        """
         put_body = {'quota': kwargs}
         uri = '/quotas/%s' % tenant_id
         return self.update_resource(uri, put_body)
diff --git a/tempest/lib/services/network/service_providers_client.py b/tempest/lib/services/network/service_providers_client.py
index 0ee9bc3..01313a0 100644
--- a/tempest/lib/services/network/service_providers_client.py
+++ b/tempest/lib/services/network/service_providers_client.py
@@ -16,6 +16,11 @@
 class ServiceProvidersClient(base.BaseNetworkClient):
 
     def list_service_providers(self, **filters):
-        """Lists service providers."""
+        """Lists service providers.
+
+        For a full list of available parameters, please refer to the official
+        API reference:
+        https://developer.openstack.org/api-ref/network/v2/index.html#list-service-providers
+        """
         uri = '/service-providers'
         return self.list_resources(uri, **filters)
diff --git a/tempest/lib/services/object_storage/account_client.py b/tempest/lib/services/object_storage/account_client.py
index 67f01a6..6b097c1 100644
--- a/tempest/lib/services/object_storage/account_client.py
+++ b/tempest/lib/services/object_storage/account_client.py
@@ -34,7 +34,7 @@
         Account Metadata can be created, updated or deleted based on
         metadata header or value. For detailed info, please refer to the
         official API reference:
-        http://developer.openstack.org/api-ref/object-storage/?expanded=create-update-or-delete-account-metadata-detail
+        https://developer.openstack.org/api-ref/object-store/#create-update-or-delete-account-metadata
         """
         headers = {}
         if create_update_metadata:
diff --git a/tempest/lib/services/object_storage/container_client.py b/tempest/lib/services/object_storage/container_client.py
index 2da8e24..430e0d4 100644
--- a/tempest/lib/services/object_storage/container_client.py
+++ b/tempest/lib/services/object_storage/container_client.py
@@ -97,7 +97,7 @@
 
         For a full list of available parameters, please refer to the official
         API reference:
-        https://developer.openstack.org/api-ref/object-storage/?expanded=show-container-details-and-list-objects-detail
+        https://developer.openstack.org/api-ref/object-store/#show-container-details-and-list-objects
         """
 
         url = str(container_name)
diff --git a/tempest/tests/fake_config.py b/tempest/tests/fake_config.py
index ee63684..f1d3bba 100644
--- a/tempest/tests/fake_config.py
+++ b/tempest/tests/fake_config.py
@@ -40,10 +40,12 @@
                               group='identity')
         self.conf.set_default('neutron', True, group='service_available')
         self.conf.set_default('heat', True, group='service_available')
-        if not os.path.exists(str(os.environ.get('OS_TEST_LOCK_PATH'))):
-            os.mkdir(str(os.environ.get('OS_TEST_LOCK_PATH')))
+        lock_path = str(os.environ.get('OS_TEST_LOCK_PATH',
+                                       os.environ.get('TMPDIR', '/tmp')))
+        if not os.path.exists(lock_path):
+            os.mkdir(lock_path)
         lockutils.set_defaults(
-            lock_path=str(os.environ.get('OS_TEST_LOCK_PATH')),
+            lock_path=lock_path,
         )
         self.conf.set_default('auth_version', 'v2', group='identity')
         for config_option in ['username', 'password', 'project_name']: