Merge "Use addClassResourceCleanup in test_container_sync"
diff --git a/.zuul.yaml b/.zuul.yaml
index 9f91455..0588bc9 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -82,7 +82,7 @@
- openstack/neutron-vpnaas
- openstack/nova-lxd
- openstack/novajoin-tempest-plugin
- - openstack/octavia
+ - openstack/octavia-tempest-plugin
- openstack/oswin-tempest-plugin
- openstack/panko
- openstack/patrole
diff --git a/requirements.txt b/requirements.txt
index 2300214..cd74449 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,7 +12,7 @@
oslo.config>=5.1.0 # Apache-2.0
oslo.log>=3.30.0 # Apache-2.0
oslo.serialization!=2.19.1,>=2.18.0 # Apache-2.0
-oslo.utils>=3.31.0 # Apache-2.0
+oslo.utils>=3.33.0 # Apache-2.0
six>=1.10.0 # MIT
fixtures>=3.0.0 # Apache-2.0/BSD
PyYAML>=3.10 # MIT
diff --git a/roles/run-tempest/README.rst b/roles/run-tempest/README.rst
index 001586e..b5defb7 100644
--- a/roles/run-tempest/README.rst
+++ b/roles/run-tempest/README.rst
@@ -16,9 +16,19 @@
:default: ''
A regular expression used to select the tests.
+
It works only when used with some specific tox environments
('all', 'all-plugin'.)
+ Multi-line and commented regexs can be achieved by doing this:
+
+ ::
+ vars:
+ tempest_test_regex: |
+ (?x) # Ignore comments and whitespaces
+ # Line with only a comment.
+ (tempest\.(api|scenario|thirdparty)).*$ # Run only api scenario and third party
+
.. zuul:rolevar:: tox_venvlist
:default: smoke
diff --git a/tempest/api/compute/servers/test_server_actions.py b/tempest/api/compute/servers/test_server_actions.py
index e2be249..5c3e9f0 100644
--- a/tempest/api/compute/servers/test_server_actions.py
+++ b/tempest/api/compute/servers/test_server_actions.py
@@ -582,6 +582,12 @@
compute.shelve_server(self.client, self.server_id,
force_shelve_offload=True)
+ def _unshelve_server():
+ server_info = self.client.show_server(self.server_id)['server']
+ if 'SHELVED' in server_info['status']:
+ self.client.unshelve_server(self.server_id)
+ self.addOnException(_unshelve_server)
+
server = self.client.show_server(self.server_id)['server']
image_name = server['name'] + '-shelved'
params = {'name': image_name}
diff --git a/tempest/api/compute/servers/test_servers_negative.py b/tempest/api/compute/servers/test_servers_negative.py
index d067bb3..9b545af 100644
--- a/tempest/api/compute/servers/test_servers_negative.py
+++ b/tempest/api/compute/servers/test_servers_negative.py
@@ -477,6 +477,12 @@
# shelve a shelved server.
compute.shelve_server(self.client, self.server_id)
+ def _unshelve_server():
+ server_info = self.client.show_server(self.server_id)['server']
+ if 'SHELVED' in server_info['status']:
+ self.client.unshelve_server(self.server_id)
+ self.addOnException(_unshelve_server)
+
server = self.client.show_server(self.server_id)['server']
image_name = server['name'] + '-shelved'
params = {'name': image_name}
diff --git a/tempest/api/network/admin/test_external_network_extension.py b/tempest/api/network/admin/test_external_network_extension.py
index 4d41e33..49a9cdb 100644
--- a/tempest/api/network/admin/test_external_network_extension.py
+++ b/tempest/api/network/admin/test_external_network_extension.py
@@ -130,5 +130,3 @@
subnet_list = self.admin_subnets_client.list_subnets()
self.assertNotIn(subnet['id'],
(s['id'] for s in subnet_list))
- # Removes subnet from the cleanup list
- self.subnets.remove(subnet)
diff --git a/tempest/api/network/base.py b/tempest/api/network/base.py
index bdfda0a..8670165 100644
--- a/tempest/api/network/base.py
+++ b/tempest/api/network/base.py
@@ -88,7 +88,6 @@
@classmethod
def resource_setup(cls):
super(BaseNetworkTest, cls).resource_setup()
- cls.networks = []
cls.subnets = []
cls.ports = []
cls.routers = []
@@ -101,27 +100,6 @@
cls.mask_bits = CONF.network.project_network_v6_mask_bits
@classmethod
- def resource_cleanup(cls):
- if CONF.service_available.neutron:
- # Clean up ports
- for port in cls.ports:
- test_utils.call_and_ignore_notfound_exc(
- cls.ports_client.delete_port, port['id'])
- # Clean up routers
- for router in cls.routers:
- test_utils.call_and_ignore_notfound_exc(
- cls.delete_router, router)
- # Clean up subnets
- for subnet in cls.subnets:
- test_utils.call_and_ignore_notfound_exc(
- cls.subnets_client.delete_subnet, subnet['id'])
- # Clean up networks
- for network in cls.networks:
- test_utils.call_and_ignore_notfound_exc(
- cls.networks_client.delete_network, network['id'])
- super(BaseNetworkTest, cls).resource_cleanup()
-
- @classmethod
def create_network(cls, network_name=None, **kwargs):
"""Wrapper utility that returns a test network."""
network_name = network_name or data_utils.rand_name(
@@ -129,7 +107,9 @@
body = cls.networks_client.create_network(name=network_name, **kwargs)
network = body['network']
- cls.networks.append(network)
+ cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+ cls.networks_client.delete_network,
+ network['id'])
return network
@classmethod
@@ -172,6 +152,9 @@
message = 'Available CIDR for subnet creation could not be found'
raise exceptions.BuildErrorException(message)
subnet = body['subnet']
+ cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+ cls.subnets_client.delete_subnet,
+ subnet['id'])
cls.subnets.append(subnet)
return subnet
@@ -181,6 +164,8 @@
body = cls.ports_client.create_port(network_id=network['id'],
**kwargs)
port = body['port']
+ cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+ cls.ports_client.delete_port, port['id'])
cls.ports.append(port)
return port
@@ -207,6 +192,8 @@
name=router_name, external_gateway_info=ext_gw_info,
admin_state_up=admin_state_up, **kwargs)
router = body['router']
+ cls.addClassResourceCleanup(test_utils.call_and_ignore_notfound_exc,
+ cls.delete_router, router)
cls.routers.append(router)
return router
diff --git a/tempest/api/network/test_networks.py b/tempest/api/network/test_networks.py
index 1c59556..7345fd1 100644
--- a/tempest/api/network/test_networks.py
+++ b/tempest/api/network/test_networks.py
@@ -104,15 +104,6 @@
self.assertThat(actual, custom_matchers.MatchesDictExceptForKeys(
expected, exclude_keys))
- def _delete_network(self, network):
- # Deleting network also deletes its subnets if exists
- self.networks_client.delete_network(network['id'])
- if network in self.networks:
- self.networks.remove(network)
- for subnet in self.subnets:
- if subnet['network_id'] == network['id']:
- self.subnets.remove(subnet)
-
def _create_verify_delete_subnet(self, cidr=None, mask_bits=None,
**kwargs):
network = self.create_network()
@@ -132,8 +123,6 @@
self._compare_resource_attrs(subnet, compare_args)
self.networks_client.delete_network(net_id)
- self.networks.pop()
- self.subnets.pop()
class NetworksTest(BaseNetworkTestResources):
@@ -171,7 +160,7 @@
def test_create_update_delete_network_subnet(self):
# Create a network
network = self.create_network()
- self.addCleanup(self._delete_network, network)
+ self.addCleanup(self.networks_client.delete_network, network['id'])
net_id = network['id']
self.assertEqual('ACTIVE', network['status'])
# Verify network update
@@ -280,7 +269,7 @@
network = self.create_network()
net_id = network['id']
self.addCleanup(test_utils.call_and_ignore_notfound_exc,
- self._delete_network, network)
+ self.networks_client.delete_network, network['id'])
# Find a cidr that is not in use yet and create a subnet with it
subnet = self.create_subnet(network)
@@ -324,7 +313,7 @@
@decorators.idempotent_id('3d3852eb-3009-49ec-97ac-5ce83b73010a')
def test_update_subnet_gw_dns_host_routes_dhcp(self):
network = self.create_network()
- self.addCleanup(self._delete_network, network)
+ self.addCleanup(self.networks_client.delete_network, network['id'])
subnet = self.create_subnet(
network, **self.subnet_dict(['gateway', 'host_routes',
@@ -622,7 +611,6 @@
port = self.create_port(slaac_network)
self.assertIsNotNone(port['fixed_ips'][0]['ip_address'])
self.subnets_client.delete_subnet(subnet_slaac['id'])
- self.subnets.pop()
subnets = self.subnets_client.list_subnets()
subnet_ids = [subnet['id'] for subnet in subnets['subnets']]
self.assertNotIn(subnet_slaac['id'], subnet_ids,
diff --git a/tempest/api/network/test_routers_negative.py b/tempest/api/network/test_routers_negative.py
index c9ce55c..ddd7d3a 100644
--- a/tempest/api/network/test_routers_negative.py
+++ b/tempest/api/network/test_routers_negative.py
@@ -84,6 +84,8 @@
def test_router_remove_interface_in_use_returns_409(self):
self.routers_client.add_router_interface(self.router['id'],
subnet_id=self.subnet['id'])
+ self.addCleanup(self.routers_client.remove_router_interface,
+ self.router['id'], subnet_id=self.subnet['id'])
self.assertRaises(lib_exc.Conflict,
self.routers_client.delete_router,
self.router['id'])
diff --git a/tempest/api/object_storage/base.py b/tempest/api/object_storage/base.py
index ee72163..e8f3f8b 100644
--- a/tempest/api/object_storage/base.py
+++ b/tempest/api/object_storage/base.py
@@ -36,10 +36,14 @@
using HA proxy sync the deletion properly, otherwise, the container
might fail to be deleted because it's not empty.
- :param containers: List of containers to be deleted
+ :param containers: List of containers(or string of a container)
+ to be deleted
:param container_client: Client to be used to delete containers
:param object_client: Client to be used to delete objects
"""
+ if isinstance(containers, str):
+ containers = [containers]
+
for cont in containers:
try:
params = {'limit': 9999, 'format': 'json'}
diff --git a/tempest/api/object_storage/test_container_sync.py b/tempest/api/object_storage/test_container_sync.py
index 0477d91..322579c 100644
--- a/tempest/api/object_storage/test_container_sync.py
+++ b/tempest/api/object_storage/test_container_sync.py
@@ -33,8 +33,6 @@
class ContainerSyncTest(base.BaseObjectTest):
- clients = {}
-
credentials = [['operator', CONF.object_storage.operator_role],
['operator_alt', CONF.object_storage.operator_role]]
@@ -54,6 +52,7 @@
super(ContainerSyncTest, cls).resource_setup()
cls.containers = []
cls.objects = []
+ cls.clients = {}
# Default container-server config only allows localhost
cls.local_ip = '127.0.0.1'
diff --git a/tempest/api/volume/admin/test_snapshot_manage.py b/tempest/api/volume/admin/test_snapshot_manage.py
index 3b21b28..37a47ec 100644
--- a/tempest/api/volume/admin/test_snapshot_manage.py
+++ b/tempest/api/volume/admin/test_snapshot_manage.py
@@ -63,7 +63,7 @@
# Verify the original snapshot does not exist in snapshot list
params = {'all_tenants': 1}
all_snapshots = self.admin_snapshots_client.list_snapshots(
- detail=True, params=params)['snapshots']
+ detail=True, **params)['snapshots']
self.assertNotIn(snapshot['id'], [v['id'] for v in all_snapshots])
# Manage the snapshot
diff --git a/tempest/api/volume/test_volumes_list.py b/tempest/api/volume/test_volumes_list.py
index b5f98ea..d5358ab 100644
--- a/tempest/api/volume/test_volumes_list.py
+++ b/tempest/api/volume/test_volumes_list.py
@@ -26,15 +26,28 @@
class VolumesListTestJSON(base.BaseVolumeTest):
- # NOTE: This test creates a number of 1G volumes. To run successfully,
- # ensure that the backing file for the volume group that Nova uses
+ # NOTE: This test creates a number of 1G volumes. To run it successfully,
+ # ensure that the backing file for the volume group that Cinder uses
# has space for at least 3 1G volumes!
# If you are running a Devstack environment, ensure that the
# VOLUME_BACKING_FILE_SIZE is at least 4G in your localrc
VOLUME_FIELDS = ('id', 'name')
- def assertVolumesIn(self, fetched_list, expected_list, fields=None):
+ @classmethod
+ def _remove_volatile_fields(cls, fetched_list):
+ """Remove fields that should not be compared.
+
+ This method makes sure that Tempest does not compare e.g.
+ the volume's "updated_at" field that may change for any reason
+ internal to the operation of Cinder.
+ """
+ for volume in fetched_list:
+ for field in ('updated_at',):
+ if field in volume:
+ del volume[field]
+
+ def _assert_volumes_in(self, fetched_list, expected_list, fields=None):
"""Check out the list.
This function is aim at check out whether all of the volumes in
@@ -45,6 +58,8 @@
expected_list = map(fieldsgetter, expected_list)
fetched_list = [fieldsgetter(item) for item in fetched_list]
+ # Hopefully the expected_list has already been cleaned.
+ self._remove_volatile_fields(fetched_list)
missing_vols = [v for v in expected_list if v not in fetched_list]
if not missing_vols:
return
@@ -72,6 +87,7 @@
volume = cls.volumes_client.show_volume(volume['id'])['volume']
cls.volume_list.append(volume)
cls.volume_id_list.append(volume['id'])
+ cls._remove_volatile_fields(cls.volume_list)
def _list_by_param_value_and_assert(self, params, with_detail=False):
"""list or list_details with given params and validates result"""
@@ -103,15 +119,15 @@
# Get a list of Volumes
# Fetch all volumes
fetched_list = self.volumes_client.list_volumes()['volumes']
- self.assertVolumesIn(fetched_list, self.volume_list,
- fields=self.VOLUME_FIELDS)
+ self._assert_volumes_in(fetched_list, self.volume_list,
+ fields=self.VOLUME_FIELDS)
@decorators.idempotent_id('adcbb5a7-5ad8-4b61-bd10-5380e111a877')
def test_volume_list_with_details(self):
# Get a list of Volumes with details
# Fetch all Volumes
fetched_list = self.volumes_client.list_volumes(detail=True)['volumes']
- self.assertVolumesIn(fetched_list, self.volume_list)
+ self._assert_volumes_in(fetched_list, self.volume_list)
@decorators.idempotent_id('a28e8da4-0b56-472f-87a8-0f4d3f819c02')
def test_volume_list_by_name(self):
@@ -137,8 +153,8 @@
fetched_list = self.volumes_client.list_volumes(
params=params)['volumes']
self._list_by_param_value_and_assert(params)
- self.assertVolumesIn(fetched_list, self.volume_list,
- fields=self.VOLUME_FIELDS)
+ self._assert_volumes_in(fetched_list, self.volume_list,
+ fields=self.VOLUME_FIELDS)
@decorators.idempotent_id('2943f712-71ec-482a-bf49-d5ca06216b9f')
def test_volumes_list_details_by_status(self):
@@ -147,7 +163,7 @@
detail=True, params=params)['volumes']
for volume in fetched_list:
self.assertEqual('available', volume['status'])
- self.assertVolumesIn(fetched_list, self.volume_list)
+ self._assert_volumes_in(fetched_list, self.volume_list)
@decorators.idempotent_id('2016a942-3020-40d7-95ce-7613bf8407ce')
def test_volumes_list_by_bootable(self):
@@ -160,8 +176,8 @@
fetched_list = self.volumes_client.list_volumes(
params=params)['volumes']
self._list_by_param_value_and_assert(params)
- self.assertVolumesIn(fetched_list, self.volume_list,
- fields=self.VOLUME_FIELDS)
+ self._assert_volumes_in(fetched_list, self.volume_list,
+ fields=self.VOLUME_FIELDS)
@decorators.idempotent_id('2016a939-72ec-482a-bf49-d5ca06216b9f')
def test_volumes_list_details_by_bootable(self):
@@ -170,7 +186,7 @@
detail=True, params=params)['volumes']
for volume in fetched_list:
self.assertEqual('false', volume['bootable'])
- self.assertVolumesIn(fetched_list, self.volume_list)
+ self._assert_volumes_in(fetched_list, self.volume_list)
@decorators.idempotent_id('c0cfa863-3020-40d7-b587-e35f597d5d87')
def test_volumes_list_by_availability_zone(self):
@@ -180,8 +196,8 @@
fetched_list = self.volumes_client.list_volumes(
params=params)['volumes']
self._list_by_param_value_and_assert(params)
- self.assertVolumesIn(fetched_list, self.volume_list,
- fields=self.VOLUME_FIELDS)
+ self._assert_volumes_in(fetched_list, self.volume_list,
+ fields=self.VOLUME_FIELDS)
@decorators.idempotent_id('e1b80d13-94f0-4ba2-a40e-386af29f8db1')
def test_volumes_list_details_by_availability_zone(self):
@@ -192,7 +208,7 @@
detail=True, params=params)['volumes']
for volume in fetched_list:
self.assertEqual(zone, volume['availability_zone'])
- self.assertVolumesIn(fetched_list, self.volume_list)
+ self._assert_volumes_in(fetched_list, self.volume_list)
@decorators.idempotent_id('b5ebea1b-0603-40a0-bb41-15fcd0a53214')
def test_volume_list_with_param_metadata(self):
diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py
index 0d847bd..6435717 100644
--- a/tempest/cmd/run.py
+++ b/tempest/cmd/run.py
@@ -149,7 +149,7 @@
discover_path = os.path.join(top_level_path, 'test_discover')
file_contents = init.TESTR_CONF % (top_level_path, discover_path)
with open('.testr.conf', 'w+') as testr_conf_file:
- testr_conf_file.write(file_contents)
+ testr_conf_file.write(file_contents)
def take_action(self, parsed_args):
returncode = 0
diff --git a/tempest/lib/common/utils/linux/remote_client.py b/tempest/lib/common/utils/linux/remote_client.py
index cd4092b..1676a28 100644
--- a/tempest/lib/common/utils/linux/remote_client.py
+++ b/tempest/lib/common/utils/linux/remote_client.py
@@ -31,7 +31,7 @@
except Exception as e:
caller = test_utils.find_test_caller() or "not found"
if not isinstance(e, tempest.lib.exceptions.SSHTimeout):
- message = ('Initializing SSH connection to %(ip)s failed. '
+ message = ('Executing command on %(ip)s failed. '
'Error: %(error)s' % {'ip': self.ip_address,
'error': e})
message = '(%s) %s' % (caller, message)
diff --git a/tempest/lib/services/network/metering_label_rules_client.py b/tempest/lib/services/network/metering_label_rules_client.py
index 36cf8e3..9542e8f 100644
--- a/tempest/lib/services/network/metering_label_rules_client.py
+++ b/tempest/lib/services/network/metering_label_rules_client.py
@@ -16,6 +16,12 @@
class MeteringLabelRulesClient(base.BaseNetworkClient):
def create_metering_label_rule(self, **kwargs):
+ """Create metering label rule.
+
+ For a full list of available parameters, please refer to the official
+ API reference:
+ https://developer.openstack.org/api-ref/network/v2/index.html#create-metering-label-rule
+ """
uri = '/metering/metering-label-rules'
post_data = {'metering_label_rule': kwargs}
return self.create_resource(uri, post_data)
@@ -29,5 +35,11 @@
return self.delete_resource(uri)
def list_metering_label_rules(self, **filters):
+ """List metering label rules.
+
+ For a full list of available parameters, please refer to the official
+ API reference:
+ https://developer.openstack.org/api-ref/network/v2/index.html#list-metering-label-rules
+ """
uri = '/metering/metering-label-rules'
return self.list_resources(uri, **filters)
diff --git a/tempest/lib/services/network/quotas_client.py b/tempest/lib/services/network/quotas_client.py
index 752b253..fdd3d6b 100644
--- a/tempest/lib/services/network/quotas_client.py
+++ b/tempest/lib/services/network/quotas_client.py
@@ -18,6 +18,12 @@
class QuotasClient(base.BaseNetworkClient):
def update_quotas(self, tenant_id, **kwargs):
+ """Update quota for a project.
+
+ For a full list of available parameters, please refer to the official
+ API reference:
+ https://developer.openstack.org/api-ref/network/v2/index.html#update-quota-for-a-project
+ """
put_body = {'quota': kwargs}
uri = '/quotas/%s' % tenant_id
return self.update_resource(uri, put_body)
diff --git a/tempest/lib/services/network/service_providers_client.py b/tempest/lib/services/network/service_providers_client.py
index 0ee9bc3..01313a0 100644
--- a/tempest/lib/services/network/service_providers_client.py
+++ b/tempest/lib/services/network/service_providers_client.py
@@ -16,6 +16,11 @@
class ServiceProvidersClient(base.BaseNetworkClient):
def list_service_providers(self, **filters):
- """Lists service providers."""
+ """Lists service providers.
+
+ For a full list of available parameters, please refer to the official
+ API reference:
+ https://developer.openstack.org/api-ref/network/v2/index.html#list-service-providers
+ """
uri = '/service-providers'
return self.list_resources(uri, **filters)
diff --git a/tempest/lib/services/object_storage/account_client.py b/tempest/lib/services/object_storage/account_client.py
index 67f01a6..6b097c1 100644
--- a/tempest/lib/services/object_storage/account_client.py
+++ b/tempest/lib/services/object_storage/account_client.py
@@ -34,7 +34,7 @@
Account Metadata can be created, updated or deleted based on
metadata header or value. For detailed info, please refer to the
official API reference:
- http://developer.openstack.org/api-ref/object-storage/?expanded=create-update-or-delete-account-metadata-detail
+ https://developer.openstack.org/api-ref/object-store/#create-update-or-delete-account-metadata
"""
headers = {}
if create_update_metadata:
diff --git a/tempest/lib/services/object_storage/container_client.py b/tempest/lib/services/object_storage/container_client.py
index 2da8e24..430e0d4 100644
--- a/tempest/lib/services/object_storage/container_client.py
+++ b/tempest/lib/services/object_storage/container_client.py
@@ -97,7 +97,7 @@
For a full list of available parameters, please refer to the official
API reference:
- https://developer.openstack.org/api-ref/object-storage/?expanded=show-container-details-and-list-objects-detail
+ https://developer.openstack.org/api-ref/object-store/#show-container-details-and-list-objects
"""
url = str(container_name)
diff --git a/tempest/tests/fake_config.py b/tempest/tests/fake_config.py
index ee63684..f1d3bba 100644
--- a/tempest/tests/fake_config.py
+++ b/tempest/tests/fake_config.py
@@ -40,10 +40,12 @@
group='identity')
self.conf.set_default('neutron', True, group='service_available')
self.conf.set_default('heat', True, group='service_available')
- if not os.path.exists(str(os.environ.get('OS_TEST_LOCK_PATH'))):
- os.mkdir(str(os.environ.get('OS_TEST_LOCK_PATH')))
+ lock_path = str(os.environ.get('OS_TEST_LOCK_PATH',
+ os.environ.get('TMPDIR', '/tmp')))
+ if not os.path.exists(lock_path):
+ os.mkdir(lock_path)
lockutils.set_defaults(
- lock_path=str(os.environ.get('OS_TEST_LOCK_PATH')),
+ lock_path=lock_path,
)
self.conf.set_default('auth_version', 'v2', group='identity')
for config_option in ['username', 'password', 'project_name']:
diff --git a/tools/find_stack_traces.py b/tools/find_stack_traces.py
deleted file mode 100755
index 1f2b88b..0000000
--- a/tools/find_stack_traces.py
+++ /dev/null
@@ -1,160 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2013 IBM Corp.
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-import gzip
-import pprint
-import re
-import sys
-
-import six
-import six.moves.urllib.request as urlreq
-
-
-pp = pprint.PrettyPrinter()
-
-NOVA_TIMESTAMP = r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d\.\d\d\d"
-
-NOVA_REGEX = r"(?P<timestamp>%s) (?P<pid>\d+ )?(?P<level>(ERROR|TRACE)) " \
- "(?P<module>[\w\.]+) (?P<msg>.*)" % (NOVA_TIMESTAMP)
-
-
-class StackTrace(object):
- timestamp = None
- pid = None
- level = ""
- module = ""
- msg = ""
-
- def __init__(self, timestamp=None, pid=None, level="", module="",
- msg=""):
- self.timestamp = timestamp
- self.pid = pid
- self.level = level
- self.module = module
- self.msg = msg
-
- def append(self, msg):
- self.msg = self.msg + msg
-
- def is_same(self, data):
- return (data['timestamp'] == self.timestamp and
- data['level'] == self.level)
-
- def not_none(self):
- return self.timestamp is not None
-
- def __str__(self):
- buff = "<%s %s %s>\n" % (self.timestamp, self.level, self.module)
- for line in self.msg.splitlines():
- buff = buff + line + "\n"
- return buff
-
-
-def hunt_for_stacktrace(url):
- """Return TRACE or ERROR lines out of logs."""
- req = urlreq.Request(url)
- req.add_header('Accept-Encoding', 'gzip')
- page = urlreq.urlopen(req)
- buf = six.StringIO(page.read())
- f = gzip.GzipFile(fileobj=buf)
- content = f.read()
-
- traces = []
- trace = StackTrace()
- for line in content.splitlines():
- m = re.match(NOVA_REGEX, line)
- if m:
- data = m.groupdict()
- if trace.not_none() and trace.is_same(data):
- trace.append(data['msg'] + "\n")
- else:
- trace = StackTrace(
- timestamp=data.get('timestamp'),
- pid=data.get('pid'),
- level=data.get('level'),
- module=data.get('module'),
- msg=data.get('msg'))
-
- else:
- if trace.not_none():
- traces.append(trace)
- trace = StackTrace()
-
- # once more at the end to pick up any stragglers
- if trace.not_none():
- traces.append(trace)
-
- return traces
-
-
-def log_url(url, log):
- return "%s/%s" % (url, log)
-
-
-def collect_logs(url):
- page = urlreq.urlopen(url)
- content = page.read()
- logs = re.findall('(screen-[\w-]+\.txt\.gz)</a>', content)
- return logs
-
-
-def usage():
- print("""
-Usage: find_stack_traces.py <logurl>
-
-Hunts for stack traces in a devstack run. Must provide it a base log url
-from a tempest devstack run. Should start with http and end with /logs/.
-
-Returns a report listing stack traces out of the various files where
-they are found.
-""")
- sys.exit(0)
-
-
-def print_stats(items, fname, verbose=False):
- errors = len([x for x in items if x.level == "ERROR"])
- traces = len([x for x in items if x.level == "TRACE"])
- print("%d ERRORS found in %s" % (errors, fname))
- print("%d TRACES found in %s" % (traces, fname))
-
- if verbose:
- for item in items:
- print(item)
- print("\n\n")
-
-
-def main():
- if len(sys.argv) == 2:
- url = sys.argv[1]
- loglist = collect_logs(url)
-
- # probably wrong base url
- if not loglist:
- usage()
-
- for log in loglist:
- logurl = log_url(url, log)
- traces = hunt_for_stacktrace(logurl)
-
- if traces:
- print_stats(traces, log, verbose=True)
-
- else:
- usage()
-
-if __name__ == '__main__':
- main()