Configure logging format flexibly
Now we can get tempest log file with options in run_tests.sh.
This adds log.py and enables to configure log format more flexibly
using configuration file.
This adds a LoggerAdaptor and a Formatter to output each test name
to log lines.
Implements: blueprint add-logging-configuration
Change-Id: I88cf18bb8bbc152e62ac83a9c7dc26067b7a11bd
diff --git a/etc/logging.conf.sample b/etc/logging.conf.sample
index 5c1ea5f..685dd36 100644
--- a/etc/logging.conf.sample
+++ b/etc/logging.conf.sample
@@ -1,30 +1,41 @@
[loggers]
-keys=root
-
-[formatters]
-keys=normal,debug
+keys=root,tempest
[handlers]
-keys=file,devel
+keys=file,syslog,devel
+
+[formatters]
+keys=default,tests
[logger_root]
level=NOTSET
+handlers=syslog
+
+[logger_tempest]
+level=DEBUG
handlers=file
+qualname=tempest
[handler_file]
class=FileHandler
level=DEBUG
-formatter=normal
+formatter=tests
args=('tempest.log', 'w')
+[handler_syslog]
+class=handlers.SysLogHandler
+level=ERROR
+formatter = default
+args = ('/dev/log', handlers.SysLogHandler.LOG_USER)
+
[handler_devel]
class=StreamHandler
level=DEBUG
-formatter=debug
+formatter=default
args=(sys.stdout,)
-[formatter_normal]
-format=%(asctime)s %(levelname)s %(message)s
+[formatter_default]
+format=%(name)s: %(levelname)s: %(message)s
-[formatter_debug]
-format=%(asctime)s %(levelname)s %(module)s %(funcName)s %(message)s
+[formatter_tests]
+class = tempest.common.log.TestsFormatter
diff --git a/run_tests.sh b/run_tests.sh
index 56a6e6e..d5b2494 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -75,6 +75,16 @@
export TEMPEST_CONFIG=`basename "$config_file"`
fi
+if [ $logging -eq 1 ]; then
+ if [ ! -f "$logging_config" ]; then
+ echo "No such logging config file: $logging_config"
+ exit 1
+ fi
+ logging_config=`readlink -f "$logging_config"`
+ export TEMPEST_LOG_CONFIG_DIR=`dirname "$logging_config"`
+ export TEMPEST_LOG_CONFIG=`basename "$logging_config"`
+fi
+
cd `dirname "$0"`
export NOSE_WITH_OPENSTACK=1
@@ -84,14 +94,6 @@
export NOSE_OPENSTACK_SHOW_ELAPSED=1
export NOSE_OPENSTACK_STDOUT=1
-if [ $logging -eq 1 ]; then
- if [ ! -f "$logging_config" ]; then
- echo "No such logging config file: $logging_config"
- exit
- fi
- noseargs="$noseargs --logging-config=$logging_config"
-fi
-
if [ $no_site_packages -eq 1 ]; then
installvenvopts="--no-site-packages"
fi
diff --git a/tempest/api/compute/__init__.py b/tempest/api/compute/__init__.py
index 6cd6f69..98f198d 100644
--- a/tempest/api/compute/__init__.py
+++ b/tempest/api/compute/__init__.py
@@ -15,10 +15,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
-
from tempest import clients
+from tempest.common import log as logging
from tempest import config
from tempest.exceptions import InvalidConfiguration
diff --git a/tempest/api/compute/base.py b/tempest/api/compute/base.py
index 5f0df3c..b19b9d9 100644
--- a/tempest/api/compute/base.py
+++ b/tempest/api/compute/base.py
@@ -15,11 +15,11 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import time
from tempest.api import compute
from tempest import clients
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
import tempest.test
diff --git a/tempest/api/identity/__init__.py b/tempest/api/identity/__init__.py
index e5fdc1b..718aa15 100644
--- a/tempest/api/identity/__init__.py
+++ b/tempest/api/identity/__init__.py
@@ -15,7 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
+from tempest.common import log as logging
LOG = logging.getLogger(__name__)
diff --git a/tempest/api/image/base.py b/tempest/api/image/base.py
index f12e957..e62d84b 100644
--- a/tempest/api/image/base.py
+++ b/tempest/api/image/base.py
@@ -14,9 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
from tempest import clients
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
import tempest.test
diff --git a/tempest/api/volume/admin/test_multi_backend.py b/tempest/api/volume/admin/test_multi_backend.py
index bbc1d97..2b2b1a1 100644
--- a/tempest/api/volume/admin/test_multi_backend.py
+++ b/tempest/api/volume/admin/test_multi_backend.py
@@ -15,10 +15,10 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import testtools
from tempest.api.volume import base
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest import config
from tempest.services.volume.json.admin import volume_types_client
diff --git a/tempest/api/volume/base.py b/tempest/api/volume/base.py
index 978ec53..2839da4 100644
--- a/tempest/api/volume/base.py
+++ b/tempest/api/volume/base.py
@@ -15,10 +15,10 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import time
from tempest import clients
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions
import tempest.test
diff --git a/tempest/api/volume/test_volumes_snapshots.py b/tempest/api/volume/test_volumes_snapshots.py
index c013ae4..3a0c802 100644
--- a/tempest/api/volume/test_volumes_snapshots.py
+++ b/tempest/api/volume/test_volumes_snapshots.py
@@ -12,10 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import testtools
from tempest.api.volume import base
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest.test import attr
diff --git a/tempest/clients.py b/tempest/clients.py
index f284f16..91d54eb 100644
--- a/tempest/clients.py
+++ b/tempest/clients.py
@@ -15,8 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
+from tempest.common import log as logging
from tempest import config
from tempest import exceptions
from tempest.services import botoclients
diff --git a/tempest/common/glance_http.py b/tempest/common/glance_http.py
index 4ddaf17..d19d216 100644
--- a/tempest/common/glance_http.py
+++ b/tempest/common/glance_http.py
@@ -19,7 +19,6 @@
import hashlib
import httplib
import json
-import logging
import posixpath
import re
import socket
@@ -35,6 +34,7 @@
import OpenSSL
+from tempest.common import log as logging
from tempest import exceptions as exc
diff --git a/tempest/common/log.py b/tempest/common/log.py
new file mode 100644
index 0000000..9b35723
--- /dev/null
+++ b/tempest/common/log.py
@@ -0,0 +1,116 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 NEC Corporation.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import ConfigParser
+import inspect
+import logging
+import logging.config
+import os
+import re
+
+from oslo.config import cfg
+
+
+_DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)8s [%(name)s] %(message)s"
+_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
+
+_loggers = {}
+
+
+def getLogger(name='unknown'):
+ if len(_loggers) == 0:
+ loaded = _load_log_config()
+ getLogger.adapter = TestsAdapter if loaded else None
+
+ if name not in _loggers:
+ logger = logging.getLogger(name)
+ if getLogger.adapter:
+ _loggers[name] = getLogger.adapter(logger, name)
+ else:
+ _loggers[name] = logger
+
+ return _loggers[name]
+
+
+def _load_log_config():
+ conf_dir = os.environ.get('TEMPEST_LOG_CONFIG_DIR', None)
+ conf_file = os.environ.get('TEMPEST_LOG_CONFIG', None)
+ if not conf_dir or not conf_file:
+ return False
+
+ log_config = os.path.join(conf_dir, conf_file)
+ try:
+ logging.config.fileConfig(log_config)
+ except ConfigParser.Error, exc:
+ raise cfg.ConfigFileParseError(log_config, str(exc))
+ return True
+
+
+class TestsAdapter(logging.LoggerAdapter):
+
+ def __init__(self, logger, project_name):
+ self.logger = logger
+ self.project = project_name
+ self.regexp = re.compile(r"test_\w+\.py")
+
+ def __getattr__(self, key):
+ return getattr(self.logger, key)
+
+ def _get_test_name(self):
+ frames = inspect.stack()
+ for frame in frames:
+ binary_name = frame[1]
+ if self.regexp.search(binary_name) and 'self' in frame[0].f_locals:
+ return frame[0].f_locals.get('self').id()
+ elif frame[3] == '_run_cleanups':
+ #NOTE(myamazaki): method calling addCleanup
+ return frame[0].f_locals.get('self').case.id()
+ elif frame[3] in ['setUpClass', 'tearDownClass']:
+ #NOTE(myamazaki): setUpClass or tearDownClass
+ return "%s.%s.%s" % (frame[0].f_locals['cls'].__module__,
+ frame[0].f_locals['cls'].__name__,
+ frame[3])
+ return None
+
+ def process(self, msg, kwargs):
+ if 'extra' not in kwargs:
+ kwargs['extra'] = {}
+ extra = kwargs['extra']
+
+ test_name = self._get_test_name()
+ if test_name:
+ extra.update({'testname': test_name})
+ extra['extra'] = extra.copy()
+
+ return msg, kwargs
+
+
+class TestsFormatter(logging.Formatter):
+ def __init__(self, fmt=None, datefmt=None):
+ super(TestsFormatter, self).__init__()
+ self.default_format = _DEFAULT_LOG_FORMAT
+ self.testname_format =\
+ "%(asctime)s %(levelname)8s [%(testname)s] %(message)s"
+ self.datefmt = _DEFAULT_LOG_DATE_FORMAT
+
+ def format(self, record):
+ extra = record.__dict__.get('extra', None)
+ if extra and 'testname' in extra:
+ self._fmt = self.testname_format
+ else:
+ self._fmt = self.default_format
+ return logging.Formatter.format(self, record)
diff --git a/tempest/common/rest_client.py b/tempest/common/rest_client.py
index fba3b0f..d81af83 100644
--- a/tempest/common/rest_client.py
+++ b/tempest/common/rest_client.py
@@ -19,11 +19,11 @@
import hashlib
import httplib2
import json
-import logging
from lxml import etree
import re
import time
+from tempest.common import log as logging
from tempest import exceptions
from tempest.services.compute.xml.common import xml_to_json
diff --git a/tempest/config.py b/tempest/config.py
index 6d6bc2b..8f3e574 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -15,12 +15,12 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import os
import sys
from oslo.config import cfg
+from tempest.common import log as logging
from tempest.common.utils.misc import singleton
LOG = logging.getLogger(__name__)
diff --git a/tempest/manager.py b/tempest/manager.py
index 047ad41..25e80ad 100644
--- a/tempest/manager.py
+++ b/tempest/manager.py
@@ -15,8 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
+from tempest.common import log as logging
import tempest.config
from tempest import exceptions
# Tempest REST Fuzz testing client libs
diff --git a/tempest/scenario/test_server_advanced_ops.py b/tempest/scenario/test_server_advanced_ops.py
index e48157e..9ac0cc0 100644
--- a/tempest/scenario/test_server_advanced_ops.py
+++ b/tempest/scenario/test_server_advanced_ops.py
@@ -15,9 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
-
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest.scenario import manager
diff --git a/tempest/scenario/test_server_basic_ops.py b/tempest/scenario/test_server_basic_ops.py
index c5c6728..d318dd9 100644
--- a/tempest/scenario/test_server_basic_ops.py
+++ b/tempest/scenario/test_server_basic_ops.py
@@ -15,8 +15,7 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest.scenario import manager
diff --git a/tempest/services/compute/xml/servers_client.py b/tempest/services/compute/xml/servers_client.py
index f7e8915..6d811a5 100644
--- a/tempest/services/compute/xml/servers_client.py
+++ b/tempest/services/compute/xml/servers_client.py
@@ -16,12 +16,12 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import time
import urllib
from lxml import etree
+from tempest.common import log as logging
from tempest.common.rest_client import RestClientXML
from tempest import exceptions
from tempest.services.compute.xml.common import Document
diff --git a/tempest/services/image/v1/json/image_client.py b/tempest/services/image/v1/json/image_client.py
index a8fab7f..f0b1c28 100644
--- a/tempest/services/image/v1/json/image_client.py
+++ b/tempest/services/image/v1/json/image_client.py
@@ -18,12 +18,12 @@
import copy
import errno
import json
-import logging
import os
import time
import urllib
from tempest.common import glance_http
+from tempest.common import log as logging
from tempest.common.rest_client import RestClient
from tempest import exceptions
diff --git a/tempest/services/volume/json/snapshots_client.py b/tempest/services/volume/json/snapshots_client.py
index db614f1..17f6cba 100644
--- a/tempest/services/volume/json/snapshots_client.py
+++ b/tempest/services/volume/json/snapshots_client.py
@@ -13,10 +13,10 @@
# under the License.
import json
-import logging
import time
import urllib
+from tempest.common import log as logging
from tempest.common.rest_client import RestClient
from tempest import exceptions
diff --git a/tempest/services/volume/xml/snapshots_client.py b/tempest/services/volume/xml/snapshots_client.py
index 2209fc7..410ed3e 100644
--- a/tempest/services/volume/xml/snapshots_client.py
+++ b/tempest/services/volume/xml/snapshots_client.py
@@ -12,12 +12,12 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import time
import urllib
from lxml import etree
+from tempest.common import log as logging
from tempest.common.rest_client import RestClientXML
from tempest import exceptions
from tempest.services.compute.xml.common import Document
diff --git a/tempest/test.py b/tempest/test.py
index c69a94c..5f403b3 100644
--- a/tempest/test.py
+++ b/tempest/test.py
@@ -15,13 +15,13 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import time
import nose.plugins.attrib
import testresources
import testtools
+from tempest.common import log as logging
from tempest import config
from tempest import manager
diff --git a/tempest/thirdparty/boto/test.py b/tempest/thirdparty/boto/test.py
index afa5c69..9ff628c 100644
--- a/tempest/thirdparty/boto/test.py
+++ b/tempest/thirdparty/boto/test.py
@@ -16,7 +16,7 @@
# under the License.
import contextlib
-import logging
+import logging as orig_logging
import os
import re
import urlparse
@@ -28,6 +28,7 @@
import keystoneclient.exceptions
import tempest.clients
+from tempest.common import log as logging
from tempest.common.utils.file_utils import have_effective_read_access
import tempest.config
from tempest import exceptions
@@ -58,7 +59,7 @@
A_I_IMAGES_READY = all_read(ami_path, aki_path, ari_path)
boto_logger = logging.getLogger('boto')
level = boto_logger.level
- boto_logger.setLevel(logging.CRITICAL) # suppress logging for these
+ boto_logger.setLevel(orig_logging.CRITICAL) # suppress logging for these
def _cred_sub_check(connection_data):
if not id_matcher.match(connection_data["aws_access_key_id"]):
diff --git a/tempest/thirdparty/boto/test_ec2_instance_run.py b/tempest/thirdparty/boto/test_ec2_instance_run.py
index e8abe97..7480833 100644
--- a/tempest/thirdparty/boto/test_ec2_instance_run.py
+++ b/tempest/thirdparty/boto/test_ec2_instance_run.py
@@ -15,12 +15,11 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
from boto import exception
import testtools
from tempest import clients
+from tempest.common import log as logging
from tempest.common.utils.data_utils import rand_name
from tempest.common.utils.linux.remote_client import RemoteClient
from tempest import exceptions
diff --git a/tempest/thirdparty/boto/test_ec2_volumes.py b/tempest/thirdparty/boto/test_ec2_volumes.py
index edf0180..c90c586 100644
--- a/tempest/thirdparty/boto/test_ec2_volumes.py
+++ b/tempest/thirdparty/boto/test_ec2_volumes.py
@@ -15,9 +15,8 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
-
from tempest import clients
+from tempest.common import log as logging
from tempest.test import attr
from tempest.thirdparty.boto.test import BotoTestCase
diff --git a/tempest/thirdparty/boto/utils/s3.py b/tempest/thirdparty/boto/utils/s3.py
index ea9869b..a309a12 100644
--- a/tempest/thirdparty/boto/utils/s3.py
+++ b/tempest/thirdparty/boto/utils/s3.py
@@ -16,13 +16,14 @@
# under the License.
import contextlib
-import logging
import os
import re
import boto
import boto.s3.key
+from tempest.common import log as logging
+
LOG = logging.getLogger(__name__)
diff --git a/tempest/thirdparty/boto/utils/wait.py b/tempest/thirdparty/boto/utils/wait.py
index 6cd17a9..6b3ef27 100644
--- a/tempest/thirdparty/boto/utils/wait.py
+++ b/tempest/thirdparty/boto/utils/wait.py
@@ -15,13 +15,13 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import re
import time
import boto.exception
from testtools import TestCase
+from tempest.common import log as logging
import tempest.config
LOG = logging.getLogger(__name__)
diff --git a/tempest/whitebox/manager.py b/tempest/whitebox/manager.py
index a75edb0..aa58ab6 100644
--- a/tempest/whitebox/manager.py
+++ b/tempest/whitebox/manager.py
@@ -15,7 +15,6 @@
# License for the specific language governing permissions and limitations
# under the License.
-import logging
import os
import shlex
import subprocess
@@ -23,6 +22,7 @@
from sqlalchemy import create_engine, MetaData
+from tempest.common import log as logging
from tempest.common.ssh import Client
from tempest.common.utils.data_utils import rand_name
from tempest import exceptions