Add tools/tempest_coverage.py script.
This adds support for testing tempest's coverage on nova.
tools/tempest_coverage.py will enable coverage reporting in
nova to be started and stopped. It also performs coverage report
generation.
When it is enabled from run_tests with '-c' or '--nova_coverage'
a single text report file will be generated per nova service.
Implements: blueprint tempest-coverage-reporting
Change-Id: I00a52fb013c5a7a66a2317dbd5359a22d35bdb29
Signed-off-by: Matthew Treinish <treinish@linux.vnet.ibm.com>
diff --git a/run_tests.sh b/run_tests.sh
index 0df8a99..461ec2f 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -10,6 +10,7 @@
echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added."
echo " -s, --smoke Only run smoke tests"
echo " -w, --whitebox Only run whitebox tests"
+ echo " -c, --nova-coverage Enable Nova coverage collection"
echo " -p, --pep8 Just run pep8"
echo " -h, --help Print this usage message"
echo " -d, --debug Debug this script -- set -o xtrace"
@@ -25,6 +26,7 @@
-s|--no-site-packages) no_site_packages=1;;
-f|--force) force=1;;
-d|--debug) set -o xtrace;;
+ -c|--nova-coverage) let nova_coverage=1;;
-p|--pep8) let just_pep8=1;;
-s|--smoke) noseargs="$noseargs --attr=type=smoke";;
-w|--whitebox) noseargs="$noseargs --attr=type=whitebox";;
@@ -42,7 +44,7 @@
no_site_packages=0
force=0
wrapper=""
-
+nova_coverage=0
export NOSE_WITH_OPENSTACK=1
export NOSE_OPENSTACK_COLOR=1
@@ -82,6 +84,16 @@
${wrapper} python tools/hacking.py ${ignore} ${srcfiles}
}
+function run_coverage_start {
+ echo "Starting nova-coverage"
+ ${wrapper} python tools/tempest_coverage.py -c start
+}
+
+function run_coverage_report {
+ echo "Generating nova-coverage report"
+ ${wrapper} python tools/tempest_coverage.py -c report
+}
+
NOSETESTS="nosetests $noseargs"
if [ $never_venv -eq 0 ]
@@ -115,7 +127,15 @@
exit
fi
-run_tests || exit
+if [ $nova_coverage -eq 1 ]; then
+ run_coverage_start
+fi
+
+run_tests
+
+if [ $nova_coverage -eq 1 ]; then
+ run_coverage_report
+fi
if [ -z "$noseargs" ]; then
run_pep8
diff --git a/tools/tempest_coverage.py b/tools/tempest_coverage.py
new file mode 100755
index 0000000..73dcfbc
--- /dev/null
+++ b/tools/tempest_coverage.py
@@ -0,0 +1,194 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012 IBM
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License
+
+import json
+import os
+import re
+import shutil
+import sys
+
+from tempest.common.rest_client import RestClient
+from tempest import config
+from tempest.openstack.common import cfg
+from tempest.tests.compute import base
+
+CONF = config.TempestConfig()
+
+
+class CoverageClientJSON(RestClient):
+
+ def __init__(self, config, username, password, auth_url, tenant_name=None):
+ super(CoverageClientJSON, self).__init__(config, username, password,
+ auth_url, tenant_name)
+ self.service = self.config.compute.catalog_type
+
+ def start_coverage(self):
+ post_body = {
+ 'start': {},
+ }
+ post_body = json.dumps(post_body)
+ return self.post('os-coverage/action', post_body, self.headers)
+
+ def start_coverage_combine(self):
+ post_body = {
+ 'start': {
+ 'combine': True,
+ },
+ }
+ post_body = json.dumps(post_body)
+ return self.post('os-coverage/action', post_body, self.headers)
+
+ def stop_coverage(self):
+ post_body = {
+ 'stop': {},
+ }
+ post_body = json.dumps(post_body)
+ resp, body = self.post('os-coverage/action', post_body, self.headers)
+ body = json.loads(body)
+ return resp, body
+
+ def report_coverage_xml(self, file=None):
+ post_body = {
+ 'report': {
+ 'file': 'coverage.report',
+ 'xml': True,
+ },
+ }
+ if file:
+ post_body['report']['file'] = file
+ post_body = json.dumps(post_body)
+ resp, body = self.post('os-coverage/action', post_body, self.headers)
+ body = json.loads(body)
+ return resp, body
+
+ def report_coverage(self, file=None):
+ post_body = {
+ 'report': {
+ 'file': 'coverage.report',
+ },
+ }
+ if file:
+ post_body['report']['file'] = file
+ post_body = json.dumps(post_body)
+ resp, body = self.post('os-coverage/action', post_body, self.headers)
+ body = json.loads(body)
+ return resp, body
+
+ def report_coverage_html(self, file=None):
+ post_body = {
+ 'report': {
+ 'file': 'coverage.report',
+ 'html': True,
+ },
+ }
+ if file:
+ post_body['report']['file'] = file
+ post_body = json.dumps(post_body)
+ resp, body = self.post('os-coverage/action', post_body, self.headers)
+ body = json.loads(body)
+ return resp, body
+
+
+def parse_opts(argv):
+ cli_opts = [
+ cfg.StrOpt('command',
+ short='c',
+ default='',
+ help="This required argument is used to specify the "
+ "coverage command to run. Only 'start', "
+ "'stop', or 'report' are valid fields."),
+ cfg.StrOpt('filename',
+ default='tempest-coverage',
+ help="Specify a filename to be used for generated report "
+ "files"),
+ cfg.BoolOpt('xml',
+ default=False,
+ help='Generate XML reports instead of text'),
+ cfg.BoolOpt('html',
+ default=False,
+ help='Generate HTML reports instead of text'),
+ cfg.BoolOpt('combine',
+ default=False,
+ help='Generate a single report for all services'),
+ cfg.StrOpt('output',
+ short='o',
+ default=None,
+ help='Optional directory to copy generated coverage data or'
+ ' reports into. This directory must not already exist '
+ 'it will be created')
+ ]
+ CLI = cfg.ConfigOpts()
+ CLI.register_cli_opts(cli_opts)
+ CLI(argv[1:])
+ return CLI
+
+
+def main(argv):
+ CLI = parse_opts(argv)
+ client_args = (CONF, CONF.compute_admin.username,
+ CONF.compute_admin.password, CONF.identity.auth_url,
+ CONF.compute_admin.tenant_name)
+ coverage_client = CoverageClientJSON(*client_args)
+
+ if CLI.command == 'start':
+ if CLI.combine:
+ coverage_client.start_coverage_combine()
+ else:
+ coverage_client.start_coverage()
+
+ elif CLI.command == 'stop':
+ resp, body = coverage_client.stop_coverage()
+ if not resp['status'] == '200':
+ print 'coverage stop failed with: %s:' % (resp['status'] + ': '
+ + body)
+ exit(int(resp['status']))
+ path = body['path']
+ if CLI.output:
+ shutil.copytree(path, CLI.output)
+ else:
+ print "Data files located at: %s" % path
+
+ elif CLI.command == 'report':
+ if CLI.xml:
+ resp, body = coverage_client.report_coverage_xml(file=CLI.filename)
+ elif CLI.html:
+ resp, body = coverage_client.report_coverage_html(
+ file=CLI.filename)
+ else:
+ resp, body = coverage_client.report_coverage(file=CLI.filename)
+ if not resp['status'] == '200':
+ print 'coverage report failed with: %s:' % (resp['status'] + ': '
+ + body)
+ exit(int(resp['status']))
+ path = body['path']
+ if CLI.output:
+ if CLI.html:
+ shutil.copytree(path, CLI.output)
+ else:
+ path = os.path.dirname(path)
+ shutil.copytree(path, CLI.output)
+ else:
+ if not CLI.html:
+ path = os.path.dirname(path)
+ print 'Report files located at: %s' % path
+
+ else:
+ print 'Invalid command'
+ exit(1)
+
+
+if __name__ == "__main__":
+ main(sys.argv)
diff --git a/tox.ini b/tox.ini
index 2d8e627..991842c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -13,6 +13,11 @@
-r{toxinidir}/tools/test-requires
commands = nosetests {posargs}
+[testenv:coverage]
+commands = python tools/tempest_coverage.py -c start --combine
+ nosetests {posargs}
+ python tools/tempest_coverage.py -c report --html
+
[testenv:pep8]
deps = pep8==1.3.3
commands = python tools/hacking.py --ignore=N4,E122,E125,E126 --repeat --show-source --exclude=.venv,.tox,dist,doc,openstack,*egg .