Switch to using testr as the test runner for everything non-gating.
This commit switches the test runner in tempest to testr from nose for all
jobs that aren't gating. This will allow the usage of parallel testing with
the use of a group_regex in testr. Group_regex will ensure that the classes
get scheduled together and the run times are tracked together. Than the
tools/run_test_classes.py script will filter the test_ids and pass only the
classes to subunit.run to ensure we are only running setupClass once.
This commit also adds a new option to run_tests.sh. -t/--with-testr can be
used to optionally run tempest in parallel with testr. Once running with
testr gets stable enough this will become the default for run_tests.
It also adds a testr-full tox job so we can have a tracking non-voting job
that runs the same tests as the gate but with testr.
Implements: blueprint speed-up-tempest
Change-Id: Iedc9bd92b8f8471c60c614c7d7c05046d7b32743
diff --git a/.testr.conf b/.testr.conf
index a0262d8..fbea056 100644
--- a/.testr.conf
+++ b/.testr.conf
@@ -1,4 +1,5 @@
[DEFAULT]
-test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tempest $LISTOPT $IDOPTION
-test_id_option=--load-list $IDFILE
-test_list_option=--list
+test_command=${PYTHON:-python} -m subunit.run $LISTOPT $IDOPTION
+test_id_option=$(${PYTHON:-python} -m tools/run_test_classes $IDFILE)
+test_list_option=discover -t ./ ./tempest --list
+group_regex=([^\.]*\.)*
diff --git a/run_tests.sh b/run_tests.sh
index 366564e..d5081c7 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -11,6 +11,7 @@
echo " -u, --update Update the virtual environment with any newer package versions"
echo " -s, --smoke Only run smoke tests"
echo " -w, --whitebox Only run whitebox tests"
+ echo " -t, --with-testr Run using testr instead of nose"
echo " -c, --nova-coverage Enable Nova coverage collection"
echo " -C, --config Config file location"
echo " -p, --pep8 Just run pep8"
@@ -26,6 +27,7 @@
just_pep8=0
venv=.venv
with_venv=tools/with_venv.sh
+with_testr=0
always_venv=0
never_venv=0
no_site_packages=0
@@ -37,7 +39,7 @@
logging=0
logging_config=etc/logging.conf
-if ! options=$(getopt -o VNnfuswcphdSC:lL: -l virtual-env,no-virtual-env,no-site-packages,force,update,smoke,whitebox,nova-coverage,pep8,help,debug,stdout,config:,logging,logging-config: -- "$@")
+if ! options=$(getopt -o VNnfuswtcphdSC:lL: -l virtual-env,no-virtual-env,no-site-packages,force,update,smoke,whitebox,with-testr,nova-coverage,pep8,help,debug,stdout,config:,logging,logging-config: -- "$@")
then
# parse error
usage
@@ -60,6 +62,7 @@
-p|--pep8) let just_pep8=1;;
-s|--smoke) noseargs="$noseargs --attr=type=smoke";;
-w|--whitebox) noseargs="$noseargs --attr=type=whitebox";;
+ -t|--with-testr) with_testr=1;;
-S|--stdout) noseargs="$noseargs -s";;
-l|--logging) logging=1;;
-L|--logging-config) logging_config=$2; shift;;
@@ -105,8 +108,20 @@
noseargs="$noseargs tempest"
fi
+function testr_init {
+ if [ ! -d .testrepository ]; then
+ ${wrapper} testr init
+ fi
+}
+
function run_tests {
- ${wrapper} $NOSETESTS
+ if [ $with_testr -eq 1 ]; then
+ testr_init
+ ${wrapper} find . -type f -name "*.pyc" -delete
+ ${wrapper} testr run --parallel $noseargs
+ else
+ ${wrapper} $NOSETESTS
+ fi
}
function run_pep8 {
diff --git a/tools/run_test_classes.py b/tools/run_test_classes.py
new file mode 100755
index 0000000..c01b046
--- /dev/null
+++ b/tools/run_test_classes.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# Copyright 2013 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import sys
+
+
+def filter_classes(test_ids):
+ test_classes = map(lambda x: x.rsplit('.', 1)[0], test_ids)
+
+ #Remove duplicates from the list
+ uniq_class = {}
+ result = []
+ for test_class in test_classes:
+ if test_class in uniq_class:
+ continue
+ uniq_class[test_class] = 1
+ result.append(test_class)
+ return result
+
+
+def usage():
+ msg = """
+ This command is used to filter out the unique list of test cases (classes)
+ from a list of testr test_ids.
+
+ Usage: run_test_classes.py <test id file>
+ """
+ print(msg)
+ sys.exit(1)
+
+
+def main():
+ if len(sys.argv) == 2:
+ test_list_path = sys.argv[1]
+ test_list_file = open(test_list_path, 'r')
+ test_list = test_list_file.readlines()
+ for test_class in filter_classes(test_list):
+ print test_class
+ test_list_file.close()
+ else:
+ usage()
+
+if __name__ == '__main__':
+ main()
diff --git a/tox.ini b/tox.ini
index ae1b11e..04b845a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,24 +3,16 @@
[testenv]
setenv = VIRTUAL_ENV={envdir}
- NOSE_WITH_OPENSTACK=1
- NOSE_OPENSTACK_COLOR=1
- NOSE_OPENSTACK_RED=15
- NOSE_OPENSTACK_YELLOW=3
- NOSE_OPENSTACK_SHOW_ELAPSED=1
- NOSE_OPENSTACK_STDOUT=1
+ LANG=en_US.UTF-8
+ LANGUAGE=en_US:en
+ LC_ALL=C
[testenv:all]
sitepackages = True
setenv = VIRTUAL_ENV={envdir}
- NOSE_WITH_OPENSTACK=1
- NOSE_OPENSTACK_COLOR=1
- NOSE_OPENSTACK_RED=15
- NOSE_OPENSTACK_YELLOW=3
- NOSE_OPENSTACK_SHOW_ELAPSED=1
- NOSE_OPENSTACK_STDOUT=1
commands =
- nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit --xunit-file=nosetests-all.xml -sv tempest
+ python setup.py testr --slowest
+
[testenv:full]
sitepackages = True
@@ -34,6 +26,12 @@
commands =
nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit --xunit-file=nosetests-full.xml -sv tempest/api tempest/scenario tempest/thirdparty tempest/cli
+[testenv:testr-full]
+sitepackages = True
+setenv = VIRTUAL_ENV={envdir}
+commands =
+ python setup.py testr --slowest --testr-args='tempest.api tempest.scenario tempest.thirdparty tempest.cli'
+
[testenv:smoke]
sitepackages = True
setenv = VIRTUAL_ENV={envdir}
@@ -46,7 +44,6 @@
commands =
nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --attr=type=smoke --xunit-file=nosetests-smoke.xml tempest
-
[testenv:coverage]
sitepackages = True
setenv = VIRTUAL_ENV={envdir}