Add unittest framework + tests for wrapper scripts
This commit adds a framework for running unittests on tempest to
verify that tempest works as expect. The first tests added are to
verify the response codes of the test runner wrapper scripts we use
on jenkins jobs.
Change-Id: If8e91238054593999e7b5bc34da499bd0ac02311
diff --git a/doc/source/field_guide/unit_tests.rst b/doc/source/field_guide/unit_tests.rst
new file mode 120000
index 0000000..67a8b20
--- /dev/null
+++ b/doc/source/field_guide/unit_tests.rst
@@ -0,0 +1 @@
+../../../tempest/tests/README.rst
\ No newline at end of file
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 00c4e9a..f70cdd1 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -32,6 +32,7 @@
field_guide/stress
field_guide/thirdparty
field_guide/whitebox
+ field_guide/unit_tests
------------------
API and test cases
diff --git a/tempest/hacking/checks.py b/tempest/hacking/checks.py
index f9eb968..8cfd548 100644
--- a/tempest/hacking/checks.py
+++ b/tempest/hacking/checks.py
@@ -53,17 +53,6 @@
" in tempest/api/* tests"))
-def import_no_files_in_tests(physical_line, filename):
- """Check for merges that try to land into tempest/tests
-
- T103: tempest/tests directory is deprecated
- """
-
- if "tempest/tests" in filename:
- return (0, ("T103: tempest/tests is deprecated"))
-
-
def factory(register):
register(skip_bugs)
register(import_no_clients_in_api)
- register(import_no_files_in_tests)
diff --git a/tempest/tests/README.rst b/tempest/tests/README.rst
new file mode 100644
index 0000000..4098686
--- /dev/null
+++ b/tempest/tests/README.rst
@@ -0,0 +1,25 @@
+Tempest Guide to Unit tests
+===========================
+
+What are these tests?
+---------------------
+
+Unit tests are the self checks for Tempest. They provide functional
+verification and regression checking for the internal components of tempest.
+They should be used to just verify that the individual pieces of tempest are
+working as expected. They should not require an external service to be running
+and should be able to run solely from the tempest tree.
+
+Why are these tests in tempest?
+-------------------------------
+These tests exist to make sure that the mechanisms that we use inside of
+tempest to are valid and remain functional. They are only here for self
+validation of tempest.
+
+
+Scope of these tests
+--------------------
+Unit tests should not require an external service to be running or any extra
+configuration to run. Any state that is required for a test should either be
+mocked out or created in a temporary test directory. (see test_wrappers.py for
+an example of using a temporary test directory)
diff --git a/tempest/tests/__init__.py b/tempest/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tempest/tests/__init__.py
diff --git a/tempest/tests/files/__init__.py b/tempest/tests/files/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tempest/tests/files/__init__.py
diff --git a/tempest/tests/files/failing-tests b/tempest/tests/files/failing-tests
new file mode 100644
index 0000000..0ec5421
--- /dev/null
+++ b/tempest/tests/files/failing-tests
@@ -0,0 +1,25 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import testtools
+
+class FakeTestClass(testtools.TestCase):
+ def test_pass(self):
+ self.assertTrue(False)
+
+ def test_pass_list(self):
+ test_list = ['test', 'a', 'b']
+ self.assertIn('fail', test_list)
diff --git a/tempest/tests/files/passing-tests b/tempest/tests/files/passing-tests
new file mode 100644
index 0000000..2f5b7c9
--- /dev/null
+++ b/tempest/tests/files/passing-tests
@@ -0,0 +1,25 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import testtools
+
+class FakeTestClass(testtools.TestCase):
+ def test_pass(self):
+ self.assertTrue(True)
+
+ def test_pass_list(self):
+ test_list = ['test', 'a', 'b']
+ self.assertIn('test', test_list)
diff --git a/tempest/tests/files/setup.cfg b/tempest/tests/files/setup.cfg
new file mode 100644
index 0000000..8639baa
--- /dev/null
+++ b/tempest/tests/files/setup.cfg
@@ -0,0 +1,20 @@
+[metadata]
+name = tempest_unit_tests
+version = 1
+summary = Fake Project for testing wrapper scripts
+author = OpenStack QA
+author-email = openstack-qa@lists.openstack.org
+home-page = http://www.openstack.org/
+classifier =
+ Intended Audience :: Information Technology
+ Intended Audience :: System Administrators
+ Intended Audience :: Developers
+ License :: OSI Approved :: Apache Software License
+ Operating System :: POSIX :: Linux
+ Programming Language :: Python
+ Programming Language :: Python :: 2
+ Programming Language :: Python :: 2.7
+
+[global]
+setup-hooks =
+ pbr.hooks.setup_hook
diff --git a/tempest/tests/files/testr-conf b/tempest/tests/files/testr-conf
new file mode 100644
index 0000000..d5ad083
--- /dev/null
+++ b/tempest/tests/files/testr-conf
@@ -0,0 +1,5 @@
+[DEFAULT]
+test_command=${PYTHON:-python} -m subunit.run discover -t ./ ./tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list
+group_regex=([^\.]*\.)*
diff --git a/tempest/tests/test_wrappers.py b/tempest/tests/test_wrappers.py
new file mode 100644
index 0000000..aeea98d
--- /dev/null
+++ b/tempest/tests/test_wrappers.py
@@ -0,0 +1,103 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2013 IBM Corp.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import shutil
+import subprocess
+import tempfile
+import testtools
+
+from tempest.test import attr
+
+DEVNULL = open(os.devnull, 'wb')
+
+
+class TestWrappers(testtools.TestCase):
+ def setUp(self):
+ super(TestWrappers, self).setUp()
+ # Setup test dirs
+ self.directory = tempfile.mkdtemp(prefix='tempest-unit')
+ self.test_dir = os.path.join(self.directory, 'tests')
+ os.mkdir(self.test_dir)
+ # Setup Test files
+ self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
+ self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
+ self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
+ self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
+ self.init_file = os.path.join(self.test_dir, '__init__.py')
+ self.setup_py = os.path.join(self.directory, 'setup.py')
+ shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file)
+ shutil.copy('tempest/tests/files/passing-tests', self.passing_file)
+ shutil.copy('tempest/tests/files/failing-tests', self.failing_file)
+ shutil.copy('setup.py', self.setup_py)
+ shutil.copy('tempest/tests/files/setup.cfg', self.setup_cfg_file)
+ shutil.copy('tempest/tests/files/__init__.py', self.init_file)
+
+ @attr(type='smoke')
+ def test_pretty_tox(self):
+ # Copy wrapper script and requirements:
+ pretty_tox = os.path.join(self.directory, 'pretty_tox.sh')
+ shutil.copy('tools/pretty_tox.sh', pretty_tox)
+ # Change directory, run wrapper and check result
+ self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+ os.chdir(self.directory)
+ # Git init is required for the pbr testr command. pbr requires a git
+ # version or an sdist to work. so make the test directory a git repo
+ # too.
+ subprocess.call(['git', 'init'])
+ exit_code = subprocess.call('sh pretty_tox.sh tests.passing',
+ shell=True, stdout=DEVNULL, stderr=DEVNULL)
+ self.assertEquals(exit_code, 0)
+
+ @attr(type='smoke')
+ def test_pretty_tox_fails(self):
+ # Copy wrapper script and requirements:
+ pretty_tox = os.path.join(self.directory, 'pretty_tox.sh')
+ shutil.copy('tools/pretty_tox.sh', pretty_tox)
+ # Change directory, run wrapper and check result
+ self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+ os.chdir(self.directory)
+ # Git init is required for the pbr testr command. pbr requires a git
+ # version or an sdist to work. so make the test directory a git repo
+ # too.
+ subprocess.call(['git', 'init'])
+ exit_code = subprocess.call('sh pretty_tox.sh', shell=True,
+ stdout=DEVNULL, stderr=DEVNULL)
+ self.assertEquals(exit_code, 1)
+
+ @attr(type='smoke')
+ def test_pretty_tox_serial(self):
+ # Copy wrapper script and requirements:
+ pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh')
+ shutil.copy('tools/pretty_tox_serial.sh', pretty_tox)
+ # Change directory, run wrapper and check result
+ self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+ os.chdir(self.directory)
+ exit_code = subprocess.call('sh pretty_tox_serial.sh tests.passing',
+ shell=True, stdout=DEVNULL, stderr=DEVNULL)
+ self.assertEquals(exit_code, 0)
+
+ @attr(type='smoke')
+ def test_pretty_tox_serial_fails(self):
+ # Copy wrapper script and requirements:
+ pretty_tox = os.path.join(self.directory, 'pretty_tox_serial.sh')
+ shutil.copy('tools/pretty_tox_serial.sh', pretty_tox)
+ # Change directory, run wrapper and check result
+ self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+ os.chdir(self.directory)
+ exit_code = subprocess.call('sh pretty_tox_serial.sh', shell=True,
+ stdout=DEVNULL, stderr=DEVNULL)
+ self.assertEquals(exit_code, 1)
diff --git a/tox.ini b/tox.ini
index dc48735..a101a9e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,13 +19,13 @@
# The regex below is used to select which tests to run and exclude the slow tag:
# See the testrepostiory bug: https://bugs.launchpad.net/testrepository/+bug/1208610
commands =
- sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}'
+ sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}'
[testenv:testr-full]
sitepackages = True
setenv = VIRTUAL_ENV={envdir}
commands =
- sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli)) {posargs}'
+ sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests)) {posargs}'
[testenv:heat-slow]
sitepackages = True
@@ -44,7 +44,7 @@
NOSE_OPENSTACK_SHOW_ELAPSED=1
NOSE_OPENSTACK_STDOUT=1
commands =
- nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli {posargs}
+ nosetests --logging-format '%(asctime)-15s %(message)s' --with-xunit -sv --xunit-file=nosetests-full.xml tempest/api tempest/scenario tempest/thirdparty tempest/cli tempest/tests {posargs}
[testenv:py26-smoke]
setenv = VIRTUAL_ENV={envdir}
@@ -71,7 +71,7 @@
setenv = VIRTUAL_ENV={envdir}
commands =
python -m tools/tempest_coverage -c start --combine
- sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli))'
+ sh tools/pretty_tox.sh '(?!.*\[.*\bslow\b.*\])(^tempest\.(api|scenario|thirdparty|cli|tests))'
python -m tools/tempest_coverage -c report --html {posargs}
[testenv:stress]