Merge "Add tempest run command"
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 98b006d..c73fac3 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -52,6 +52,7 @@
cleanup
javelin
workspace
+ run
==================
Indices and tables
diff --git a/doc/source/run.rst b/doc/source/run.rst
new file mode 100644
index 0000000..07fa5f7
--- /dev/null
+++ b/doc/source/run.rst
@@ -0,0 +1,5 @@
+-----------
+Tempest Run
+-----------
+
+.. automodule:: tempest.cmd.run
diff --git a/releasenotes/notes/add-tempest-run-3d0aaf69c2ca4115.yaml b/releasenotes/notes/add-tempest-run-3d0aaf69c2ca4115.yaml
new file mode 100644
index 0000000..429bf52
--- /dev/null
+++ b/releasenotes/notes/add-tempest-run-3d0aaf69c2ca4115.yaml
@@ -0,0 +1,4 @@
+---
+features:
+ - Adds the tempest run command to the unified tempest CLI. This new command
+ is used for running tempest tests.
diff --git a/setup.cfg b/setup.cfg
index 0bf493c..66a8743 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -42,6 +42,7 @@
list-plugins = tempest.cmd.list_plugins:TempestListPlugins
verify-config = tempest.cmd.verify_tempest_config:TempestVerifyConfig
workspace = tempest.cmd.workspace:TempestWorkspace
+ run = tempest.cmd.run:TempestRun
oslo.config.opts =
tempest.config = tempest.config:list_opts
diff --git a/tempest/cmd/run.py b/tempest/cmd/run.py
new file mode 100644
index 0000000..b4b7ebb
--- /dev/null
+++ b/tempest/cmd/run.py
@@ -0,0 +1,181 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""
+Runs tempest tests
+
+This command is used for running the tempest tests
+
+Test Selection
+==============
+Tempest run has several options:
+
+ * **--regex/-r**: This is a selection regex like what testr uses. It will run
+ any tests that match on re.match() with the regex
+ * **--smoke**: Run all the tests tagged as smoke
+
+You can also use the **--list-tests** option in conjunction with selection
+arguments to list which tests will be run.
+
+Test Execution
+==============
+There are several options to control how the tests are executed. By default
+tempest will run in parallel with a worker for each CPU present on the machine.
+If you want to adjust the number of workers use the **--concurrency** option
+and if you want to run tests serially use **--serial**
+
+Test Output
+===========
+By default tempest run's output to STDOUT will be generated using the
+subunit-trace output filter. But, if you would prefer a subunit v2 stream be
+output to STDOUT use the **--subunit** flag
+
+"""
+
+import io
+import os
+import sys
+import threading
+
+from cliff import command
+from os_testr import subunit_trace
+from oslo_log import log as logging
+from testrepository.commands import run_argv
+
+from tempest import config
+
+
+LOG = logging.getLogger(__name__)
+CONF = config.CONF
+
+
+class TempestRun(command.Command):
+
+ def _set_env(self):
+ # NOTE(mtreinish): This is needed so that testr doesn't gobble up any
+ # stacktraces on failure.
+ if 'TESTR_PDB' in os.environ:
+ return
+ else:
+ os.environ["TESTR_PDB"] = ""
+
+ def take_action(self, parsed_args):
+ self._set_env()
+ # Local exceution mode
+ if os.path.isfile('.testr.conf'):
+ # If you're running in local execution mode and there is not a
+ # testrepository dir create one
+ if not os.path.isdir('.testrepository'):
+ returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout,
+ sys.stderr)
+ if returncode:
+ sys.exit(returncode)
+ else:
+ print("No .testr.conf file was found for local exceution")
+ sys.exit(2)
+
+ regex = self._build_regex(parsed_args)
+ if parsed_args.list_tests:
+ argv = ['tempest', 'list-tests', regex]
+ returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
+ else:
+ options = self._build_options(parsed_args)
+ returncode = self._run(regex, options)
+ sys.exit(returncode)
+
+ def get_description(self):
+ return 'Run tempest'
+
+ def get_parser(self, prog_name):
+ parser = super(TempestRun, self).get_parser(prog_name)
+ parser = self._add_args(parser)
+ return parser
+
+ def _add_args(self, parser):
+ # test selection args
+ regex = parser.add_mutually_exclusive_group()
+ regex.add_argument('--smoke', action='store_true',
+ help="Run the smoke tests only")
+ regex.add_argument('--regex', '-r', default='',
+ help='A normal testr selection regex used to '
+ 'specify a subset of tests to run')
+ # list only args
+ parser.add_argument('--list-tests', '-l', action='store_true',
+ help='List tests',
+ default=False)
+ # exectution args
+ parser.add_argument('--concurrency', '-w',
+ help="The number of workers to use, defaults to "
+ "the number of cpus")
+ parallel = parser.add_mutually_exclusive_group()
+ parallel.add_argument('--parallel', dest='parallel',
+ action='store_true',
+ help='Run tests in parallel (this is the'
+ ' default)')
+ parallel.add_argument('--serial', dest='parallel',
+ action='store_false',
+ help='Run tests serially')
+ # output args
+ parser.add_argument("--subunit", action='store_true',
+ help='Enable subunit v2 output')
+
+ parser.set_defaults(parallel=True)
+ return parser
+
+ def _build_regex(self, parsed_args):
+ regex = ''
+ if parsed_args.smoke:
+ regex = 'smoke'
+ elif parsed_args.regex:
+ regex = parsed_args.regex
+ return regex
+
+ def _build_options(self, parsed_args):
+ options = []
+ if parsed_args.subunit:
+ options.append("--subunit")
+ if parsed_args.parallel:
+ options.append("--parallel")
+ if parsed_args.concurrency:
+ options.append("--concurrency=%s" % parsed_args.concurrency)
+ return options
+
+ def _run(self, regex, options):
+ returncode = 0
+ argv = ['tempest', 'run', regex] + options
+ if '--subunit' in options:
+ returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
+ else:
+ argv.append('--subunit')
+ stdin = io.StringIO()
+ stdout_r, stdout_w = os.pipe()
+ subunit_w = os.fdopen(stdout_w, 'wt')
+ subunit_r = os.fdopen(stdout_r)
+ returncodes = {}
+
+ def run_argv_thread():
+ returncodes['testr'] = run_argv(argv, stdin, subunit_w,
+ sys.stderr)
+ subunit_w.close()
+
+ run_thread = threading.Thread(target=run_argv_thread)
+ run_thread.start()
+ returncodes['subunit-trace'] = subunit_trace.trace(subunit_r,
+ sys.stdout)
+ run_thread.join()
+ subunit_r.close()
+ # python version of pipefail
+ if returncodes['testr']:
+ returncode = returncodes['testr']
+ elif returncodes['subunit-trace']:
+ returncode = returncodes['subunit-trace']
+ return returncode
diff --git a/tempest/tests/cmd/test_run.py b/tempest/tests/cmd/test_run.py
new file mode 100644
index 0000000..9aa06e5
--- /dev/null
+++ b/tempest/tests/cmd/test_run.py
@@ -0,0 +1,110 @@
+# Copyright 2015 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import os
+import shutil
+import subprocess
+import tempfile
+
+import mock
+
+from tempest.cmd import run
+from tempest.tests import base
+
+DEVNULL = open(os.devnull, 'wb')
+
+
+class TestTempestRun(base.TestCase):
+
+ def setUp(self):
+ super(TestTempestRun, self).setUp()
+ self.run_cmd = run.TempestRun(None, None)
+
+ def test_build_options(self):
+ args = mock.Mock(spec=argparse.Namespace)
+ setattr(args, "subunit", True)
+ setattr(args, "parallel", False)
+ setattr(args, "concurrency", 10)
+ options = self.run_cmd._build_options(args)
+ self.assertEqual(['--subunit',
+ '--concurrency=10'],
+ options)
+
+ def test__build_regex_default(self):
+ args = mock.Mock(spec=argparse.Namespace)
+ setattr(args, 'smoke', False)
+ setattr(args, 'regex', '')
+ self.assertEqual('', self.run_cmd._build_regex(args))
+
+ def test__build_regex_smoke(self):
+ args = mock.Mock(spec=argparse.Namespace)
+ setattr(args, "smoke", True)
+ setattr(args, 'regex', '')
+ self.assertEqual('smoke', self.run_cmd._build_regex(args))
+
+ def test__build_regex_regex(self):
+ args = mock.Mock(spec=argparse.Namespace)
+ setattr(args, 'smoke', False)
+ setattr(args, "regex", 'i_am_a_fun_little_regex')
+ self.assertEqual('i_am_a_fun_little_regex',
+ self.run_cmd._build_regex(args))
+
+
+class TestRunReturnCode(base.TestCase):
+ def setUp(self):
+ super(TestRunReturnCode, self).setUp()
+ # Setup test dirs
+ self.directory = tempfile.mkdtemp(prefix='tempest-unit')
+ self.addCleanup(shutil.rmtree, self.directory)
+ self.test_dir = os.path.join(self.directory, 'tests')
+ os.mkdir(self.test_dir)
+ # Setup Test files
+ self.testr_conf_file = os.path.join(self.directory, '.testr.conf')
+ self.setup_cfg_file = os.path.join(self.directory, 'setup.cfg')
+ self.passing_file = os.path.join(self.test_dir, 'test_passing.py')
+ self.failing_file = os.path.join(self.test_dir, 'test_failing.py')
+ self.init_file = os.path.join(self.test_dir, '__init__.py')
+ self.setup_py = os.path.join(self.directory, 'setup.py')
+ shutil.copy('tempest/tests/files/testr-conf', self.testr_conf_file)
+ shutil.copy('tempest/tests/files/passing-tests', self.passing_file)
+ shutil.copy('tempest/tests/files/failing-tests', self.failing_file)
+ shutil.copy('setup.py', self.setup_py)
+ shutil.copy('tempest/tests/files/setup.cfg', self.setup_cfg_file)
+ shutil.copy('tempest/tests/files/__init__.py', self.init_file)
+ # Change directory, run wrapper and check result
+ self.addCleanup(os.chdir, os.path.abspath(os.curdir))
+ os.chdir(self.directory)
+
+ def assertRunExit(self, cmd, expected):
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE)
+ out, err = p.communicate()
+ msg = ("Running %s got an unexpected returncode\n"
+ "Stdout: %s\nStderr: %s" % (' '.join(cmd), out, err))
+ self.assertEqual(p.returncode, expected, msg)
+
+ def test_tempest_run_passes(self):
+ # Git init is required for the pbr testr command. pbr requires a git
+ # version or an sdist to work. so make the test directory a git repo
+ # too.
+ subprocess.call(['git', 'init'], stderr=DEVNULL)
+ self.assertRunExit(['tempest', 'run', '--regex', 'passing'], 0)
+
+ def test_tempest_run_fails(self):
+ # Git init is required for the pbr testr command. pbr requires a git
+ # version or an sdist to work. so make the test directory a git repo
+ # too.
+ subprocess.call(['git', 'init'], stderr=DEVNULL)
+ self.assertRunExit(['tempest', 'run'], 1)