| # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| # not use this file except in compliance with the License. You may obtain |
| # a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| # License for the specific language governing permissions and limitations |
| # under the License. |
| |
| """ |
| Runs tempest tests |
| |
| This command is used for running the tempest tests |
| |
| Test Selection |
| ============== |
| Tempest run has several options: |
| |
| * ``--regex/-r``: This is a selection regex like what stestr uses. It will run |
| any tests that match on re.match() with the regex |
| * ``--smoke/-s``: Run all the tests tagged as smoke |
| * ``--black-regex``: It allows to do simple test exclusion via passing a |
| rejection/black regexp |
| |
| There are also the ``--blacklist-file`` and ``--whitelist-file`` options that |
| let you pass a filepath to tempest run with the file format being a line |
| separated regex, with '#' used to signify the start of a comment on a line. |
| For example:: |
| |
| # Regex file |
| ^regex1 # Match these tests |
| .*regex2 # Match those tests |
| |
| These arguments are just passed into stestr, you can refer to the stestr |
| selection docs for more details on how these operate: |
| http://stestr.readthedocs.io/en/latest/MANUAL.html#test-selection |
| |
| You can also use the ``--list-tests`` option in conjunction with selection |
| arguments to list which tests will be run. |
| |
| You can also use the ``--load-list`` option that lets you pass a filepath to |
| tempest run with the file format being in a non-regex format, similar to the |
| tests generated by the ``--list-tests`` option. You can specify target tests |
| by removing unnecessary tests from a list file which is generated from |
| ``--list-tests`` option. |
| |
| Test Execution |
| ============== |
| There are several options to control how the tests are executed. By default |
| tempest will run in parallel with a worker for each CPU present on the machine. |
| If you want to adjust the number of workers use the ``--concurrency`` option |
| and if you want to run tests serially use ``--serial/-t`` |
| |
| Running with Workspaces |
| ----------------------- |
| Tempest run enables you to run your tempest tests from any setup tempest |
| workspace it relies on you having setup a tempest workspace with either the |
| ``tempest init`` or ``tempest workspace`` commands. Then using the |
| ``--workspace`` CLI option you can specify which one of your workspaces you |
| want to run tempest from. Using this option you don't have to run Tempest |
| directly with you current working directory being the workspace, Tempest will |
| take care of managing everything to be executed from there. |
| |
| Running from Anywhere |
| --------------------- |
| Tempest run provides you with an option to execute tempest from anywhere on |
| your system. You are required to provide a config file in this case with the |
| ``--config-file`` option. When run tempest will create a .stestr |
| directory and a .stestr.conf file in your current working directory. This way |
| you can use stestr commands directly to inspect the state of the previous run. |
| |
| Test Output |
| =========== |
| By default tempest run's output to STDOUT will be generated using the |
| subunit-trace output filter. But, if you would prefer a subunit v2 stream be |
| output to STDOUT use the ``--subunit`` flag |
| |
| Combining Runs |
| ============== |
| |
| There are certain situations in which you want to split a single run of tempest |
| across 2 executions of tempest run. (for example to run part of the tests |
| serially and others in parallel) To accomplish this but still treat the results |
| as a single run you can leverage the ``--combine`` option which will append |
| the current run's results with the previous runs. |
| """ |
| |
| import os |
| import sys |
| |
| from cliff import command |
| from oslo_serialization import jsonutils as json |
| import six |
| from stestr import commands |
| |
| from tempest import clients |
| from tempest.cmd import cleanup_service |
| from tempest.cmd import init |
| from tempest.cmd import workspace |
| from tempest.common import credentials_factory as credentials |
| from tempest import config |
| |
| if six.PY2: |
| # Python 2 has not FileNotFoundError exception |
| FileNotFoundError = IOError |
| |
| CONF = config.CONF |
| SAVED_STATE_JSON = "saved_state.json" |
| |
| |
| class TempestRun(command.Command): |
| |
| def _set_env(self, config_file=None): |
| if config_file: |
| if os.path.exists(os.path.abspath(config_file)): |
| CONF.set_config_path(os.path.abspath(config_file)) |
| else: |
| raise FileNotFoundError( |
| "Config file: %s doesn't exist" % config_file) |
| |
| # NOTE(mtreinish): This is needed so that stestr doesn't gobble up any |
| # stacktraces on failure. |
| if 'TESTR_PDB' in os.environ: |
| return |
| else: |
| os.environ["TESTR_PDB"] = "" |
| # NOTE(dims): most of our .stestr.conf try to test for PYTHON |
| # environment variable and fall back to "python", under python3 |
| # if it does not exist. we should set it to the python3 executable |
| # to deal with this situation better for now. |
| if six.PY3 and 'PYTHON' not in os.environ: |
| os.environ['PYTHON'] = sys.executable |
| |
| def _create_stestr_conf(self): |
| top_level_path = os.path.dirname(os.path.dirname(__file__)) |
| discover_path = os.path.join(top_level_path, 'test_discover') |
| file_contents = init.STESTR_CONF % (discover_path, top_level_path) |
| with open('.stestr.conf', 'w+') as stestr_conf_file: |
| stestr_conf_file.write(file_contents) |
| |
| def take_action(self, parsed_args): |
| if parsed_args.config_file: |
| self._set_env(parsed_args.config_file) |
| else: |
| self._set_env() |
| # Workspace execution mode |
| if parsed_args.workspace: |
| workspace_mgr = workspace.WorkspaceManager( |
| parsed_args.workspace_path) |
| path = workspace_mgr.get_workspace(parsed_args.workspace) |
| if not path: |
| sys.exit( |
| "The %r workspace isn't registered in " |
| "%r. Use 'tempest init' to " |
| "register the workspace." % |
| (parsed_args.workspace, workspace_mgr.path)) |
| os.chdir(path) |
| if not os.path.isfile('.stestr.conf'): |
| self._create_stestr_conf() |
| # local execution with config file mode |
| elif parsed_args.config_file and not os.path.isfile('.stestr.conf'): |
| self._create_stestr_conf() |
| elif not os.path.isfile('.stestr.conf'): |
| print("No .stestr.conf file was found for local execution") |
| sys.exit(2) |
| if parsed_args.state: |
| self._init_state() |
| |
| regex = self._build_regex(parsed_args) |
| return_code = 0 |
| if parsed_args.list_tests: |
| return_code = commands.list_command( |
| filters=regex, whitelist_file=parsed_args.whitelist_file, |
| blacklist_file=parsed_args.blacklist_file, |
| black_regex=parsed_args.black_regex) |
| |
| else: |
| serial = not parsed_args.parallel |
| return_code = commands.run_command( |
| filters=regex, subunit_out=parsed_args.subunit, |
| serial=serial, concurrency=parsed_args.concurrency, |
| blacklist_file=parsed_args.blacklist_file, |
| whitelist_file=parsed_args.whitelist_file, |
| black_regex=parsed_args.black_regex, |
| load_list=parsed_args.load_list, combine=parsed_args.combine) |
| if return_code > 0: |
| sys.exit(return_code) |
| return return_code |
| |
| def get_description(self): |
| return 'Run tempest' |
| |
| def _init_state(self): |
| print("Initializing saved state.") |
| data = {} |
| self.global_services = cleanup_service.get_global_cleanup_services() |
| self.admin_mgr = clients.Manager( |
| credentials.get_configured_admin_credentials()) |
| admin_mgr = self.admin_mgr |
| kwargs = {'data': data, |
| 'is_dry_run': False, |
| 'saved_state_json': data, |
| 'is_preserve': False, |
| 'is_save_state': True} |
| for service in self.global_services: |
| svc = service(admin_mgr, **kwargs) |
| svc.run() |
| |
| with open(SAVED_STATE_JSON, 'w+') as f: |
| f.write(json.dumps(data, sort_keys=True, |
| indent=2, separators=(',', ': '))) |
| |
| def get_parser(self, prog_name): |
| parser = super(TempestRun, self).get_parser(prog_name) |
| parser = self._add_args(parser) |
| return parser |
| |
| def _add_args(self, parser): |
| # workspace args |
| parser.add_argument('--workspace', default=None, |
| help='Name of tempest workspace to use for running' |
| ' tests. You can see a list of workspaces ' |
| 'with tempest workspace list') |
| parser.add_argument('--workspace-path', default=None, |
| dest='workspace_path', |
| help="The path to the workspace file, the default " |
| "is ~/.tempest/workspace.yaml") |
| # Configuration flags |
| parser.add_argument('--config-file', default=None, dest='config_file', |
| help='Configuration file to run tempest with') |
| # test selection args |
| regex = parser.add_mutually_exclusive_group() |
| regex.add_argument('--smoke', '-s', action='store_true', |
| help="Run the smoke tests only") |
| regex.add_argument('--regex', '-r', default='', |
| help='A normal stestr selection regex used to ' |
| 'specify a subset of tests to run') |
| parser.add_argument('--black-regex', dest='black_regex', |
| help='A regex to exclude tests that match it') |
| parser.add_argument('--whitelist-file', '--whitelist_file', |
| help="Path to a whitelist file, this file " |
| "contains a separate regex on each " |
| "newline.") |
| parser.add_argument('--blacklist-file', '--blacklist_file', |
| help='Path to a blacklist file, this file ' |
| 'contains a separate regex exclude on ' |
| 'each newline') |
| parser.add_argument('--load-list', '--load_list', |
| help='Path to a non-regex whitelist file, ' |
| 'this file contains a separate test ' |
| 'on each newline. This command ' |
| 'supports files created by the tempest ' |
| 'run ``--list-tests`` command') |
| # list only args |
| parser.add_argument('--list-tests', '-l', action='store_true', |
| help='List tests', |
| default=False) |
| # execution args |
| parser.add_argument('--concurrency', '-w', |
| type=int, default=0, |
| help="The number of workers to use, defaults to " |
| "the number of cpus") |
| parallel = parser.add_mutually_exclusive_group() |
| parallel.add_argument('--parallel', dest='parallel', |
| action='store_true', |
| help='Run tests in parallel (this is the' |
| ' default)') |
| parallel.add_argument('--serial', '-t', dest='parallel', |
| action='store_false', |
| help='Run tests serially') |
| parser.add_argument('--save-state', dest='state', |
| action='store_true', |
| help="To save the state of the cloud before " |
| "running tempest.") |
| # output args |
| parser.add_argument("--subunit", action='store_true', |
| help='Enable subunit v2 output') |
| parser.add_argument("--combine", action='store_true', |
| help='Combine the output of this run with the ' |
| "previous run's as a combined stream in the " |
| "stestr repository after it finish") |
| |
| parser.set_defaults(parallel=True) |
| return parser |
| |
| def _build_regex(self, parsed_args): |
| regex = None |
| if parsed_args.smoke: |
| regex = ['smoke'] |
| elif parsed_args.regex: |
| regex = parsed_args.regex.split() |
| return regex |