tempest/cmd/run.py

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""
Runs tempest tests

This command is used for running the tempest tests

Test Selection
==============
Tempest run has several options:

 * **--regex/-r**: This is a selection regex like what testr uses. It will run
                   any tests that match on re.match() with the regex
 * **--smoke/-s**: Run all the tests tagged as smoke

There are also the **--blacklist-file** and **--whitelist-file** options that
let you pass a filepath to tempest run with the file format being a line
separated regex, with '#' used to signify the start of a comment on a line.
For example::

    # Regex file
    ^regex1 # Match these tests
    .*regex2 # Match those tests

The blacklist file will be used to construct a negative lookahead regex and
the whitelist file will simply OR all the regexes in the file. The whitelist
and blacklist file options are mutually exclusive so you can't use them
together. However, you can combine either with a normal regex or the *--smoke*
flag. When used with a blacklist file the generated regex will be combined to
something like::

    ^((?!black_regex1|black_regex2).)*$cli_regex1

When combined with a whitelist file all the regexes from the file and the CLI
regexes will be ORed.

You can also use the **--list-tests** option in conjunction with selection
arguments to list which tests will be run.

You can also use the **--load-list** option that lets you pass a filepath to
tempest run with the file format being in a non-regex format, similar to the
tests generated by the **--list-tests** option. You can specify target tests
by removing unnecessary tests from a list file which is generated from
**--list-tests** option.

Test Execution
==============
There are several options to control how the tests are executed. By default
tempest will run in parallel with a worker for each CPU present on the machine.
If you want to adjust the number of workers use the **--concurrency** option
and if you want to run tests serially use **--serial/-t**

Running with Workspaces
-----------------------
Tempest run enables you to run your tempest tests from any setup tempest
workspace it relies on you having setup a tempest workspace with either the
``tempest init`` or ``tempest workspace`` commands. Then using the
``--workspace`` CLI option you can specify which one of your workspaces you
want to run tempest from. Using this option you don't have to run Tempest
directly with you current working directory being the workspace, Tempest will
take care of managing everything to be executed from there.

Running from Anywhere
---------------------
Tempest run provides you with an option to execute tempest from anywhere on
your system. You are required to provide a config file in this case with the
``--config-file`` option. When run tempest will create a .testrepository
directory and a .testr.conf file in your current working directory. This way
you can use testr commands directly to inspect the state of the previous run.

Test Output
===========
By default tempest run's output to STDOUT will be generated using the
subunit-trace output filter. But, if you would prefer a subunit v2 stream be
output to STDOUT use the **--subunit** flag

Combining Runs
==============

There are certain situations in which you want to split a single run of tempest
across 2 executions of tempest run. (for example to run part of the tests
serially and others in parallel) To accomplish this but still treat the results
as a single run you can leverage the **--combine** option which will append
the current run's results with the previous runs.
"""

import io
import os
import sys
import tempfile
import threading

from cliff import command
from os_testr import regex_builder
from os_testr import subunit_trace
from oslo_serialization import jsonutils as json
import six
from testrepository.commands import run_argv

from tempest import clients
from tempest.cmd import cleanup_service
from tempest.cmd import init
from tempest.cmd import workspace
from tempest.common import credentials_factory as credentials
from tempest import config


CONF = config.CONF
SAVED_STATE_JSON = "saved_state.json"


class TempestRun(command.Command):

    def _set_env(self, config_file=None):
        if config_file:
            CONF.set_config_path(os.path.abspath(config_file))
        # NOTE(mtreinish): This is needed so that testr doesn't gobble up any
        # stacktraces on failure.
        if 'TESTR_PDB' in os.environ:
            return
        else:
            os.environ["TESTR_PDB"] = ""
        # NOTE(dims): most of our .testr.conf try to test for PYTHON
        # environment variable and fall back to "python", under python3
        # if it does not exist. we should set it to the python3 executable
        # to deal with this situation better for now.
        if six.PY3 and 'PYTHON' not in os.environ:
            os.environ['PYTHON'] = sys.executable

    def _create_testrepository(self):
        if not os.path.isdir('.testrepository'):
            returncode = run_argv(['testr', 'init'], sys.stdin, sys.stdout,
                                  sys.stderr)
            if returncode:
                sys.exit(returncode)

    def _create_testr_conf(self):
        top_level_path = os.path.dirname(os.path.dirname(__file__))
        discover_path = os.path.join(top_level_path, 'test_discover')
        file_contents = init.TESTR_CONF % (top_level_path, discover_path)
        with open('.testr.conf', 'w+') as testr_conf_file:
                testr_conf_file.write(file_contents)

    def take_action(self, parsed_args):
        returncode = 0
        if parsed_args.config_file:
            self._set_env(parsed_args.config_file)
        else:
            self._set_env()
        # Workspace execution mode
        if parsed_args.workspace:
            workspace_mgr = workspace.WorkspaceManager(
                parsed_args.workspace_path)
            path = workspace_mgr.get_workspace(parsed_args.workspace)
            if not path:
                sys.exit(
                    "The %r workspace isn't registered in "
                    "%r. Use 'tempest init' to "
                    "register the workspace." %
                    (parsed_args.workspace, workspace_mgr.path))
            os.chdir(path)
            # NOTE(mtreinish): tempest init should create a .testrepository dir
            # but since workspaces can be imported let's sanity check and
            # ensure that one is created
            self._create_testrepository()
        # Local execution mode
        elif os.path.isfile('.testr.conf'):
            # If you're running in local execution mode and there is not a
            # testrepository dir create one
            self._create_testrepository()
        # local execution with config file mode
        elif parsed_args.config_file:
            self._create_testr_conf()
            self._create_testrepository()
        else:
            print("No .testr.conf file was found for local execution")
            sys.exit(2)
        if parsed_args.state:
            self._init_state()
        else:
            pass

        if parsed_args.combine:
            temp_stream = tempfile.NamedTemporaryFile()
            return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin,
                                   temp_stream, sys.stderr)
            if return_code > 0:
                sys.exit(return_code)

        regex = self._build_regex(parsed_args)
        if parsed_args.list_tests:
            argv = ['tempest', 'list-tests', regex]
            returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
        else:
            options = self._build_options(parsed_args)
            returncode = self._run(regex, options)
            if returncode > 0:
                sys.exit(returncode)

        if parsed_args.combine:
            return_code = run_argv(['tempest', 'last', '--subunit'], sys.stdin,
                                   temp_stream, sys.stderr)
            if return_code > 0:
                sys.exit(return_code)
            returncode = run_argv(['tempest', 'load', temp_stream.name],
                                  sys.stdin, sys.stdout, sys.stderr)
        sys.exit(returncode)

    def get_description(self):
        return 'Run tempest'

    def _init_state(self):
        print("Initializing saved state.")
        data = {}
        self.global_services = cleanup_service.get_global_cleanup_services()
        self.admin_mgr = clients.Manager(
            credentials.get_configured_admin_credentials())
        admin_mgr = self.admin_mgr
        kwargs = {'data': data,
                  'is_dry_run': False,
                  'saved_state_json': data,
                  'is_preserve': False,
                  'is_save_state': True}
        for service in self.global_services:
            svc = service(admin_mgr, **kwargs)
            svc.run()

        with open(SAVED_STATE_JSON, 'w+') as f:
            f.write(json.dumps(data,
                    sort_keys=True, indent=2, separators=(',', ': ')))

    def get_parser(self, prog_name):
        parser = super(TempestRun, self).get_parser(prog_name)
        parser = self._add_args(parser)
        return parser

    def _add_args(self, parser):
        # workspace args
        parser.add_argument('--workspace', default=None,
                            help='Name of tempest workspace to use for running'
                                 ' tests. You can see a list of workspaces '
                                 'with tempest workspace list')
        parser.add_argument('--workspace-path', default=None,
                            dest='workspace_path',
                            help="The path to the workspace file, the default "
                                 "is ~/.tempest/workspace.yaml")
        # Configuration flags
        parser.add_argument('--config-file', default=None, dest='config_file',
                            help='Configuration file to run tempest with')
        # test selection args
        regex = parser.add_mutually_exclusive_group()
        regex.add_argument('--smoke', '-s', action='store_true',
                           help="Run the smoke tests only")
        regex.add_argument('--regex', '-r', default='',
                           help='A normal testr selection regex used to '
                                'specify a subset of tests to run')
        list_selector = parser.add_mutually_exclusive_group()
        list_selector.add_argument('--whitelist-file', '--whitelist_file',
                                   help="Path to a whitelist file, this file "
                                        "contains a separate regex on each "
                                        "newline.")
        list_selector.add_argument('--blacklist-file', '--blacklist_file',
                                   help='Path to a blacklist file, this file '
                                        'contains a separate regex exclude on '
                                        'each newline')
        list_selector.add_argument('--load-list', '--load_list',
                                   help='Path to a non-regex whitelist file, '
                                        'this file contains a seperate test '
                                        'on each newline. This command'
                                        'supports files created by the tempest'
                                        'run ``--list-tests`` command')
        # list only args
        parser.add_argument('--list-tests', '-l', action='store_true',
                            help='List tests',
                            default=False)
        # execution args
        parser.add_argument('--concurrency', '-w',
                            help="The number of workers to use, defaults to "
                                 "the number of cpus")
        parallel = parser.add_mutually_exclusive_group()
        parallel.add_argument('--parallel', dest='parallel',
                              action='store_true',
                              help='Run tests in parallel (this is the'
                                   ' default)')
        parallel.add_argument('--serial', '-t', dest='parallel',
                              action='store_false',
                              help='Run tests serially')
        parser.add_argument('--save-state', dest='state',
                            action='store_true',
                            help="To save the state of the cloud before "
                                 "running tempest.")
        # output args
        parser.add_argument("--subunit", action='store_true',
                            help='Enable subunit v2 output')
        parser.add_argument("--combine", action='store_true',
                            help='Combine the output of this run with the '
                                 "previous run's as a combined stream in the "
                                 "testr repository after it finish")

        parser.set_defaults(parallel=True)
        return parser

    def _build_regex(self, parsed_args):
        regex = ''
        if parsed_args.smoke:
            regex = 'smoke'
        elif parsed_args.regex:
            regex = parsed_args.regex
        if parsed_args.whitelist_file or parsed_args.blacklist_file:
            regex = regex_builder.construct_regex(parsed_args.blacklist_file,
                                                  parsed_args.whitelist_file,
                                                  regex, False)
        return regex

    def _build_options(self, parsed_args):
        options = []
        if parsed_args.subunit:
            options.append("--subunit")
        if parsed_args.parallel:
            options.append("--parallel")
        if parsed_args.concurrency:
            options.append("--concurrency=%s" % parsed_args.concurrency)
        if parsed_args.load_list:
            options.append("--load-list=%s" % parsed_args.load_list)
        return options

    def _run(self, regex, options):
        returncode = 0
        argv = ['tempest', 'run', regex] + options
        if '--subunit' in options:
            returncode = run_argv(argv, sys.stdin, sys.stdout, sys.stderr)
        else:
            argv.append('--subunit')
            stdin = io.StringIO()
            stdout_r, stdout_w = os.pipe()
            subunit_w = os.fdopen(stdout_w, 'wt')
            subunit_r = os.fdopen(stdout_r)
            returncodes = {}

            def run_argv_thread():
                returncodes['testr'] = run_argv(argv, stdin, subunit_w,
                                                sys.stderr)
                subunit_w.close()

            run_thread = threading.Thread(target=run_argv_thread)
            run_thread.start()
            returncodes['subunit-trace'] = subunit_trace.trace(
                subunit_r, sys.stdout, post_fails=True, print_failures=True)
            run_thread.join()
            subunit_r.close()
            # python version of pipefail
            if returncodes['testr']:
                returncode = returncodes['testr']
            elif returncodes['subunit-trace']:
                returncode = returncodes['subunit-trace']
        return returncode