tools/hacking.py

#!/usr/bin/env python
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2012, Cloudscaling
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""tempest HACKING file compliance testing

built on top of pep8.py
"""

import fnmatch
import inspect
import logging
import os
import re
import subprocess
import sys
import tokenize
import warnings

import pep8

# Don't need this for testing
logging.disable('LOG')

#N1xx comments
#N2xx except
#N3xx imports
#N4xx docstrings
#N5xx dictionaries/lists
#N6xx calling methods
#N7xx localization
#N8xx git commit messages

IMPORT_EXCEPTIONS = ['sqlalchemy', 'migrate']
DOCSTRING_TRIPLE = ['"""', "'''"]
VERBOSE_MISSING_IMPORT = os.getenv('HACKING_VERBOSE_MISSING_IMPORT', 'False')


# Monkey patch broken excluded filter in pep8
# See https://github.com/jcrocholl/pep8/pull/111
def excluded(self, filename):
    """
    Check if options.exclude contains a pattern that matches filename.
    """
    basename = os.path.basename(filename)
    return any((pep8.filename_match(filename, self.options.exclude,
                                    default=False),
                pep8.filename_match(basename, self.options.exclude,
                                    default=False)))


def input_dir(self, dirname):
    """Check all files in this directory and all subdirectories."""
    dirname = dirname.rstrip('/')
    if self.excluded(dirname):
        return 0
    counters = self.options.report.counters
    verbose = self.options.verbose
    filepatterns = self.options.filename
    runner = self.runner
    for root, dirs, files in os.walk(dirname):
        if verbose:
            print('directory ' + root)
        counters['directories'] += 1
        for subdir in sorted(dirs):
            if self.excluded(os.path.join(root, subdir)):
                dirs.remove(subdir)
        for filename in sorted(files):
            # contain a pattern that matches?
            if ((pep8.filename_match(filename, filepatterns) and
                 not self.excluded(filename))):
                runner(os.path.join(root, filename))


def is_import_exception(mod):
    return (mod in IMPORT_EXCEPTIONS or
            any(mod.startswith(m + '.') for m in IMPORT_EXCEPTIONS))


def import_normalize(line):
    # convert "from x import y" to "import x.y"
    # handle "from x import y as z" to "import x.y as z"
    split_line = line.split()
    if ("import" in line and line.startswith("from ") and "," not in line and
        split_line[2] == "import" and split_line[3] != "*" and
        split_line[1] != "__future__" and
       (len(split_line) == 4 or
       (len(split_line) == 6 and split_line[4] == "as"))):
        return "import %s.%s" % (split_line[1], split_line[3])
    else:
        return line


def tempest_todo_format(physical_line):
    """Check for 'TODO()'.

    tempest HACKING guide recommendation for TODO:
    Include your name with TODOs as in "#TODO(termie)"
    N101
    """
    pos = physical_line.find('TODO')
    pos1 = physical_line.find('TODO(')
    pos2 = physical_line.find('#')  # make sure it's a comment
    if (pos != pos1 and pos2 >= 0 and pos2 < pos):
        return pos, "TEMPEST N101: Use TODO(NAME)"


def tempest_except_format(logical_line):
    """Check for 'except:'.

    tempest HACKING guide recommends not using except:
    Do not write "except:", use "except Exception:" at the very least
    N201
    """
    if logical_line.startswith("except:"):
        yield 6, "TEMPEST N201: no 'except:' at least use 'except Exception:'"


def tempest_except_format_assert(logical_line):
    """Check for 'assertRaises(Exception'.

    tempest HACKING guide recommends not using assertRaises(Exception...):
    Do not use overly broad Exception type
    N202
    """
    if logical_line.startswith("self.assertRaises(Exception"):
        yield 1, "TEMPEST N202: assertRaises Exception too broad"


def tempest_one_import_per_line(logical_line):
    """Check for import format.

    tempest HACKING guide recommends one import per line:
    Do not import more than one module per line

    Examples:
    BAD: from tempest.common.rest_client import RestClient, RestClientXML
    N301
    """
    pos = logical_line.find(',')
    parts = logical_line.split()
    if (pos > -1 and (parts[0] == "import" or
                      parts[0] == "from" and parts[2] == "import") and
        not is_import_exception(parts[1])):
        yield pos, "TEMPEST N301: one import per line"

_missingImport = set([])


def tempest_import_module_only(logical_line):
    """Check for import module only.

    tempest HACKING guide recommends importing only modules:
    Do not import objects, only modules
    N302 import only modules
    N303 Invalid Import
    N304 Relative Import
    """
    def importModuleCheck(mod, parent=None, added=False):
        """
        If can't find module on first try, recursively check for relative
        imports
        """
        current_path = os.path.dirname(pep8.current_file)
        try:
            with warnings.catch_warnings():
                warnings.simplefilter('ignore', DeprecationWarning)
                valid = True
                if parent:
                    if is_import_exception(parent):
                        return
                    parent_mod = __import__(parent, globals(), locals(),
                                            [mod], -1)
                    valid = inspect.ismodule(getattr(parent_mod, mod))
                else:
                    __import__(mod, globals(), locals(), [], -1)
                    valid = inspect.ismodule(sys.modules[mod])
                if not valid:
                    if added:
                        sys.path.pop()
                        added = False
                        return logical_line.find(mod), ("TEMPEST N304: No "
                                                        "relative  imports. "
                                                        "'%s' is a relative "
                                                        "import"
                                                        % logical_line)
                    return logical_line.find(mod), ("TEMPEST N302: import only"
                                                    " modules. '%s' does not "
                                                    "import a module"
                                                    % logical_line)

        except (ImportError, NameError) as exc:
            if not added:
                added = True
                sys.path.append(current_path)
                return importModuleCheck(mod, parent, added)
            else:
                name = logical_line.split()[1]
                if name not in _missingImport:
                    if VERBOSE_MISSING_IMPORT != 'False':
                        print >> sys.stderr, ("ERROR: import '%s' in %s "
                                              "failed: %s" %
                                              (name, pep8.current_file, exc))
                    _missingImport.add(name)
                added = False
                sys.path.pop()
                return

        except AttributeError:
            # Invalid import
            return logical_line.find(mod), ("TEMPEST N303: Invalid import, "
                                            "AttributeError raised")

    # convert "from x import y" to " import x.y"
    # convert "from x import y as z" to " import x.y"
    import_normalize(logical_line)
    split_line = logical_line.split()

    if (logical_line.startswith("import ") and "," not in logical_line and
            (len(split_line) == 2 or
            (len(split_line) == 4 and split_line[2] == "as"))):
        mod = split_line[1]
        rval = importModuleCheck(mod)
        if rval is not None:
            yield rval

    # TODO(jogo) handle "from x import *"

#TODO(jogo): import template: N305


def tempest_import_alphabetical(logical_line, line_number, lines):
    """Check for imports in alphabetical order.

    Tempest HACKING guide recommendation for imports:
    imports in human alphabetical order
    N306
    """
    # handle import x
    # use .lower since capitalization shouldn't dictate order
    split_line = import_normalize(logical_line.strip()).lower().split()
    split_previous = import_normalize(lines[
                                      line_number - 2]).strip().lower().split()
    # with or without "as y"
    length = [2, 4]
    if (len(split_line) in length and len(split_previous) in length and
        split_line[0] == "import" and split_previous[0] == "import"):
        if split_line[1] < split_previous[1]:
            yield (0, "TEMPEST N306: imports not in alphabetical order"
                      " (%s, %s)"
                      % (split_previous[1], split_line[1]))


def tempest_docstring_start_space(physical_line):
    """Check for docstring not start with space.

    tempest HACKING guide recommendation for docstring:
    Docstring should not start with space
    N401
    """
    pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE])  # start
    if (pos != -1 and len(physical_line) > pos + 1):
        if (physical_line[pos + 3] == ' '):
            return (pos, "TEMPEST N401: one line docstring should not start"
                         " with a space")


def tempest_docstring_one_line(physical_line):
    """Check one line docstring end.

    tempest HACKING guide recommendation for one line docstring:
    A one line docstring looks like this and ends in a period.
    N402
    """
    pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE])  # start
    end = max([physical_line[-4:-1] == i for i in DOCSTRING_TRIPLE])  # end
    if (pos != -1 and end and len(physical_line) > pos + 4):
        if (physical_line[-5] != '.'):
            return pos, "TEMPEST N402: one line docstring needs a period"


def tempest_docstring_multiline_end(physical_line):
    """Check multi line docstring end.

    Tempest HACKING guide recommendation for docstring:
    Docstring should end on a new line
    N403
    """
    pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE])  # start
    if (pos != -1 and len(physical_line) == pos):
        if (physical_line[pos + 3] == ' '):
            return (pos, "TEMPEST N403: multi line docstring end on new line")


FORMAT_RE = re.compile("%(?:"
                       "%|"           # Ignore plain percents
                       "(\(\w+\))?"   # mapping key
                       "([#0 +-]?"    # flag
                       "(?:\d+|\*)?"  # width
                       "(?:\.\d+)?"   # precision
                       "[hlL]?"       # length mod
                       "\w))")        # type


class LocalizationError(Exception):
    pass


def check_i18n():
    """Generator that checks token stream for localization errors.

    Expects tokens to be ``send``ed one by one.
    Raises LocalizationError if some error is found.
    """
    while True:
        try:
            token_type, text, _, _, line = yield
        except GeneratorExit:
            return
        if (token_type == tokenize.NAME and text == "_" and
            not line.startswith('def _(msg):')):

            while True:
                token_type, text, start, _, _ = yield
                if token_type != tokenize.NL:
                    break
            if token_type != tokenize.OP or text != "(":
                continue  # not a localization call

            format_string = ''
            while True:
                token_type, text, start, _, _ = yield
                if token_type == tokenize.STRING:
                    format_string += eval(text)
                elif token_type == tokenize.NL:
                    pass
                else:
                    break

            if not format_string:
                raise LocalizationError(start,
                                        "TEMEPST N701: Empty localization "
                                        "string")
            if token_type != tokenize.OP:
                raise LocalizationError(start,
                                        "TEMPEST N701: Invalid localization "
                                        "call")
            if text != ")":
                if text == "%":
                    raise LocalizationError(start,
                                            "TEMPEST N702: Formatting "
                                            "operation should be outside"
                                            " of localization method call")
                elif text == "+":
                    raise LocalizationError(start,
                                            "TEMPEST N702: Use bare string "
                                            "concatenation instead of +")
                else:
                    raise LocalizationError(start,
                                            "TEMPEST N702: Argument to _ must"
                                            " be just a string")

            format_specs = FORMAT_RE.findall(format_string)
            positional_specs = [(key, spec) for key, spec in format_specs
                                if not key and spec]
            # not spec means %%, key means %(smth)s
            if len(positional_specs) > 1:
                raise LocalizationError(start,
                                        "TEMPEST N703: Multiple positional "
                                        "placeholders")


def tempest_localization_strings(logical_line, tokens):
    """Check localization in line.

    N701: bad localization call
    N702: complex expression instead of string as argument to _()
    N703: multiple positional placeholders
    """

    gen = check_i18n()
    next(gen)
    try:
        map(gen.send, tokens)
        gen.close()
    except LocalizationError as e:
        yield e.args

#TODO(jogo) Dict and list objects

current_file = ""


def readlines(filename):
    """Record the current file being tested."""
    pep8.current_file = filename
    return open(filename).readlines()


def add_tempest():
    """Monkey patch in tempest guidelines.

    Look for functions that start with tempest_  and have arguments
    and add them to pep8 module
    Assumes you know how to write pep8.py checks
    """
    for name, function in globals().items():
        if not inspect.isfunction(function):
            continue
        args = inspect.getargspec(function)[0]
        if args and name.startswith("tempest"):
            exec("pep8.%s = %s" % (name, name))


def once_git_check_commit_title():
    """Check git commit messages.

    tempest HACKING recommends not referencing a bug or blueprint
    in first line, it should provide an accurate description of the change
    N801
    N802 Title limited to 50 chars
    """
    #Get title of most recent commit

    subp = subprocess.Popen(['git', 'log', '--no-merges', '--pretty=%s', '-1'],
                            stdout=subprocess.PIPE)
    title = subp.communicate()[0]
    if subp.returncode:
        raise Exception("git log failed with code %s" % subp.returncode)

    #From https://github.com/openstack/openstack-ci-puppet
    #       /blob/master/modules/gerrit/manifests/init.pp#L74
    #Changeid|bug|blueprint
    git_keywords = (r'(I[0-9a-f]{8,40})|'
                    '([Bb]ug|[Ll][Pp])[\s\#:]*(\d+)|'
                    '([Bb]lue[Pp]rint|[Bb][Pp])[\s\#:]*([A-Za-z0-9\\-]+)')
    GIT_REGEX = re.compile(git_keywords)

    error = False
    #NOTE(jogo) if match regex but over 3 words, acceptable title
    if GIT_REGEX.search(title) is not None and len(title.split()) <= 3:
        print ("N801: git commit title ('%s') should provide an accurate "
               "description of the change, not just a reference to a bug "
               "or blueprint" % title.strip())
        error = True
    if len(title.decode('utf-8')) > 72:
        print ("N802: git commit title ('%s') should be under 50 chars"
               % title.strip())
        error = True
    return error

if __name__ == "__main__":
    #include tempest path
    sys.path.append(os.getcwd())
    #Run once tests (not per line)
    once_error = once_git_check_commit_title()
    #TEMPEST error codes start with an N
    pep8.ERRORCODE_REGEX = re.compile(r'[EWN]\d{3}')
    add_tempest()
    pep8.current_file = current_file
    pep8.readlines = readlines
    pep8.StyleGuide.excluded = excluded
    pep8.StyleGuide.input_dir = input_dir
    try:
        pep8._main()
        sys.exit(once_error)
    finally:
        if len(_missingImport) > 0:
            print >> sys.stderr, ("%i imports missing in this test environment"
                                  % len(_missingImport))