attempt to get to flake8/hacking plugins
this is the infrastructure changes, plus 1 fix, to get us towards
flake8 and hacking plugins.
We need to remove an exit call in __init__ for config to get this
to pass. I think long term this gets addressed by config becoming
a test resource, but it will take some time at summit to figure
that out.
Change-Id: Iedd7931e85da5518cb2a8d58717e37b805267d2c
diff --git a/tempest/clients.py b/tempest/clients.py
index 678b595..38334d7 100644
--- a/tempest/clients.py
+++ b/tempest/clients.py
@@ -22,11 +22,14 @@
from tempest.services import botoclients
from tempest.services.compute.json.extensions_client import \
ExtensionsClientJSON
+from tempest.services.compute.json.fixed_ips_client import FixedIPsClientJSON
from tempest.services.compute.json.flavors_client import FlavorsClientJSON
from tempest.services.compute.json.floating_ips_client import \
FloatingIPsClientJSON
from tempest.services.compute.json.hosts_client import HostsClientJSON
from tempest.services.compute.json.images_client import ImagesClientJSON
+from tempest.services.compute.json.interfaces_client import \
+ InterfacesClientJSON
from tempest.services.compute.json.keypairs_client import KeyPairsClientJSON
from tempest.services.compute.json.limits_client import LimitsClientJSON
from tempest.services.compute.json.quotas_client import QuotasClientJSON
@@ -36,10 +39,13 @@
from tempest.services.compute.json.volumes_extensions_client import \
VolumesExtensionsClientJSON
from tempest.services.compute.xml.extensions_client import ExtensionsClientXML
+from tempest.services.compute.xml.fixed_ips_client import FixedIPsClientXML
from tempest.services.compute.xml.flavors_client import FlavorsClientXML
from tempest.services.compute.xml.floating_ips_client import \
FloatingIPsClientXML
from tempest.services.compute.xml.images_client import ImagesClientXML
+from tempest.services.compute.xml.interfaces_client import \
+ InterfacesClientXML
from tempest.services.compute.xml.keypairs_client import KeyPairsClientXML
from tempest.services.compute.xml.limits_client import LimitsClientXML
from tempest.services.compute.xml.quotas_client import QuotasClientXML
@@ -48,10 +54,10 @@
from tempest.services.compute.xml.servers_client import ServersClientXML
from tempest.services.compute.xml.volumes_extensions_client import \
VolumesExtensionsClientXML
-from tempest.services.identity.v3.json.endpoints_client import \
- EndPointClientJSON
from tempest.services.identity.json.identity_client import IdentityClientJSON
from tempest.services.identity.json.identity_client import TokenClientJSON
+from tempest.services.identity.v3.json.endpoints_client import \
+ EndPointClientJSON
from tempest.services.identity.v3.xml.endpoints_client import EndPointClientXML
from tempest.services.identity.xml.identity_client import IdentityClientXML
from tempest.services.identity.xml.identity_client import TokenClientXML
@@ -73,12 +79,6 @@
VolumeTypesClientXML
from tempest.services.volume.xml.snapshots_client import SnapshotsClientXML
from tempest.services.volume.xml.volumes_client import VolumesClientXML
-from tempest.services.compute.json.interfaces_client import \
- InterfacesClientJSON
-from tempest.services.compute.xml.interfaces_client import \
- InterfacesClientXML
-from tempest.services.compute.json.fixed_ips_client import FixedIPsClientJSON
-from tempest.services.compute.xml.fixed_ips_client import FixedIPsClientXML
LOG = logging.getLogger(__name__)
diff --git a/tempest/config.py b/tempest/config.py
index 9c41660..556e2a7 100644
--- a/tempest/config.py
+++ b/tempest/config.py
@@ -420,6 +420,9 @@
def __init__(self):
"""Initialize a configuration from a conf directory and conf file."""
+ config_files = []
+
+ failsafe_path = "/etc/tempest/" + self.DEFAULT_CONFIG_FILE
# Environment variables override defaults...
conf_dir = os.environ.get('TEMPEST_CONFIG_DIR',
@@ -431,16 +434,17 @@
if not (os.path.isfile(path) or
'TEMPEST_CONFIG_DIR' in os.environ or
'TEMPEST_CONFIG' in os.environ):
- path = "/etc/tempest/" + self.DEFAULT_CONFIG_FILE
+ path = failsafe_path
LOG.info("Using tempest config file %s" % path)
if not os.path.exists(path):
msg = "Config file %(path)s not found" % locals()
print >> sys.stderr, RuntimeError(msg)
- sys.exit(os.EX_NOINPUT)
+ else:
+ config_files.append(path)
- cfg.CONF([], project='tempest', default_config_files=[path])
+ cfg.CONF([], project='tempest', default_config_files=config_files)
register_compute_opts(cfg.CONF)
register_identity_opts(cfg.CONF)
diff --git a/tempest/testboto.py b/tempest/testboto.py
index cee8843..8faf8ab 100644
--- a/tempest/testboto.py
+++ b/tempest/testboto.py
@@ -282,9 +282,10 @@
@classmethod
def get_lfunction_gone(cls, obj):
- """ If the object is instance of a well know type returns back with
+ """If the object is instance of a well know type returns back with
with the correspoding function otherwise it assumes the obj itself
- is the function"""
+ is the function.
+ """
ec = cls.ec2_error_code
if isinstance(obj, ec2.instance.Instance):
colusure_matcher = ec.client.InvalidInstanceID.NotFound
diff --git a/tools/check_source.sh b/tools/check_source.sh
index 089ad70..2d66ba5 100755
--- a/tools/check_source.sh
+++ b/tools/check_source.sh
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
-python tools/hacking.py --ignore=E122,E125,E126 --repeat --show-source --exclude=.venv,.tox,dist,doc,openstack,*egg .
+flake8 --ignore=E122,E125,E126,H302,H304,H404,F --show-source --exclude=.git,.venv,.tox,dist,doc,openstack,*egg .
pep8_ret=$?
pyflakes tempest stress setup.py tools cli bin | grep "imported but unused"
diff --git a/tools/hacking.py b/tools/hacking.py
deleted file mode 100755
index 7e46b74..0000000
--- a/tools/hacking.py
+++ /dev/null
@@ -1,525 +0,0 @@
-#!/usr/bin/env python
-# vim: tabstop=4 shiftwidth=4 softtabstop=4
-
-# Copyright (c) 2012, Cloudscaling
-# All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-"""tempest HACKING file compliance testing
-
-built on top of pep8.py
-"""
-
-import inspect
-import logging
-import os
-import re
-import subprocess
-import sys
-import tokenize
-import warnings
-
-import pep8
-
-# Don't need this for testing
-logging.disable('LOG')
-
-#T1xx comments
-#T2xx except
-#T3xx imports
-#T4xx docstrings
-#T5xx dictionaries/lists
-#T6xx calling methods
-#T7xx localization
-#N8xx git commit messages
-
-IMPORT_EXCEPTIONS = ['sqlalchemy', 'migrate']
-DOCSTRING_TRIPLE = ['"""', "'''"]
-VERBOSE_MISSING_IMPORT = os.getenv('HACKING_VERBOSE_MISSING_IMPORT', 'False')
-
-
-# Monkey patch broken excluded filter in pep8
-# See https://github.com/jcrocholl/pep8/pull/111
-def excluded(self, filename):
- """
- Check if options.exclude contains a pattern that matches filename.
- """
- basename = os.path.basename(filename)
- return any((pep8.filename_match(filename, self.options.exclude,
- default=False),
- pep8.filename_match(basename, self.options.exclude,
- default=False)))
-
-
-def input_dir(self, dirname):
- """Check all files in this directory and all subdirectories."""
- dirname = dirname.rstrip('/')
- if self.excluded(dirname):
- return 0
- counters = self.options.report.counters
- verbose = self.options.verbose
- filepatterns = self.options.filename
- runner = self.runner
- for root, dirs, files in os.walk(dirname):
- if verbose:
- print('directory ' + root)
- counters['directories'] += 1
- for subdir in sorted(dirs):
- if self.excluded(os.path.join(root, subdir)):
- dirs.remove(subdir)
- for filename in sorted(files):
- # contain a pattern that matches?
- if ((pep8.filename_match(filename, filepatterns) and
- not self.excluded(filename))):
- runner(os.path.join(root, filename))
-
-
-def is_import_exception(mod):
- return (mod in IMPORT_EXCEPTIONS or
- any(mod.startswith(m + '.') for m in IMPORT_EXCEPTIONS))
-
-
-def import_normalize(line):
- # convert "from x import y" to "import x.y"
- # handle "from x import y as z" to "import x.y as z"
- split_line = line.split()
- if ("import" in line and line.startswith("from ") and "," not in line and
- split_line[2] == "import" and split_line[3] != "*" and
- split_line[1] != "__future__" and
- (len(split_line) == 4 or
- (len(split_line) == 6 and split_line[4] == "as"))):
- return "import %s.%s" % (split_line[1], split_line[3])
- else:
- return line
-
-
-def tempest_todo_format(physical_line):
- """Check for 'TODO()'.
-
- tempest HACKING guide recommendation for TODO:
- Include your name with TODOs as in "#TODO(termie)"
- T101
- """
- pos = physical_line.find('TODO')
- pos1 = physical_line.find('TODO(')
- pos2 = physical_line.find('#') # make sure it's a comment
- if (pos != pos1 and pos2 >= 0 and pos2 < pos):
- return pos, "T101: Use TODO(NAME)"
-
-
-def tempest_except_format(logical_line):
- """Check for 'except:'.
-
- tempest HACKING guide recommends not using except:
- Do not write "except:", use "except Exception:" at the very least
- T201
- """
- if logical_line.startswith("except:"):
- yield 6, "T201: no 'except:' at least use 'except Exception:'"
-
-
-def tempest_except_format_assert(logical_line):
- """Check for 'assertRaises(Exception'.
-
- tempest HACKING guide recommends not using assertRaises(Exception...):
- Do not use overly broad Exception type
- T202
- """
- if logical_line.startswith("self.assertRaises(Exception"):
- yield 1, "T202: assertRaises Exception too broad"
-
-
-def tempest_one_import_per_line(logical_line):
- """Check for import format.
-
- tempest HACKING guide recommends one import per line:
- Do not import more than one module per line
-
- Examples:
- BAD: from tempest.common.rest_client import RestClient, RestClientXML
- T301
- """
- pos = logical_line.find(',')
- parts = logical_line.split()
- if (pos > -1 and (parts[0] == "import" or
- parts[0] == "from" and parts[2] == "import") and
- not is_import_exception(parts[1])):
- yield pos, "T301: one import per line"
-
-_missingImport = set([])
-
-
-def tempest_import_module_only(logical_line):
- """Check for import module only.
-
- tempest HACKING guide recommends importing only modules:
- Do not import objects, only modules
- T302 import only modules
- T303 Invalid Import
- T304 Relative Import
- """
- def importModuleCheck(mod, parent=None, added=False):
- """
- If can't find module on first try, recursively check for relative
- imports
- """
- current_path = os.path.dirname(pep8.current_file)
- try:
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', DeprecationWarning)
- valid = True
- if parent:
- if is_import_exception(parent):
- return
- parent_mod = __import__(parent, globals(), locals(),
- [mod], -1)
- valid = inspect.ismodule(getattr(parent_mod, mod))
- else:
- __import__(mod, globals(), locals(), [], -1)
- valid = inspect.ismodule(sys.modules[mod])
- if not valid:
- if added:
- sys.path.pop()
- added = False
- return logical_line.find(mod), ("T304: No "
- "relative imports. "
- "'%s' is a relative "
- "import"
- % logical_line)
- return logical_line.find(mod), ("T302: import only"
- " modules. '%s' does not "
- "import a module"
- % logical_line)
-
- except (ImportError, NameError) as exc:
- if not added:
- added = True
- sys.path.append(current_path)
- return importModuleCheck(mod, parent, added)
- else:
- name = logical_line.split()[1]
- if name not in _missingImport:
- if VERBOSE_MISSING_IMPORT != 'False':
- print >> sys.stderr, ("ERROR: import '%s' in %s "
- "failed: %s" %
- (name, pep8.current_file, exc))
- _missingImport.add(name)
- added = False
- sys.path.pop()
- return
-
- except AttributeError:
- # Invalid import
- return logical_line.find(mod), ("T303: Invalid import, "
- "AttributeError raised")
-
- # convert "from x import y" to " import x.y"
- # convert "from x import y as z" to " import x.y"
- import_normalize(logical_line)
- split_line = logical_line.split()
-
- if (logical_line.startswith("import ") and "," not in logical_line and
- (len(split_line) == 2 or
- (len(split_line) == 4 and split_line[2] == "as"))):
- mod = split_line[1]
- rval = importModuleCheck(mod)
- if rval is not None:
- yield rval
-
- # TODO(jogo) handle "from x import *"
-
-#TODO(jogo): import template: T305
-
-
-def tempest_import_alphabetical(logical_line, line_number, lines):
- """Check for imports in alphabetical order.
-
- Tempest HACKING guide recommendation for imports:
- imports in human alphabetical order
- T306
- """
- # handle import x
- # use .lower since capitalization shouldn't dictate order
- split_line = import_normalize(logical_line.strip()).lower().split()
- split_previous = import_normalize(lines[
- line_number - 2]).strip().lower().split()
- # with or without "as y"
- length = [2, 4]
- if (len(split_line) in length and len(split_previous) in length and
- split_line[0] == "import" and split_previous[0] == "import"):
- if split_line[1] < split_previous[1]:
- yield (0, "T306: imports not in alphabetical order"
- " (%s, %s)"
- % (split_previous[1], split_line[1]))
-
-
-def tempest_docstring_start_space(physical_line):
- """Check for docstring not start with space.
-
- tempest HACKING guide recommendation for docstring:
- Docstring should not start with space
- T401
- """
- pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start
- end = max([physical_line[-4:-1] == i for i in DOCSTRING_TRIPLE]) # end
- if (pos != -1 and end and len(physical_line) > pos + 4):
- if (physical_line[pos + 3] == ' '):
- return (pos, "T401: one line docstring should not start"
- " with a space")
-
-
-def tempest_docstring_one_line(physical_line):
- """Check one line docstring end.
-
- tempest HACKING guide recommendation for one line docstring:
- A one line docstring looks like this and ends in a period.
- T402
- """
- pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start
- end = max([physical_line[-4:-1] == i for i in DOCSTRING_TRIPLE]) # end
- if (pos != -1 and end and len(physical_line) > pos + 4):
- if (physical_line[-5] != '.'):
- return pos, "T402: one line docstring needs a period"
-
-
-def tempest_docstring_multiline_end(physical_line):
- """Check multi line docstring end.
-
- Tempest HACKING guide recommendation for docstring:
- Docstring should end on a new line
- T403
- """
- pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE]) # start
- if (pos != -1 and len(physical_line) == pos):
- if (physical_line[pos + 3] == ' '):
- return (pos, "T403: multi line docstring end on new line")
-
-
-def tempest_no_test_docstring(physical_line, previous_logical, filename):
- """Check that test_ functions don't have docstrings
-
- This ensure we get better results out of tempest, instead
- of them being hidden behind generic descriptions of the
- functions.
-
- T404
- """
- if "tempest/test" in filename:
- pos = max([physical_line.find(i) for i in DOCSTRING_TRIPLE])
- if pos != -1:
- if previous_logical.startswith("def test_"):
- return (pos, "T404: test functions must "
- "not have doc strings")
-
-SKIP_DECORATOR = '@testtools.skip('
-
-
-def tempest_skip_bugs(physical_line):
- """Check skip lines for proper bug entries
-
- T601: Bug not in skip line
- T602: Bug in message formatted incorrectly
- """
-
- pos = physical_line.find(SKIP_DECORATOR)
-
- skip_re = re.compile(r'^\s*@testtools.skip.*')
-
- if pos != -1 and skip_re.match(physical_line):
- bug = re.compile(r'^.*\bbug\b.*', re.IGNORECASE)
- if bug.match(physical_line) is None:
- return (pos, 'T601: skips must have an associated bug')
-
- bug_re = re.compile(r'.*skip\(.*Bug\s\#\d+', re.IGNORECASE)
-
- if bug_re.match(physical_line) is None:
- return (pos, 'T602: Bug number formatted incorrectly')
-
-
-FORMAT_RE = re.compile("%(?:"
- "%|" # Ignore plain percents
- "(\(\w+\))?" # mapping key
- "([#0 +-]?" # flag
- "(?:\d+|\*)?" # width
- "(?:\.\d+)?" # precision
- "[hlL]?" # length mod
- "\w))") # type
-
-
-class LocalizationError(Exception):
- pass
-
-
-def check_i18n():
- """Generator that checks token stream for localization errors.
-
- Expects tokens to be ``send``ed one by one.
- Raises LocalizationError if some error is found.
- """
- while True:
- try:
- token_type, text, _, _, line = yield
- except GeneratorExit:
- return
- if (token_type == tokenize.NAME and text == "_" and
- not line.startswith('def _(msg):')):
-
- while True:
- token_type, text, start, _, _ = yield
- if token_type != tokenize.NL:
- break
- if token_type != tokenize.OP or text != "(":
- continue # not a localization call
-
- format_string = ''
- while True:
- token_type, text, start, _, _ = yield
- if token_type == tokenize.STRING:
- format_string += eval(text)
- elif token_type == tokenize.NL:
- pass
- else:
- break
-
- if not format_string:
- raise LocalizationError(start,
- "T701: Empty localization "
- "string")
- if token_type != tokenize.OP:
- raise LocalizationError(start,
- "T701: Invalid localization "
- "call")
- if text != ")":
- if text == "%":
- raise LocalizationError(start,
- "T702: Formatting "
- "operation should be outside"
- " of localization method call")
- elif text == "+":
- raise LocalizationError(start,
- "T702: Use bare string "
- "concatenation instead of +")
- else:
- raise LocalizationError(start,
- "T702: Argument to _ must"
- " be just a string")
-
- format_specs = FORMAT_RE.findall(format_string)
- positional_specs = [(key, spec) for key, spec in format_specs
- if not key and spec]
- # not spec means %%, key means %(smth)s
- if len(positional_specs) > 1:
- raise LocalizationError(start,
- "T703: Multiple positional "
- "placeholders")
-
-
-def tempest_localization_strings(logical_line, tokens):
- """Check localization in line.
-
- T701: bad localization call
- T702: complex expression instead of string as argument to _()
- T703: multiple positional placeholders
- """
-
- gen = check_i18n()
- next(gen)
- try:
- map(gen.send, tokens)
- gen.close()
- except LocalizationError as e:
- yield e.args
-
-#TODO(jogo) Dict and list objects
-
-current_file = ""
-
-
-def readlines(filename):
- """Record the current file being tested."""
- pep8.current_file = filename
- return open(filename).readlines()
-
-
-def add_tempest():
- """Monkey patch in tempest guidelines.
-
- Look for functions that start with tempest_ and have arguments
- and add them to pep8 module
- Assumes you know how to write pep8.py checks
- """
- for name, function in globals().items():
- if not inspect.isfunction(function):
- continue
- args = inspect.getargspec(function)[0]
- if args and name.startswith("tempest"):
- exec("pep8.%s = %s" % (name, name))
-
-
-def once_git_check_commit_title():
- """Check git commit messages.
-
- tempest HACKING recommends not referencing a bug or blueprint
- in first line, it should provide an accurate description of the change
- T801
- T802 Title limited to 50 chars
- """
- #Get title of most recent commit
-
- subp = subprocess.Popen(['git', 'log', '--no-merges', '--pretty=%s', '-1'],
- stdout=subprocess.PIPE)
- title = subp.communicate()[0]
- if subp.returncode:
- raise Exception("git log failed with code %s" % subp.returncode)
-
- #From https://github.com/openstack/openstack-ci-puppet
- # /blob/master/modules/gerrit/manifests/init.pp#L74
- #Changeid|bug|blueprint
- git_keywords = (r'(I[0-9a-f]{8,40})|'
- '([Bb]ug|[Ll][Pp])[\s\#:]*(\d+)|'
- '([Bb]lue[Pp]rint|[Bb][Pp])[\s\#:]*([A-Za-z0-9\\-]+)')
- GIT_REGEX = re.compile(git_keywords)
-
- error = False
- #NOTE(jogo) if match regex but over 3 words, acceptable title
- if GIT_REGEX.search(title) is not None and len(title.split()) <= 3:
- print ("T801: git commit title ('%s') should provide an accurate "
- "description of the change, not just a reference to a bug "
- "or blueprint" % title.strip())
- error = True
- if len(title.decode('utf-8')) > 72:
- print ("T802: git commit title ('%s') should be under 50 chars"
- % title.strip())
- error = True
- return error
-
-if __name__ == "__main__":
- #include tempest path
- sys.path.append(os.getcwd())
- #Run once tests (not per line)
- once_error = once_git_check_commit_title()
- #TEMPEST error codes start with a T
- pep8.ERRORCODE_REGEX = re.compile(r'[EWT]\d{3}')
- add_tempest()
- pep8.current_file = current_file
- pep8.readlines = readlines
- pep8.StyleGuide.excluded = excluded
- pep8.StyleGuide.input_dir = input_dir
- try:
- pep8._main()
- sys.exit(once_error)
- finally:
- if len(_missingImport) > 0:
- print >> sys.stderr, ("%i imports missing in this test environment"
- % len(_missingImport))
diff --git a/tools/test-requires b/tools/test-requires
index f701dab..3d4c2d6 100644
--- a/tools/test-requires
+++ b/tools/test-requires
@@ -1,5 +1,4 @@
-pep8==1.3.3
-pylint==0.19
+flake8
+hacking
#TODO(afazekas): ensure pg_config installed
psycopg2
-pyflakes