Tests for parsers are added
Tests for full parser and simplified reference parser
are added. The new tests simultaneously act as documentation
for parsers. Also some mostly cosmetic changes are applied
to parser building functions and Parser() class.
diff --git a/reclass/values/parser.py b/reclass/values/parser.py
index 27e6d2d..3f7ac1f 100644
--- a/reclass/values/parser.py
+++ b/reclass/values/parser.py
@@ -16,9 +16,13 @@
from .scaitem import ScaItem
from reclass.errors import ParseError
-from reclass.values.parser_funcs import STR, REF, INV
+from reclass.values.parser_funcs import tags
import reclass.values.parser_funcs as parsers
+import collections
+import six
+
+
class Parser(object):
def __init__(self):
@@ -43,7 +47,7 @@
def parse(self, value, settings):
def full_parse():
try:
- return self.ref_parser.parseString(value).asList()
+ return self.ref_parser.parseString(value)
except pp.ParseException as e:
raise ParseError(e.msg, e.line, e.col, e.lineno)
@@ -55,30 +59,31 @@
return ScaItem(value, self._settings)
elif sentinel_count == 1: # speed up: try a simple reference
try:
- tokens = self.simple_ref_parser.parseString(value).asList()
+ tokens = self.simple_ref_parser.parseString(value)
except pp.ParseException:
tokens = full_parse() # fall back on the full parser
else:
tokens = full_parse() # use the full parser
+ tokens = parsers.listify(tokens)
items = self._create_items(tokens)
if len(items) == 1:
return items[0]
return CompItem(items, self._settings)
- _create_dict = { STR: (lambda s, v: ScaItem(v, s._settings)),
- REF: (lambda s, v: s._create_ref(v)),
- INV: (lambda s, v: s._create_inv(v)) }
+ _item_builders = {tags.STR: (lambda s, v: ScaItem(v, s._settings)),
+ tags.REF: (lambda s, v: s._create_ref(v)),
+ tags.INV: (lambda s, v: s._create_inv(v)) }
def _create_items(self, tokens):
- return [ self._create_dict[t](self, v) for t, v in tokens ]
+ return [self._item_builders[t](self, v) for t, v in tokens ]
def _create_ref(self, tokens):
- items = [ self._create_dict[t](self, v) for t, v in tokens ]
+ items = [ self._item_builders[t](self, v) for t, v in tokens ]
return RefItem(items, self._settings)
def _create_inv(self, tokens):
- items = [ ScaItem(v, self._settings) for t, v in tokens ]
+ items = [ScaItem(v, self._settings) for t, v in tokens]
if len(items) == 1:
return InvItem(items[0], self._settings)
return InvItem(CompItem(items), self._settings)
diff --git a/reclass/values/parser_funcs.py b/reclass/values/parser_funcs.py
index f702910..db34ceb 100644
--- a/reclass/values/parser_funcs.py
+++ b/reclass/values/parser_funcs.py
@@ -8,12 +8,13 @@
from __future__ import print_function
from __future__ import unicode_literals
-import pyparsing as pp
+import collections
+import enum
import functools
+import pyparsing as pp
+import six
-STR = 1
-REF = 2
-INV = 3
+tags = enum.Enum('Tags', ['STR', 'REF', 'INV'])
_OBJ = 'OBJ'
_LOGICAL = 'LOGICAL'
@@ -42,6 +43,20 @@
tokens[0] = (tag, token)
return functools.partial(inner, tag)
+def _asList(x):
+ if isinstance(x, pp.ParseResults):
+ return x.asList()
+ return x
+
+def listify(w, modifier=_asList):
+ if (isinstance(w, collections.Iterable) and
+ not isinstance(w, six.string_types)):
+ cls = type(w)
+ if cls == pp.ParseResults:
+ cls = list
+ return cls([listify(x) for x in w])
+ return modifier(w)
+
def get_expression_parser():
sign = pp.Optional(pp.Literal('-'))
number = pp.Word(pp.nums)
@@ -117,10 +132,10 @@
ref_escape_close = pp.Literal(_REF_ESCAPE_CLOSE).setParseAction(pp.replaceWith(_REF_CLOSE))
ref_text = pp.CharsNotIn(_REF_EXCLUDES) | pp.CharsNotIn(_REF_CLOSE_FIRST, exact=1)
ref_content = pp.Combine(pp.OneOrMore(ref_not_open + ref_not_close + ref_text))
- ref_string = pp.MatchFirst([double_escape, ref_escape_open, ref_escape_close, ref_content]).setParseAction(_tag_with(STR))
+ ref_string = pp.MatchFirst([double_escape, ref_escape_open, ref_escape_close, ref_content]).setParseAction(_tag_with(tags.STR))
ref_item = pp.Forward()
ref_items = pp.OneOrMore(ref_item)
- reference = (ref_open + pp.Group(ref_items) + ref_close).setParseAction(_tag_with(REF))
+ reference = (ref_open + pp.Group(ref_items) + ref_close).setParseAction(_tag_with(tags.REF))
ref_item << (reference | ref_string)
inv_open = pp.Literal(_INV_OPEN).suppress()
@@ -131,13 +146,17 @@
inv_escape_close = pp.Literal(_INV_ESCAPE_CLOSE).setParseAction(pp.replaceWith(_INV_CLOSE))
inv_text = pp.CharsNotIn(_INV_CLOSE_FIRST)
inv_content = pp.Combine(pp.OneOrMore(inv_not_close + inv_text))
- inv_string = pp.MatchFirst([double_escape, inv_escape_open, inv_escape_close, inv_content]).setParseAction(_tag_with(STR))
+ inv_string = pp.MatchFirst(
+ [double_escape, inv_escape_open, inv_escape_close, inv_content]
+ ).setParseAction(_tag_with(tags.STR))
inv_items = pp.OneOrMore(inv_string)
- export = (inv_open + pp.Group(inv_items) + inv_close).setParseAction(_tag_with(INV))
+ export = (inv_open + pp.Group(inv_items) + inv_close).setParseAction(_tag_with(tags.INV))
text = pp.CharsNotIn(_EXCLUDES) | pp.CharsNotIn('', exact=1)
content = pp.Combine(pp.OneOrMore(ref_not_open + inv_not_open + text))
- string = pp.MatchFirst([double_escape, ref_escape_open, inv_escape_open, content]).setParseAction(_tag_with(STR))
+ string = pp.MatchFirst(
+ [double_escape, ref_escape_open, inv_escape_open, content]
+ ).setParseAction(_tag_with(tags.STR))
item = reference | export | string
line = pp.OneOrMore(item) + s_end
@@ -151,9 +170,9 @@
INV_OPEN, INV_CLOSE = settings.export_sentinels
EXCLUDES = ESCAPE + REF_OPEN + REF_CLOSE + INV_OPEN + INV_CLOSE
- string = pp.CharsNotIn(EXCLUDES).setParseAction(_tag_with(STR))
+ string = pp.CharsNotIn(EXCLUDES).setParseAction(_tag_with(tags.STR))
ref_open = pp.Literal(REF_OPEN).suppress()
ref_close = pp.Literal(REF_CLOSE).suppress()
- reference = (ref_open + pp.Group(string) + ref_close).setParseAction(_tag_with(REF))
+ reference = (ref_open + pp.Group(string) + ref_close).setParseAction(_tag_with(tags.REF))
line = pp.StringStart() + pp.Optional(string) + reference + pp.Optional(string) + s_end
return line.leaveWhitespace()
diff --git a/reclass/values/tests/test_parser_functions.py b/reclass/values/tests/test_parser_functions.py
new file mode 100644
index 0000000..a660c76
--- /dev/null
+++ b/reclass/values/tests/test_parser_functions.py
@@ -0,0 +1,116 @@
+from reclass import settings
+from reclass.values import parser_funcs as pf
+import unittest
+import ddt
+
+
+SETTINGS = settings.Settings()
+
+# Test cases for parsers. Each test case is a two-tuple of input string and
+# expected output. NOTE: default values for sentinels are used here to avoid
+# cluttering up the code.
+test_pairs_simple = (
+ # Basic test cases.
+ ('${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ # Basic combinations.
+ ('bar${foo}', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ ('bar${foo}baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ ('${foo}baz', [(pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ # Whitespace preservation cases.
+ ('bar ${foo}', [(pf.tags.STR, 'bar '),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ ('bar ${foo baz}', [(pf.tags.STR, 'bar '),
+ (pf.tags.REF, [(pf.tags.STR, 'foo baz')])]),
+ ('bar${foo} baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz')]),
+ (' bar${foo} baz ', [(pf.tags.STR, ' bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz ')]),
+)
+
+# Simple parser test cases are also included in this test grouop.
+test_pairs_full = (
+ # Single elements sanity.
+ ('foo', [(pf.tags.STR, 'foo')]),
+ ('$foo', [(pf.tags.STR, '$foo')]),
+ ('{foo}', [(pf.tags.STR, '{foo}')]),
+ ('[foo]', [(pf.tags.STR, '[foo]')]),
+ ('$(foo)', [(pf.tags.STR, '$(foo)')]),
+ ('$[foo]', [(pf.tags.INV, [(pf.tags.STR, 'foo')])]),
+
+ # Escape sequences.
+ # NOTE: these sequences apparently are not working as expected.
+ #(r'\\\\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ #(r'\\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ #(r'\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+
+ # Basic combinations.
+ ('bar$[foo]', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')])]),
+ ('bar$[foo]baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ ('$[foo]baz', [(pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+
+ # Whitespace preservation in various positions.
+ (' foo ', [(pf.tags.STR, ' foo ')]),
+ ('foo bar', [(pf.tags.STR, 'foo bar')]),
+ ('bar $[foo baz]', [(pf.tags.STR, 'bar '),
+ (pf.tags.INV, [(pf.tags.STR, 'foo baz')])]),
+ ('bar$[foo] baz ', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz ')]),
+
+ # Nested references and inventory items.
+ ('${foo}${bar}',[(pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.REF, [(pf.tags.STR, 'bar')])]),
+ ('${foo${bar}}',[(pf.tags.REF, [(pf.tags.STR, 'foo'),
+ (pf.tags.REF, [(pf.tags.STR, 'bar')])])]),
+ ('$[foo]$[bar]',[(pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.INV, [(pf.tags.STR, 'bar')])]),
+ # NOTE: the cases below do not work as expected, which is probably a bug.
+ # Any nesting in INV creates a string.
+ #('${$[foo]}', [(pf.tags.REF, [(pf.tags.INV, [(pf.tags.STR, 'foo')])])]),
+ #('$[${foo}]', [(pf.tags.INV, [(pf.tags.REF, [(pf.tags.STR, 'foo')])])]),
+ #('$[foo$[bar]]',[(pf.tags.INV, [(pf.tags.STR, 'foo'),
+ # (pf.tags.INV, [(pf.tags.STR, 'bar')])])]),
+
+) + test_pairs_simple
+
+
+@ddt.ddt
+class TestRefParser(unittest.TestCase):
+
+ @ddt.data(*test_pairs_full)
+ def test_standard_reference_parser(self, data):
+ instring, expected = data
+ parser = pf.get_ref_parser(SETTINGS)
+
+ result = pf.listify(parser.parseString(instring).asList())
+
+ self.assertEquals(expected, result)
+
+
+@ddt.ddt
+class TestSimpleRefParser(unittest.TestCase):
+
+ @ddt.data(*test_pairs_simple)
+ def test_standard_reference_parser(self, data):
+ # NOTE: simple reference parser can parse references only. It fails
+ # on inventory items.
+ instring, expected = data
+ parser = pf.get_simple_ref_parser(SETTINGS)
+
+ result = pf.listify(parser.parseString(instring).asList())
+
+ self.assertEquals(expected, result)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/requirements.txt b/requirements.txt
index 5b3aadd..5f6aed1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@
pyyaml
six
enum34
+ddt
diff --git a/setup.py b/setup.py
index 884be88..ab23207 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@
url = URL,
packages = find_packages(exclude=['*tests']), #FIXME validate this
entry_points = { 'console_scripts': console_scripts },
- install_requires = ['pyparsing', 'pyyaml', 'six', 'enum34'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
+ install_requires = ['pyparsing', 'pyyaml', 'six', 'enum34', 'ddt'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
classifiers=[
'Development Status :: 4 - Beta',