Merge pull request #73 from a-ovchinnikov/develop
Tests for parsers are added
diff --git a/reclass/values/parser.py b/reclass/values/parser.py
index 27e6d2d..3f7ac1f 100644
--- a/reclass/values/parser.py
+++ b/reclass/values/parser.py
@@ -16,9 +16,13 @@
from .scaitem import ScaItem
from reclass.errors import ParseError
-from reclass.values.parser_funcs import STR, REF, INV
+from reclass.values.parser_funcs import tags
import reclass.values.parser_funcs as parsers
+import collections
+import six
+
+
class Parser(object):
def __init__(self):
@@ -43,7 +47,7 @@
def parse(self, value, settings):
def full_parse():
try:
- return self.ref_parser.parseString(value).asList()
+ return self.ref_parser.parseString(value)
except pp.ParseException as e:
raise ParseError(e.msg, e.line, e.col, e.lineno)
@@ -55,30 +59,31 @@
return ScaItem(value, self._settings)
elif sentinel_count == 1: # speed up: try a simple reference
try:
- tokens = self.simple_ref_parser.parseString(value).asList()
+ tokens = self.simple_ref_parser.parseString(value)
except pp.ParseException:
tokens = full_parse() # fall back on the full parser
else:
tokens = full_parse() # use the full parser
+ tokens = parsers.listify(tokens)
items = self._create_items(tokens)
if len(items) == 1:
return items[0]
return CompItem(items, self._settings)
- _create_dict = { STR: (lambda s, v: ScaItem(v, s._settings)),
- REF: (lambda s, v: s._create_ref(v)),
- INV: (lambda s, v: s._create_inv(v)) }
+ _item_builders = {tags.STR: (lambda s, v: ScaItem(v, s._settings)),
+ tags.REF: (lambda s, v: s._create_ref(v)),
+ tags.INV: (lambda s, v: s._create_inv(v)) }
def _create_items(self, tokens):
- return [ self._create_dict[t](self, v) for t, v in tokens ]
+ return [self._item_builders[t](self, v) for t, v in tokens ]
def _create_ref(self, tokens):
- items = [ self._create_dict[t](self, v) for t, v in tokens ]
+ items = [ self._item_builders[t](self, v) for t, v in tokens ]
return RefItem(items, self._settings)
def _create_inv(self, tokens):
- items = [ ScaItem(v, self._settings) for t, v in tokens ]
+ items = [ScaItem(v, self._settings) for t, v in tokens]
if len(items) == 1:
return InvItem(items[0], self._settings)
return InvItem(CompItem(items), self._settings)
diff --git a/reclass/values/parser_funcs.py b/reclass/values/parser_funcs.py
index f702910..db34ceb 100644
--- a/reclass/values/parser_funcs.py
+++ b/reclass/values/parser_funcs.py
@@ -8,12 +8,13 @@
from __future__ import print_function
from __future__ import unicode_literals
-import pyparsing as pp
+import collections
+import enum
import functools
+import pyparsing as pp
+import six
-STR = 1
-REF = 2
-INV = 3
+tags = enum.Enum('Tags', ['STR', 'REF', 'INV'])
_OBJ = 'OBJ'
_LOGICAL = 'LOGICAL'
@@ -42,6 +43,20 @@
tokens[0] = (tag, token)
return functools.partial(inner, tag)
+def _asList(x):
+ if isinstance(x, pp.ParseResults):
+ return x.asList()
+ return x
+
+def listify(w, modifier=_asList):
+ if (isinstance(w, collections.Iterable) and
+ not isinstance(w, six.string_types)):
+ cls = type(w)
+ if cls == pp.ParseResults:
+ cls = list
+ return cls([listify(x) for x in w])
+ return modifier(w)
+
def get_expression_parser():
sign = pp.Optional(pp.Literal('-'))
number = pp.Word(pp.nums)
@@ -117,10 +132,10 @@
ref_escape_close = pp.Literal(_REF_ESCAPE_CLOSE).setParseAction(pp.replaceWith(_REF_CLOSE))
ref_text = pp.CharsNotIn(_REF_EXCLUDES) | pp.CharsNotIn(_REF_CLOSE_FIRST, exact=1)
ref_content = pp.Combine(pp.OneOrMore(ref_not_open + ref_not_close + ref_text))
- ref_string = pp.MatchFirst([double_escape, ref_escape_open, ref_escape_close, ref_content]).setParseAction(_tag_with(STR))
+ ref_string = pp.MatchFirst([double_escape, ref_escape_open, ref_escape_close, ref_content]).setParseAction(_tag_with(tags.STR))
ref_item = pp.Forward()
ref_items = pp.OneOrMore(ref_item)
- reference = (ref_open + pp.Group(ref_items) + ref_close).setParseAction(_tag_with(REF))
+ reference = (ref_open + pp.Group(ref_items) + ref_close).setParseAction(_tag_with(tags.REF))
ref_item << (reference | ref_string)
inv_open = pp.Literal(_INV_OPEN).suppress()
@@ -131,13 +146,17 @@
inv_escape_close = pp.Literal(_INV_ESCAPE_CLOSE).setParseAction(pp.replaceWith(_INV_CLOSE))
inv_text = pp.CharsNotIn(_INV_CLOSE_FIRST)
inv_content = pp.Combine(pp.OneOrMore(inv_not_close + inv_text))
- inv_string = pp.MatchFirst([double_escape, inv_escape_open, inv_escape_close, inv_content]).setParseAction(_tag_with(STR))
+ inv_string = pp.MatchFirst(
+ [double_escape, inv_escape_open, inv_escape_close, inv_content]
+ ).setParseAction(_tag_with(tags.STR))
inv_items = pp.OneOrMore(inv_string)
- export = (inv_open + pp.Group(inv_items) + inv_close).setParseAction(_tag_with(INV))
+ export = (inv_open + pp.Group(inv_items) + inv_close).setParseAction(_tag_with(tags.INV))
text = pp.CharsNotIn(_EXCLUDES) | pp.CharsNotIn('', exact=1)
content = pp.Combine(pp.OneOrMore(ref_not_open + inv_not_open + text))
- string = pp.MatchFirst([double_escape, ref_escape_open, inv_escape_open, content]).setParseAction(_tag_with(STR))
+ string = pp.MatchFirst(
+ [double_escape, ref_escape_open, inv_escape_open, content]
+ ).setParseAction(_tag_with(tags.STR))
item = reference | export | string
line = pp.OneOrMore(item) + s_end
@@ -151,9 +170,9 @@
INV_OPEN, INV_CLOSE = settings.export_sentinels
EXCLUDES = ESCAPE + REF_OPEN + REF_CLOSE + INV_OPEN + INV_CLOSE
- string = pp.CharsNotIn(EXCLUDES).setParseAction(_tag_with(STR))
+ string = pp.CharsNotIn(EXCLUDES).setParseAction(_tag_with(tags.STR))
ref_open = pp.Literal(REF_OPEN).suppress()
ref_close = pp.Literal(REF_CLOSE).suppress()
- reference = (ref_open + pp.Group(string) + ref_close).setParseAction(_tag_with(REF))
+ reference = (ref_open + pp.Group(string) + ref_close).setParseAction(_tag_with(tags.REF))
line = pp.StringStart() + pp.Optional(string) + reference + pp.Optional(string) + s_end
return line.leaveWhitespace()
diff --git a/reclass/values/tests/test_parser_functions.py b/reclass/values/tests/test_parser_functions.py
new file mode 100644
index 0000000..a660c76
--- /dev/null
+++ b/reclass/values/tests/test_parser_functions.py
@@ -0,0 +1,116 @@
+from reclass import settings
+from reclass.values import parser_funcs as pf
+import unittest
+import ddt
+
+
+SETTINGS = settings.Settings()
+
+# Test cases for parsers. Each test case is a two-tuple of input string and
+# expected output. NOTE: default values for sentinels are used here to avoid
+# cluttering up the code.
+test_pairs_simple = (
+ # Basic test cases.
+ ('${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ # Basic combinations.
+ ('bar${foo}', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ ('bar${foo}baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ ('${foo}baz', [(pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ # Whitespace preservation cases.
+ ('bar ${foo}', [(pf.tags.STR, 'bar '),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ ('bar ${foo baz}', [(pf.tags.STR, 'bar '),
+ (pf.tags.REF, [(pf.tags.STR, 'foo baz')])]),
+ ('bar${foo} baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz')]),
+ (' bar${foo} baz ', [(pf.tags.STR, ' bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz ')]),
+)
+
+# Simple parser test cases are also included in this test grouop.
+test_pairs_full = (
+ # Single elements sanity.
+ ('foo', [(pf.tags.STR, 'foo')]),
+ ('$foo', [(pf.tags.STR, '$foo')]),
+ ('{foo}', [(pf.tags.STR, '{foo}')]),
+ ('[foo]', [(pf.tags.STR, '[foo]')]),
+ ('$(foo)', [(pf.tags.STR, '$(foo)')]),
+ ('$[foo]', [(pf.tags.INV, [(pf.tags.STR, 'foo')])]),
+
+ # Escape sequences.
+ # NOTE: these sequences apparently are not working as expected.
+ #(r'\\\\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ #(r'\\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ #(r'\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+
+ # Basic combinations.
+ ('bar$[foo]', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')])]),
+ ('bar$[foo]baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ ('$[foo]baz', [(pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+
+ # Whitespace preservation in various positions.
+ (' foo ', [(pf.tags.STR, ' foo ')]),
+ ('foo bar', [(pf.tags.STR, 'foo bar')]),
+ ('bar $[foo baz]', [(pf.tags.STR, 'bar '),
+ (pf.tags.INV, [(pf.tags.STR, 'foo baz')])]),
+ ('bar$[foo] baz ', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz ')]),
+
+ # Nested references and inventory items.
+ ('${foo}${bar}',[(pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.REF, [(pf.tags.STR, 'bar')])]),
+ ('${foo${bar}}',[(pf.tags.REF, [(pf.tags.STR, 'foo'),
+ (pf.tags.REF, [(pf.tags.STR, 'bar')])])]),
+ ('$[foo]$[bar]',[(pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.INV, [(pf.tags.STR, 'bar')])]),
+ # NOTE: the cases below do not work as expected, which is probably a bug.
+ # Any nesting in INV creates a string.
+ #('${$[foo]}', [(pf.tags.REF, [(pf.tags.INV, [(pf.tags.STR, 'foo')])])]),
+ #('$[${foo}]', [(pf.tags.INV, [(pf.tags.REF, [(pf.tags.STR, 'foo')])])]),
+ #('$[foo$[bar]]',[(pf.tags.INV, [(pf.tags.STR, 'foo'),
+ # (pf.tags.INV, [(pf.tags.STR, 'bar')])])]),
+
+) + test_pairs_simple
+
+
+@ddt.ddt
+class TestRefParser(unittest.TestCase):
+
+ @ddt.data(*test_pairs_full)
+ def test_standard_reference_parser(self, data):
+ instring, expected = data
+ parser = pf.get_ref_parser(SETTINGS)
+
+ result = pf.listify(parser.parseString(instring).asList())
+
+ self.assertEquals(expected, result)
+
+
+@ddt.ddt
+class TestSimpleRefParser(unittest.TestCase):
+
+ @ddt.data(*test_pairs_simple)
+ def test_standard_reference_parser(self, data):
+ # NOTE: simple reference parser can parse references only. It fails
+ # on inventory items.
+ instring, expected = data
+ parser = pf.get_simple_ref_parser(SETTINGS)
+
+ result = pf.listify(parser.parseString(instring).asList())
+
+ self.assertEquals(expected, result)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/requirements.txt b/requirements.txt
index 5b3aadd..5f6aed1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@
pyyaml
six
enum34
+ddt
diff --git a/setup.py b/setup.py
index 884be88..ab23207 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@
url = URL,
packages = find_packages(exclude=['*tests']), #FIXME validate this
entry_points = { 'console_scripts': console_scripts },
- install_requires = ['pyparsing', 'pyyaml', 'six', 'enum34'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
+ install_requires = ['pyparsing', 'pyyaml', 'six', 'enum34', 'ddt'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
classifiers=[
'Development Status :: 4 - Beta',