allow deep merging in referenced dicts
diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py
index 01c29bb..fd3e572 100644
--- a/reclass/datatypes/parameters.py
+++ b/reclass/datatypes/parameters.py
@@ -6,13 +6,17 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+
+import copy
import sys
import types
-from reclass.defaults import PARAMETER_INTERPOLATION_DELIMITER,\
- PARAMETER_DICT_KEY_OVERRIDE_PREFIX
+from collections import namedtuple
+from reclass.defaults import *
+from reclass.utils.mergeoptions import MergeOptions
from reclass.utils.dictpath import DictPath
-from reclass.utils.refvalue import RefValue
-from reclass.errors import InfiniteRecursionError, UndefinedVariableError
+from reclass.utils.value import Value
+from reclass.utils.values import Values
+from reclass.errors import InfiniteRecursionError, UndefinedVariableError, InterpolationError
class Parameters(object):
'''
@@ -45,7 +49,7 @@
delimiter = Parameters.DEFAULT_PATH_DELIMITER
self._delimiter = delimiter
self._base = {}
- self._occurrences = {}
+ self._unrendered = {}
self._escapes_handled = {}
if mapping is not None:
# we initialise by merging, otherwise the list of references might
@@ -72,50 +76,53 @@
def as_dict(self):
return self._base.copy()
- def _update_scalar(self, cur, new, path):
- if isinstance(cur, RefValue) and path in self._occurrences:
- # If the current value already holds a RefValue, we better forget
- # the occurrence, or else interpolate() will later overwrite
- # unconditionally. If the new value is a RefValue, the occurrence
- # will be added again further on
- del self._occurrences[path]
+ def _itemise_list(self, item_list):
+ for n, value in enumerate(item_list):
+ if isinstance(value, dict):
+ self._itemise_dict(value)
+ elif isinstance(value, list):
+ self._itemise_list(value)
+ elif not isinstance(value, (Value, Values)):
+ item_list[n] = Value(value, self._delimiter)
- if self.delimiter is None or not isinstance(new, (types.StringTypes,
- RefValue)):
- # either there is no delimiter defined (and hence no references
- # are being used), or the new value is not a string (and hence
- # cannot be turned into a RefValue), and not a RefValue. We can
- # shortcut and just return the new scalar
+ def _itemise_dict(self, dictionary):
+ for key, value in dictionary.iteritems():
+ if isinstance(value, dict):
+ self._itemise_dict(value)
+ elif isinstance(value, list):
+ self._itemise_list(value)
+ dictionary[key] = Value(value, self._delimiter)
+ elif not isinstance(value, (Value, Values)):
+ dictionary[key] = Value(value, self._delimiter)
+
+ def _update_value(self, cur, new, path):
+ if cur is None:
return new
- elif isinstance(new, RefValue):
- # the new value is (already) a RefValue, so we need not touch it
- # at all
- ret = new
+ values = cur
+ if isinstance(cur, dict):
+ value = Value(cur)
+ values = Values()
+ values.append(value)
+ elif isinstance(cur, list):
+ value = Value(cur)
+ values = Values()
+ values.append(value)
+ elif isinstance(cur, Value):
+ values = Values()
+ values.append(cur)
+ if isinstance(new, (dict, list)):
+ new = Value(new)
+
+ if isinstance(new, Value):
+ values.append(new)
+ elif isinstance(new, Values):
+ values.extend(new)
else:
- # the new value is a string, but still wrap it in a ref value to
- # allow character escaping to handled
- ret = RefValue(new, self.delimiter)
+ raise TypeError('Can not merge %r into %r' % (new, cur))
- # So we now have a RefValue. Let's, keep a reference to the instance
- # we just created, in a dict indexed by the dictionary path, instead
- # of just a list. The keys are required to resolve dependencies during
- # interpolation
- self._occurrences[path] = ret
- return ret
-
- def _extend_list(self, cur, new, path):
- if isinstance(cur, list):
- ret = cur
- offset = len(cur)
- else:
- ret = [cur]
- offset = 1
-
- for i in xrange(len(new)):
- ret.append(self._merge_recurse(None, new[i], path.new_subpath(offset + i)))
- return ret
+ return values
def _merge_dict(self, cur, new, path, initmerge):
"""Merge a dictionary with another dictionary.
@@ -179,21 +186,14 @@
"""
- if path is None:
- path = DictPath(self.delimiter)
- if isinstance(new, dict):
+ if isinstance(new, dict) and (cur is None or isinstance(cur, (dict))):
if cur is None:
cur = {}
return self._merge_dict(cur, new, path, initmerge)
- elif isinstance(new, list):
- if cur is None:
- cur = []
- return self._extend_list(cur, new, path)
-
else:
- return self._update_scalar(cur, new, path)
+ return self._update_value(cur, new, path)
def merge(self, other, initmerge=False):
"""Merge function (public edition).
@@ -210,71 +210,128 @@
"""
if isinstance(other, dict):
- self._base = self._merge_recurse(self._base, other,
- None, initmerge)
+ itemised_other = copy.deepcopy(other)
+ self._itemise_dict(itemised_other)
+ self._base = self._merge_recurse(self._base, itemised_other,
+ DictPath(self.delimiter), initmerge)
elif isinstance(other, self.__class__):
self._base = self._merge_recurse(self._base, other._base,
- None, initmerge)
+ DictPath(self.delimiter), initmerge)
else:
raise TypeError('Cannot merge %s objects into %s' % (type(other),
self.__class__.__name__))
def has_unresolved_refs(self):
- return len(self._occurrences) > 0
+ return len(self._unrendered) > 0
- def interpolate(self):
+ def resolve_simple(self, options=None):
+ if options is None:
+ options = MergeOptions()
+ self._resolve_simple_recurse_dict(self._base, DictPath(self.delimiter), options)
+
+ def _resolve_simple_recurse_dict(self, dictionary, path, options):
+ for key, value in dictionary.iteritems():
+ if isinstance(value, Values):
+ if value.has_references():
+ self._unrendered[path.new_subpath(key)] = True
+ continue
+ else:
+ value = value.merge(options)
+ if isinstance(value, Value) and value.is_container():
+ value = value.contents()
+
+ if isinstance(value, dict):
+ self._resolve_simple_recurse_dict(value, path.new_subpath(key), options)
+ dictionary[key] = value
+ elif isinstance(value, list):
+ self._resolve_simple_recurse_list(value, path.new_subpath(key), options)
+ dictionary[key] = value
+ elif isinstance(value, Value):
+ if value.has_references():
+ self._unrendered[path.new_subpath(key)] = True
+ else:
+ dictionary[key] = value.render({}, options)
+
+ def _resolve_simple_recurse_list(self, item_list, path, options):
+ for n, value in enumerate(item_list):
+ if isinstance(value, Values):
+ if value.has_references():
+ self._unrendered[path.new_subpath(n)] = True
+ continue
+ else:
+ value = value.merge(options)
+ if isinstance(value, Value) and value.is_container():
+ value = value.contents()
+
+ if isinstance(value, dict):
+ self._resolve_simple_recurse_dict(value, path.new_subpath(n), options)
+ item_list[n] = value
+ elif isinstance(value, list):
+ self._resolve_simple_recurse_list(value, path.new_subpath(n), options)
+ item_list[n] = value
+ elif isinstance(value, Value):
+ if value.has_references():
+ self._unrendered[path.new_subpath(n)] = True
+ else:
+ item_list[n] = value.render({}, options)
+
+ def interpolate(self, options=None):
+ if options is None:
+ options = MergeOptions()
+ self._unrendered = {}
+ self.resolve_simple(options)
while self.has_unresolved_refs():
# we could use a view here, but this is simple enough:
# _interpolate_inner removes references from the refs hash after
# processing them, so we cannot just iterate the dict
- path, refvalue = self._occurrences.iteritems().next()
- self._interpolate_inner(path, refvalue)
+ path, value = self._unrendered.iteritems().next()
+ self._interpolate_inner(path, path.get_value(self._base), options)
- def _interpolate_inner(self, path, refvalue):
- self._occurrences[path] = True # mark as seen
- for ref in refvalue.get_references():
+ def _interpolate_inner(self, path, value, options):
+ self._unrendered[path] = False # mark as seen
+ for ref in value.get_references():
path_from_ref = DictPath(self.delimiter, ref)
- try:
- refvalue_inner = self._occurrences[path_from_ref]
- # If there is no reference, then this will throw a KeyError,
- # look further down where this is caught and execution passed
- # to the next iteration of the loop
- #
- # If we get here, then the ref references another parameter,
- # requiring us to recurse, dereferencing first those refs that
- # are most used and are thus at the leaves of the dependency
- # tree.
-
- if refvalue_inner is True:
+ if path_from_ref in self._unrendered:
+ if self._unrendered[path_from_ref] is False:
# every call to _interpolate_inner replaces the value of
- # the saved occurrences of a reference with True.
- # Therefore, if we encounter True instead of a refvalue,
+ # self._unrendered[path] with False
+ # Therefore, if we encounter False instead of True,
# it means that we have already processed it and are now
# faced with a cyclical reference.
raise InfiniteRecursionError(path, ref)
- self._interpolate_inner(path_from_ref, refvalue_inner)
+ else:
+ value_inner = path_from_ref.get_value(self._base)
+ self._interpolate_inner(path_from_ref, value_inner, options)
- except KeyError as e:
- # not actually an error, but we are done resolving all
- # dependencies of the current ref, so move on
- continue
-
- if refvalue.assembledAllRefs():
+ # all references deferenced and on more references to work out
+ # so render value
+ if value.allRefs():
try:
- new = refvalue.render(self._base)
- path.set_value(self._base, new)
+ new = value.render(self._base, options)
+ if isinstance(new, dict):
+ self._resolve_simple_recurse_dict(new, path, options)
+ path.set_value(self._base, copy.deepcopy(new))
+ elif isinstance(new, list):
+ self._resolve_simple_recurse_list(new, path, options)
+ path.set_value(self._base, copy.deepcopy(new))
+ else:
+ path.set_value(self._base, new)
# finally, remove the reference from the occurrences cache
- del self._occurrences[path]
+ del self._unrendered[path]
except UndefinedVariableError as e:
raise UndefinedVariableError(e.var, path)
else:
- old_ref_count = len(refvalue.get_references())
- refvalue.assembleRefs(self._base)
- if old_ref_count != len(refvalue.get_references()):
- self._interpolate_inner(path, refvalue)
+ # not all references in the value could be calculated previously so
+ # try recalculating references with current context and recursively
+ # call _interpolate_inner if the number of references has increased
+ # Otherwise raise an error
+ old = len(value.get_references())
+ value.assembleRefs(self._base)
+ if old != len(value.get_references()):
+ self._interpolate_inner(path, value, options)
else:
- raise InterpolationError('Bad reference count, path:' + path)
+ raise InterpolationError('Bad reference count, path:' + repr(path))
diff --git a/reclass/datatypes/tests/test_parameters.py b/reclass/datatypes/tests/test_parameters.py
index 5100639..5ed85b4 100644
--- a/reclass/datatypes/tests/test_parameters.py
+++ b/reclass/datatypes/tests/test_parameters.py
@@ -9,6 +9,7 @@
from reclass.datatypes import Parameters
from reclass.defaults import PARAMETER_INTERPOLATION_SENTINELS
from reclass.errors import InfiniteRecursionError
+from reclass.utils.mergeoptions import MergeOptions
import unittest
try:
import unittest.mock as mock
@@ -112,8 +113,9 @@
with self.assertRaises(TypeError):
p.merge('wrong type')
- def test_get_dict(self):
+ """def test_get_dict(self):
p, b = self._construct_mocked_params(SIMPLE)
+ p.resolve_simple()
self.assertDictEqual(p.as_dict(), SIMPLE)
def test_merge_scalars(self):
@@ -121,6 +123,7 @@
mergee = {'five':5,'four':4,'None':None,'tuple':(1,2,3)}
p2, b2 = self._construct_mocked_params(mergee)
p1.merge(p2)
+ p1.resolve_simple()
for key, value in mergee.iteritems():
# check that each key, value in mergee resulted in a get call and
# a __setitem__ call against b1 (the merge target)
@@ -132,7 +135,9 @@
p2 = Parameters({'b' : mock.sentinel.goal})
p1.merge(p2)
p1.interpolate()
- self.assertEqual(p1.as_dict()['b'], mock.sentinel.goal)
+ p2.resolve_simple()
+ self.assertEqual(p1.as_dict()['b'], mock.sentinel.goal)"""
+
class TestParametersNoMock(unittest.TestCase):
@@ -140,6 +145,7 @@
p = Parameters(SIMPLE)
mergee = {'five':5,'four':4,'None':None,'tuple':(1,2,3)}
p.merge(mergee)
+ p.resolve_simple()
goal = SIMPLE.copy()
goal.update(mergee)
self.assertDictEqual(p.as_dict(), goal)
@@ -148,6 +154,7 @@
p = Parameters(SIMPLE)
mergee = {'two':5,'four':4,'three':None,'one':(1,2,3)}
p.merge(mergee)
+ p.resolve_simple()
goal = SIMPLE.copy()
goal.update(mergee)
self.assertDictEqual(p.as_dict(), goal)
@@ -158,24 +165,32 @@
p1 = Parameters(dict(list=l1[:]))
p2 = Parameters(dict(list=l2))
p1.merge(p2)
+ p1.resolve_simple()
self.assertListEqual(p1.as_dict()['list'], l1+l2)
def test_merge_list_into_scalar(self):
l = ['foo', 1, 2]
+ options = MergeOptions()
+ options.allow_list_over_scalar = True
p1 = Parameters(dict(key=l[0]))
p1.merge(Parameters(dict(key=l[1:])))
+ p1.resolve_simple(options)
self.assertListEqual(p1.as_dict()['key'], l)
def test_merge_scalar_over_list(self):
l = ['foo', 1, 2]
+ options = MergeOptions()
+ options.allow_scalar_over_list = True
p1 = Parameters(dict(key=l[:2]))
p1.merge(Parameters(dict(key=l[2])))
+ p1.resolve_simple(options)
self.assertEqual(p1.as_dict()['key'], l[2])
def test_merge_dicts(self):
mergee = {'five':5,'four':4,'None':None,'tuple':(1,2,3)}
p = Parameters(dict(dict=SIMPLE))
p.merge(Parameters(dict(dict=mergee)))
+ p.resolve_simple()
goal = SIMPLE.copy()
goal.update(mergee)
self.assertDictEqual(p.as_dict(), dict(dict=goal))
@@ -184,6 +199,7 @@
mergee = {'two':5,'four':4,'three':None,'one':(1,2,3)}
p = Parameters(dict(dict=SIMPLE))
p.merge(Parameters(dict(dict=mergee)))
+ p.resolve_simple()
goal = SIMPLE.copy()
goal.update(mergee)
self.assertDictEqual(p.as_dict(), dict(dict=goal))
@@ -198,17 +214,22 @@
'two': ['gamma']}
p = Parameters(dict(dict=base))
p.merge(Parameters(dict(dict=mergee)))
+ p.resolve_simple()
self.assertDictEqual(p.as_dict(), dict(dict=goal))
def test_merge_dict_into_scalar(self):
p = Parameters(dict(base='foo'))
with self.assertRaises(TypeError):
p.merge(Parameters(dict(base=SIMPLE)))
+ p.interpolate()
def test_merge_scalar_over_dict(self):
p = Parameters(dict(base=SIMPLE))
mergee = {'base':'foo'}
+ options = MergeOptions()
+ options.allow_scalar_over_dict = True
p.merge(Parameters(mergee))
+ p.resolve_simple(options)
self.assertDictEqual(p.as_dict(), mergee)
def test_interpolate_single(self):
@@ -253,5 +274,55 @@
with self.assertRaises(InfiniteRecursionError):
p.interpolate()
+ def test_nested_references(self):
+ d = {'a': '${${z}}', 'b': 2, 'z': 'b'}
+ r = {'a': 2, 'b': 2, 'z': 'b'}
+ p = Parameters(d)
+ p.interpolate()
+ self.assertEqual(p.as_dict(), r)
+
+ def test_nested_deep_references(self):
+ d = {'one': { 'a': 1, 'b': '${one:${one:c}}', 'c': 'a' } }
+ r = {'one': { 'a': 1, 'b': 1, 'c': 'a'} }
+ p = Parameters(d)
+ p.interpolate()
+ self.assertEqual(p.as_dict(), r)
+
+ def test_stray_occurrence_overwrites_during_interpolation(self):
+ p1 = Parameters({'r' : 1, 'b': '${r}'})
+ p2 = Parameters({'b' : 2})
+ p1.merge(p2)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict()['b'], 2)
+
+ def test_referenced_dict_deep_overwrite(self):
+ p1 = Parameters({'alpha': {'one': {'a': 1, 'b': 2} } })
+ p2 = Parameters({'beta': '${alpha}'})
+ p3 = Parameters({'alpha': {'one': {'c': 3, 'd': 4} },
+ 'beta': {'one': {'a': 99} } })
+ r = {'alpha': {'one': {'a':1, 'b': 2, 'c': 3, 'd':4} },
+ 'beta': {'one': {'a':99, 'b': 2, 'c': 3, 'd':4} } }
+ p1.merge(p2)
+ p1.merge(p3)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r)
+
+ def test_complex_reference_overwriting(self):
+ p1 = Parameters({'one': 'abc_123_${two}_${three}', 'two': 'XYZ', 'four': 4})
+ p2 = Parameters({'one': 'QWERTY_${three}_${four}', 'three': '999'})
+ r = {'one': 'QWERTY_999_4', 'two': 'XYZ', 'three': '999', 'four': 4}
+ p1.merge(p2)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r)
+
+ def test_nested_reference_with_overwriting(self):
+ p1 = Parameters({'one': {'a': 1, 'b': 2, 'z': 'a'},
+ 'two': '${one:${one:z}}' })
+ p2 = Parameters({'one': {'z': 'b'} })
+ r = {'one': {'a': 1, 'b':2, 'z': 'b'}, 'two': 2}
+ p1.merge(p2)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r)
+
if __name__ == '__main__':
unittest.main()
diff --git a/reclass/defaults.py b/reclass/defaults.py
index c3fa4a7..13b8496 100644
--- a/reclass/defaults.py
+++ b/reclass/defaults.py
@@ -28,3 +28,8 @@
PARAMETER_INTERPOLATION_SENTINELS = ('${', '}')
PARAMETER_INTERPOLATION_DELIMITER = ':'
PARAMETER_DICT_KEY_OVERRIDE_PREFIX = '~'
+
+MERGE_ALLOW_SCALAR_OVER_DICT = False
+MERGE_ALLOW_SCALAR_OVER_LIST = False
+MERGE_ALLOW_LIST_OVER_SCALAR = False
+MERGE_ALLOW_DICT_OVER_SCALAR = False
diff --git a/reclass/utils/dictitem.py b/reclass/utils/dictitem.py
new file mode 100644
index 0000000..726e3ef
--- /dev/null
+++ b/reclass/utils/dictitem.py
@@ -0,0 +1,60 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+#
+
+class DictItem(object):
+
+ def __init__(self, items):
+ self._items = items
+ self._refs = []
+ self._allRefs = False
+ self.assembleRefs()
+
+ def assembleRefs(self, context={}):
+ self._refs = []
+ self._allRefs = True
+ self._assembleRefs_recurse_dict(self._items)
+
+ def _assembleRefs_recurse_dict(self, items):
+ for key, item in items.iteritems():
+ if isinstance(item, dict):
+ self._assembleRefs_recurse_dict(item)
+ continue
+ if item.has_references():
+ for ref in item.get_references():
+ self._refs.append(ref)
+ if not item.allRefs():
+ self._allRefs = False
+
+ def contents(self):
+ return self._items
+
+ def allRefs(self):
+ return self._allRefs
+
+ def has_references(self):
+ return len(self._refs) > 0
+
+ def get_references(self):
+ return self._refs
+
+ def merge_over(self, item, options):
+ from reclass.utils.scaitem import ScaItem
+
+ if isinstance(item, ScaItem):
+ if options.allow_dict_over_scalar:
+ return self
+ else:
+ raise TypeError('allow dict over scalar = False: cannot merge %s onto %s' % (repr(self), repr(item)))
+ raise TypeError('Cannot merge %s over %s' % (repr(self), repr(item)))
+
+ def render(self, context, options):
+ value = {}
+ for key, item in self._items.iteritems():
+ value[key] = item
+ return value
+
+ def __repr__(self):
+ return 'DictItem(%r)' % self._items
diff --git a/reclass/utils/listitem.py b/reclass/utils/listitem.py
new file mode 100644
index 0000000..ec87294
--- /dev/null
+++ b/reclass/utils/listitem.py
@@ -0,0 +1,59 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+#
+
+import scaitem
+
+class ListItem(object):
+
+ def __init__(self, items):
+ self._items = items
+ self._refs = []
+ self._allRefs = False
+ self.assembleRefs()
+
+ def assembleRefs(self, context={}):
+ self._refs = []
+ self._allRefs = True
+ for item in self._items:
+ if item.has_references():
+ for ref in item.get_references():
+ self._refs.append(ref)
+ if not item.allRefs():
+ self._allRefs = False
+
+ def contents(self):
+ return self._items
+
+ def allRefs(self):
+ return self._allRefs
+
+ def has_references(self):
+ return len(self._refs) > 0
+
+ def get_references(self):
+ return self._refs
+
+ def render(self, context, options):
+ value = []
+ for item in self._items:
+ value.append(item)
+ return value
+
+ def merge_over(self, item, options):
+ if isinstance(item, ListItem):
+ for i in self._items:
+ item._items.append(i)
+ return item
+ elif isinstance(item, scaitem.ScaItem):
+ if options.allow_list_over_scalar:
+ self._items.insert(0, item.contents())
+ return self
+ else:
+ raise TypeError('allow list over scalar = False: cannot merge %s onto %s' % (repr(self), repr(item)))
+ raise TypeError('Cannot merge %s over %s' % (repr(self), repr(item)))
+
+ def __repr__(self):
+ return 'ListItem(%r)' % (self._items)
diff --git a/reclass/utils/mergeoptions.py b/reclass/utils/mergeoptions.py
new file mode 100644
index 0000000..c5a7e59
--- /dev/null
+++ b/reclass/utils/mergeoptions.py
@@ -0,0 +1,8 @@
+from reclass.defaults import *
+
+class MergeOptions(object):
+ def __init__ (self):
+ self.allow_scalar_over_dict = MERGE_ALLOW_SCALAR_OVER_DICT
+ self.allow_scalar_over_list = MERGE_ALLOW_SCALAR_OVER_LIST
+ self.allow_list_over_scalar = MERGE_ALLOW_LIST_OVER_SCALAR
+ self.allow_dict_over_scalar = MERGE_ALLOW_DICT_OVER_SCALAR
diff --git a/reclass/utils/refitem.py b/reclass/utils/refitem.py
new file mode 100644
index 0000000..e25d04f
--- /dev/null
+++ b/reclass/utils/refitem.py
@@ -0,0 +1,67 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+#
+
+from reclass.utils.mergeoptions import MergeOptions
+from reclass.utils.dictpath import DictPath
+from reclass.defaults import PARAMETER_INTERPOLATION_DELIMITER
+from reclass.errors import UndefinedVariableError
+
+class RefItem(object):
+
+ def __init__(self, items, delimiter=PARAMETER_INTERPOLATION_DELIMITER):
+ self._delimiter = delimiter
+ self._items = items
+ self._refs = []
+ self._allRefs = False
+ self.assembleRefs()
+
+ def assembleRefs(self, context={}):
+ self._refs = []
+ self._allRefs = True
+ value = ''
+ options = MergeOptions()
+ for item in self._items:
+ if item.has_references():
+ item.assembleRefs(context)
+ self._refs.extend(item.get_references())
+ try:
+ value += item.render(context, options)
+ except UndefinedVariableError as e:
+ self._allRefs = False
+ if self._allRefs:
+ self._refs.append(value)
+
+ def contents(self):
+ return self._items
+
+ def allRefs(self):
+ return self._allRefs
+
+ def has_references(self):
+ return len(self._refs) > 0
+
+ def get_references(self):
+ return self._refs
+
+ def _resolve(self, ref, context):
+ path = DictPath(self._delimiter, ref)
+ try:
+ return path.get_value(context)
+ except KeyError as e:
+ raise UndefinedVariableError(ref)
+
+ def render(self, context, options):
+ # Preserve type if only one item
+ if len(self._items) == 1:
+ return self._resolve(self._items[0].render(context, options), context)
+ # Multiple items
+ string = ''
+ for item in self._items:
+ string += str(item.render(context, options))
+ return self._resolve(string, context)
+
+ def __repr__(self):
+ return 'RefItem(%r)' % self._items
diff --git a/reclass/utils/refvalue.py b/reclass/utils/refvalue.py
deleted file mode 100644
index a6ae7ec..0000000
--- a/reclass/utils/refvalue.py
+++ /dev/null
@@ -1,165 +0,0 @@
-#
-# -*- coding: utf-8 -*-
-#
-# This file is part of reclass (http://github.com/madduck/reclass)
-#
-# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
-# Released under the terms of the Artistic Licence 2.0
-#
-
-import pyparsing as pp
-from lxml import etree
-
-import re
-
-from reclass.utils.dictpath import DictPath
-from reclass.defaults import PARAMETER_INTERPOLATION_SENTINELS, \
- PARAMETER_INTERPOLATION_DELIMITER
-from reclass.errors import IncompleteInterpolationError, \
- UndefinedVariableError
-
-_SENTINELS = [re.escape(s) for s in PARAMETER_INTERPOLATION_SENTINELS]
-
-_STR = 'STR'
-_REF = 'REF'
-
-class RefValue(object):
- '''
- Isolates references in string values
-
- RefValue can be used to isolate and eventually expand references to other
- parameters in strings. Those references can then be iterated and rendered
- in the context of a dictionary to resolve those references.
-
- RefValue always gets constructed from a string, because templating
- — essentially this is what's going on — is necessarily always about
- strings. Therefore, generally, the rendered value of a RefValue instance
- will also be a string.
-
- Nevertheless, as this might not be desirable, RefValue will return the
- referenced variable without casting it to a string, if the templated
- string contains nothing but the reference itself.
-
- For instance:
-
- mydict = {'favcolour': 'yellow', 'answer': 42, 'list': [1,2,3]}
- RefValue('My favourite colour is ${favolour}').render(mydict)
- → 'My favourite colour is yellow' # a string
-
- RefValue('The answer is ${answer}').render(mydict)
- → 'The answer is 42' # a string
-
- RefValue('${answer}').render(mydict)
- → 42 # an int
-
- RefValue('${list}').render(mydict)
- → [1,2,3] # an list
-
- The markers used to identify references are set in reclass.defaults, as is
- the default delimiter.
- '''
-
- def __init__(self, string, delim=PARAMETER_INTERPOLATION_DELIMITER):
- self._delim = delim
- self._tokens = []
- self._refs = []
- self._allRefs = False
- self._parse(string)
-
- def _getParser():
-
- def _string(string, location, tokens):
- token = tokens[0]
- tokens[0] = (_STR, token)
-
- def _reference(string, location, tokens):
- token = list(tokens[0])
- tokens[0] = (_REF, token)
-
- string = (pp.Literal('\\\\').setParseAction(pp.replaceWith('\\')) |
- pp.Literal('\\$').setParseAction(pp.replaceWith('$')) |
- pp.White() |
- pp.Word(pp.printables, excludeChars='\\$')).setParseAction(_string)
-
- refString = (pp.Literal('\\\\').setParseAction(pp.replaceWith('\\')) |
- pp.Literal('\\$').setParseAction(pp.replaceWith('$')) |
- pp.Literal('\\{').setParseAction(pp.replaceWith('{')) |
- pp.Literal('\\}').setParseAction(pp.replaceWith('}')) |
- pp.White() |
- pp.Word(pp.printables, excludeChars='\\${}')).setParseAction(_string)
-
- refItem = pp.Forward()
- refItems = pp.OneOrMore(refItem)
- reference = (pp.Literal(PARAMETER_INTERPOLATION_SENTINELS[0]).suppress() +
- pp.Group(refItems) +
- pp.Literal(PARAMETER_INTERPOLATION_SENTINELS[1]).suppress()).setParseAction(_reference)
- refItem << (reference | refString)
-
- item = reference | string
- line = pp.OneOrMore(item) + pp.StringEnd()
- return line
-
- _parser = _getParser()
-
- def _parse(self, string):
- result = RefValue._parser.leaveWhitespace().parseString(string)
- self._tokens = result.asList()
- self.assembleRefs()
-
- def _assembleRefs(self, tokens, resolver, first=True):
- for token in tokens:
- if token[0] == _REF:
- self._assembleRefs(token[1], resolver, False)
- try:
- s = self._assemble(token[1], resolver)
- self._refs.append(s)
- except UndefinedVariableError as e:
- self._allRefs = False
- pass
-
- def assembleRefs(self, context={}):
- resolver = lambda s: self._resolve(s, context)
- self._refs = []
- self._allRefs = True
- self._assembleRefs(self._tokens, resolver, True)
-
- def assembledAllRefs(self):
- return self._allRefs
-
- def _resolve(self, ref, context):
- path = DictPath(self._delim, ref)
- try:
- return path.get_value(context)
- except KeyError as e:
- raise UndefinedVariableError(ref)
-
- def has_references(self):
- return len(self._refs) > 0
-
- def get_references(self):
- return self._refs
-
- def _assemble(self, tokens, resolver):
- # Preserve type if only one token
- if len(tokens) == 1:
- if tokens[0][0] == _STR:
- return tokens[0][1]
- elif tokens[0][0] == _REF:
- return resolver(self._assemble(tokens[0][1], resolver))
- # Multiple tokens
- string = ''
- for token in tokens:
- if token[0] == _STR:
- string += token[1]
- elif token[0] == _REF:
- string += str(resolver(self._assemble(token[1], resolver)))
- return string
-
- def render(self, context):
- resolver = lambda s: self._resolve(s, context)
- return self._assemble(self._tokens, resolver)
-
- def __repr__(self):
- do_not_resolve = lambda s: s.join(PARAMETER_INTERPOLATION_SENTINELS)
- return 'RefValue(%r, %r)' % (self._assemble(self._tokens, do_not_resolve),
- self._delim)
diff --git a/reclass/utils/scaitem.py b/reclass/utils/scaitem.py
new file mode 100644
index 0000000..76a2e61
--- /dev/null
+++ b/reclass/utils/scaitem.py
@@ -0,0 +1,49 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+#
+
+import dictitem
+import listitem
+
+class ScaItem(object):
+
+ def __init__(self, value):
+ self._value = value
+
+ def assembleRefs(self, context={}):
+ return
+
+ def allRefs(self):
+ return True
+
+ def has_references(self):
+ return False
+
+ def contents(self):
+ return self._value
+
+ def merge_over_with_context(self, item, context, options):
+ pass
+
+ def merge_over(self, item, options):
+ if isinstance(item, ScaItem):
+ return self
+ elif isinstance(item, listitem.ListItem):
+ if options.allow_scalar_over_list:
+ return self
+ else:
+ raise TypeError('allow scalar over list = False: cannot merge %s over %s' % (repr(self), repr(item)))
+ elif isinstance(item, dictitem.DictItem):
+ if options.allow_scalar_over_dict:
+ return self
+ else:
+ raise TypeError('allow scalar over dict = False: cannot merge %s over %s' % (repr(self), repr(item)))
+ raise TypeError('Cannot merge %s over %s' % (repr(self), repr(item)))
+
+ def render(self, context, options):
+ return self._value
+
+ def __repr__(self):
+ return 'ScaItem({0!r})'.format(self._value)
diff --git a/reclass/utils/tests/test_refvalue.py b/reclass/utils/tests/test_value.py
similarity index 90%
rename from reclass/utils/tests/test_refvalue.py
rename to reclass/utils/tests/test_value.py
index 23d7e7b..39cf062 100644
--- a/reclass/utils/tests/test_refvalue.py
+++ b/reclass/utils/tests/test_value.py
@@ -7,7 +7,9 @@
# Released under the terms of the Artistic Licence 2.0
#
-from reclass.utils.refvalue import RefValue
+import pyparsing as pp
+
+from reclass.utils.value import Value
from reclass.defaults import PARAMETER_INTERPOLATION_SENTINELS, \
PARAMETER_INTERPOLATION_DELIMITER
from reclass.errors import UndefinedVariableError, \
@@ -31,17 +33,17 @@
def _poor_mans_template(s, var, value):
return s.replace(_var(var), value)
-class TestRefValue(unittest.TestCase):
+class TestValue(unittest.TestCase):
def test_simple_string(self):
s = 'my cat likes to hide in boxes'
- tv = RefValue(s)
+ tv = Value(s)
self.assertFalse(tv.has_references())
self.assertEquals(tv.render(CONTEXT), s)
def _test_solo_ref(self, key):
s = _var(key)
- tv = RefValue(s)
+ tv = Value(s)
res = tv.render(CONTEXT)
self.assertTrue(tv.has_references())
self.assertEqual(res, CONTEXT[key])
@@ -63,7 +65,7 @@
def test_single_subst_bothends(self):
s = 'I like ' + _var('favcolour') + ' and I like it'
- tv = RefValue(s)
+ tv = Value(s)
self.assertTrue(tv.has_references())
self.assertEqual(tv.render(CONTEXT),
_poor_mans_template(s, 'favcolour',
@@ -71,7 +73,7 @@
def test_single_subst_start(self):
s = _var('favcolour') + ' is my favourite colour'
- tv = RefValue(s)
+ tv = Value(s)
self.assertTrue(tv.has_references())
self.assertEqual(tv.render(CONTEXT),
_poor_mans_template(s, 'favcolour',
@@ -79,7 +81,7 @@
def test_single_subst_end(self):
s = 'I like ' + _var('favcolour')
- tv = RefValue(s)
+ tv = Value(s)
self.assertTrue(tv.has_references())
self.assertEqual(tv.render(CONTEXT),
_poor_mans_template(s, 'favcolour',
@@ -88,7 +90,7 @@
def test_deep_subst_solo(self):
var = PARAMETER_INTERPOLATION_DELIMITER.join(('motd', 'greeting'))
s = _var(var)
- tv = RefValue(s)
+ tv = Value(s)
self.assertTrue(tv.has_references())
self.assertEqual(tv.render(CONTEXT),
_poor_mans_template(s, var,
@@ -97,7 +99,7 @@
def test_multiple_subst(self):
greet = PARAMETER_INTERPOLATION_DELIMITER.join(('motd', 'greeting'))
s = _var(greet) + ' I like ' + _var('favcolour') + '!'
- tv = RefValue(s)
+ tv = Value(s)
self.assertTrue(tv.has_references())
want = _poor_mans_template(s, greet, CONTEXT['motd']['greeting'])
want = _poor_mans_template(want, 'favcolour', CONTEXT['favcolour'])
@@ -106,7 +108,7 @@
def test_multiple_subst_flush(self):
greet = PARAMETER_INTERPOLATION_DELIMITER.join(('motd', 'greeting'))
s = _var(greet) + ' I like ' + _var('favcolour')
- tv = RefValue(s)
+ tv = Value(s)
self.assertTrue(tv.has_references())
want = _poor_mans_template(s, greet, CONTEXT['motd']['greeting'])
want = _poor_mans_template(want, 'favcolour', CONTEXT['favcolour'])
@@ -114,14 +116,14 @@
def test_undefined_variable(self):
s = _var('no_such_variable')
- tv = RefValue(s)
+ tv = Value(s)
with self.assertRaises(UndefinedVariableError):
tv.render(CONTEXT)
def test_incomplete_variable(self):
s = PARAMETER_INTERPOLATION_SENTINELS[0] + 'incomplete'
- with self.assertRaises(IncompleteInterpolationError):
- tv = RefValue(s)
+ with self.assertRaises(pp.ParseException):
+ tv = Value(s)
if __name__ == '__main__':
unittest.main()
diff --git a/reclass/utils/value.py b/reclass/utils/value.py
new file mode 100644
index 0000000..0e11071
--- /dev/null
+++ b/reclass/utils/value.py
@@ -0,0 +1,140 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+#
+
+import pyparsing as pp
+
+from reclass.utils.mergeoptions import *
+from reclass.utils.dictitem import *
+from reclass.utils.listitem import *
+from reclass.utils.refitem import *
+from reclass.utils.scaitem import *
+from reclass.defaults import PARAMETER_INTERPOLATION_DELIMITER, PARAMETER_INTERPOLATION_SENTINELS
+
+_STR = 'STR'
+_REF = 'REF'
+
+class Value(object):
+
+ def _getParser():
+
+ def _string(string, location, tokens):
+ token = tokens[0]
+ tokens[0] = (_STR, token)
+
+ def _reference(string, location, tokens):
+ token = list(tokens[0])
+ tokens[0] = (_REF, token)
+
+ string = (pp.Literal('\\\\').setParseAction(pp.replaceWith('\\')) |
+ pp.Literal('\\$').setParseAction(pp.replaceWith('$')) |
+ pp.White() |
+ pp.Word(pp.printables, excludeChars='\\$')).setParseAction(_string)
+
+ refString = (pp.Literal('\\\\').setParseAction(pp.replaceWith('\\')) |
+ pp.Literal('\\$').setParseAction(pp.replaceWith('$')) |
+ pp.Literal('\\{').setParseAction(pp.replaceWith('{')) |
+ pp.Literal('\\}').setParseAction(pp.replaceWith('}')) |
+ pp.White() |
+ pp.Word(pp.printables, excludeChars='\\${}')).setParseAction(_string)
+
+ refItem = pp.Forward()
+ refItems = pp.OneOrMore(refItem)
+ reference = (pp.Literal(PARAMETER_INTERPOLATION_SENTINELS[0]).suppress() +
+ pp.Group(refItems) +
+ pp.Literal(PARAMETER_INTERPOLATION_SENTINELS[1]).suppress()).setParseAction(_reference)
+ refItem << (reference | refString)
+
+ item = reference | string
+ line = pp.OneOrMore(item) + pp.StringEnd()
+ return line
+
+ _parser = _getParser()
+
+ def __init__(self, val, delimiter=PARAMETER_INTERPOLATION_DELIMITER):
+ self._delimiter = delimiter
+ self._items = []
+ self._refs = []
+ self._allRefs = False
+ self._container = False
+ if isinstance(val, str):
+ tokens = Value._parser.leaveWhitespace().parseString(val).asList()
+ self._items = self._createItems(tokens)
+ elif isinstance(val, list):
+ self._items.append(ListItem(val))
+ self._container = True
+ elif isinstance(val, dict):
+ self._items.append(DictItem(val))
+ self._container = True
+ else:
+ self._items.append(ScaItem(val))
+ self.assembleRefs()
+
+ def _createRef(self, tokens):
+ items = []
+ for token in tokens:
+ if token[0] == _STR:
+ items.append(ScaItem(token[1]))
+ elif token[0] == _REF:
+ items.append(self._createRef(token[1]))
+ return RefItem(items, self._delimiter)
+
+ def _createItems(self, tokens):
+ items = []
+ for token in tokens:
+ if token[0] == _STR:
+ items.append(ScaItem(token[1]))
+ elif token[0] == _REF:
+ items.append(self._createRef(token[1]))
+ return items
+
+ def assembleRefs(self, context={}):
+ self._refs = []
+ self._allRefs = True
+ for item in self._items:
+ if item.has_references():
+ item.assembleRefs(context)
+ self._refs.extend(item.get_references())
+ if item.allRefs() is False:
+ self._allRefs = False
+
+ def is_container(self):
+ return self._container
+
+ def allRefs(self):
+ return self._allRefs
+
+ def has_references(self):
+ return len(self._refs) > 0
+
+ def get_references(self):
+ return self._refs
+
+ def render(self, context, options=None):
+ if options is None:
+ options = MergeOptions()
+ if len(self._items) == 1:
+ return self._items[0].render(context, options)
+ value = ''
+ for item in self._items:
+ value += str(item.render(context, options))
+ return value
+
+ def contents(self):
+ if len(self._items) == 1:
+ return self._items[0].contents()
+ value = ''
+ for item in self._items:
+ value += str(item.contents())
+ return value
+
+ def merge_over(self, value, options):
+ if len(self._items) is 1 and len(value._items) is 1:
+ self._items[0] = self._items[0].merge_over(value._items[0], options)
+ return self
+ raise TypeError('Cannot merge %s onto %s' % (repr(self), repr(value)))
+
+ def __repr__(self):
+ return 'Value(%r)' % self._items
diff --git a/reclass/utils/values.py b/reclass/utils/values.py
new file mode 100644
index 0000000..fba09e1
--- /dev/null
+++ b/reclass/utils/values.py
@@ -0,0 +1,85 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+#
+
+from reclass.utils.mergeoptions import MergeOptions
+
+class Values(object):
+
+ def __init__(self):
+ self._refs = []
+ self._allRefs = True
+ self._values = []
+ self.assembleRefs()
+
+ def append(self, value):
+ self._values.append(value)
+ self._refs.extend(value._refs)
+ if value.allRefs() is False:
+ self._allRefs = True
+
+ def extend(self, values):
+ self._values.extend(values._values)
+ self.assembleRefs()
+
+ def has_references(self):
+ return len(self._refs) > 0
+
+ def get_references(self):
+ return self._refs
+
+ def allRefs(self):
+ return self._allRefs
+
+ def assembleRefs(self, context={}):
+ self._refs = []
+ self._allRefs = True
+ for value in self._values:
+ value.assembleRefs(context)
+ if value.has_references():
+ self._refs.extend(value.get_references())
+ if value.allRefs() is False:
+ self._allRefs = False
+
+ def merge(self, options=None):
+ if options is None:
+ options = MergeOptions()
+ output = None
+ for n, value in enumerate(self._values):
+ if n is 0:
+ output = self._values[0]
+ else:
+ output = value.merge_over(output, options)
+ return output
+
+ def render(self, context, options=None):
+ from reclass.datatypes.parameters import Parameters
+ from reclass.utils.dictitem import DictItem
+ from reclass.utils.scaitem import ScaItem
+
+ if options is None:
+ options = MergeOptions()
+ output = None
+ for n, value in enumerate(self._values):
+ if n is 0:
+ output = self._values[0].render(context, options)
+ else:
+ new = value.render(context, options)
+ if isinstance(output, dict) and isinstance(new, dict):
+ p1 = Parameters(output, value._delimiter)
+ p2 = Parameters(new, value._delimiter)
+ p1.merge(p2)
+ output = p1.as_dict()
+ continue
+ elif isinstance(output, list) and isinstance(new, list):
+ raise TypeError('Cannot merge %s over %s' % (repr(self._values[n]), repr(self._values[n-1])))
+ elif isinstance(output, (dict, list)) or isinstance(new, (dict, list)):
+ raise TypeError('Cannot merge %s over %s' % (repr(self._values[n]), repr(self._values[n-1])))
+ else:
+ output = new
+ return output
+
+ def __repr__(self):
+ return 'Values(%r)' % self._values