Merge pull request #18 from salt-formulas/andrew-missed-ref2
Ignore missed/overrides of references + sum-up all interpolation errors per node.
diff --git a/README-extentions.rst b/README-extentions.rst
index 7e51233..554a62d 100644
--- a/README-extentions.rst
+++ b/README-extentions.rst
@@ -123,6 +123,59 @@
the reference ``${beta:a}`` to the value 99.
+Ignore overwritten missing references
+-------------------------
+
+Given the following classes:
+
+.. code-block:: yaml
+ # node1.yml
+ classes:
+ - class1
+ - class2
+ - class3
+
+ # class1.yml
+ parameters:
+ a: ${x}
+
+ # class2.yml
+ parameters:
+ a: ${y}
+
+ # class3.yml
+ parameters:
+ y: 1
+
+
+The parameter ``a`` only depends on the parameter ``y`` through the reference set in class2. The fact that the parameter ``x`` referenced
+in class1 is not defined does not affect the final value of the parameter ``a``. For such overwritten missing references by default a warning is
+printed but no error is raised, providing the final value of the parameter being evaluated is a scalar. If the final value is a dictionary or list
+an error will always be raised in the case of a missing reference.
+
+Default value is True to keep backward compatible behavior.
+
+.. code-block:: yaml
+
+ ignore_overwritten_missing_reference: True
+
+
+Print summary of missed references
+----------------------------------
+
+Instead of failing on the first undefinded reference error all missing reference errors are printed at once.
+
+.. code-block:: yaml
+ reclass --nodeinfo mynode
+ -> dontpanic
+ Cannot resolve ${_param:kkk}, at mkkek3:tree:to:fail, in yaml_fs:///test/classes/third.yml
+ Cannot resolve ${_param:kkk}, at mkkek3:tree:another:xxxx, in yaml_fs:///test/classes/third.yml
+ Cannot resolve ${_param:kkk}, at mykey2:tree:to:fail, in yaml_fs:///test/classes/third.yml
+
+.. code-block:: yaml
+
+ group_errors: True
+
Inventory Queries
-----------------
diff --git a/reclass/config.py b/reclass/config.py
index 5eff1a4..e9bb43b 100644
--- a/reclass/config.py
+++ b/reclass/config.py
@@ -50,6 +50,11 @@
ret.add_option('-r', '--no-refs', dest='no_refs', action="store_true",
default=defaults.get('no_refs', OPT_NO_REFS),
help='output all key values do not use yaml references [%default]')
+ ret.add_option('-1', '--single-error', dest='group_errors', action="store_false",
+ default=defaults.get('group_errors', OPT_GROUP_ERRORS),
+ help='throw errors immediately instead of grouping them together')
+ ret.add_option('-0', '--multiple-errors', dest='group_errors', action="store_true",
+ help='were possible report any errors encountered as a group')
return ret
diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py
index 9605efd..e7199ab 100644
--- a/reclass/datatypes/parameters.py
+++ b/reclass/datatypes/parameters.py
@@ -14,7 +14,7 @@
from reclass.utils.dictpath import DictPath
from reclass.values.value import Value
from reclass.values.valuelist import ValueList
-from reclass.errors import InfiniteRecursionError, ResolveError, InterpolationError, ParseError, BadReferencesError
+from reclass.errors import InfiniteRecursionError, ResolveError, ResolveErrorList, InterpolationError, ParseError, BadReferencesError
class Parameters(object):
'''
@@ -47,6 +47,7 @@
self._unrendered = None
self._escapes_handled = {}
self._inv_queries = []
+ self._resolve_errors = ResolveErrorList()
self._needs_all_envs = False
self._keep_overrides = False
if mapping is not None:
@@ -80,6 +81,9 @@
def needs_all_envs(self):
return self._needs_all_envs
+ def resolve_errors(self):
+ return self._resolve_errors
+
def as_dict(self):
return self._base.copy()
@@ -245,6 +249,8 @@
# processing them, so we cannot just iterate the dict
path, v = self._unrendered.iteritems().next()
self._interpolate_inner(path, inventory)
+ if self._resolve_errors.have_errors():
+ raise self._resolve_errors
def initialise_interpolation(self):
self._unrendered = None
@@ -255,6 +261,7 @@
self._unrendered = {}
self._inv_queries = []
self._needs_all_envs = False
+ self._resolve_errors = ResolveErrorList()
self._render_simple_dict(self._base, DictPath(self._settings.delimiter))
def _interpolate_inner(self, path, inventory):
@@ -276,7 +283,11 @@
new = value.render(self._base, inventory)
except ResolveError as e:
e.context = path
- raise
+ if self._settings.group_errors:
+ self._resolve_errors.add(e)
+ new = None
+ else:
+ raise
if isinstance(new, dict):
self._render_simple_dict(new, path)
diff --git a/reclass/datatypes/tests/test_parameters.py b/reclass/datatypes/tests/test_parameters.py
index 719c828..405f757 100644
--- a/reclass/datatypes/tests/test_parameters.py
+++ b/reclass/datatypes/tests/test_parameters.py
@@ -7,9 +7,11 @@
# Released under the terms of the Artistic Licence 2.0
#
+import copy
+
from reclass.settings import Settings
from reclass.datatypes import Parameters
-from reclass.errors import InfiniteRecursionError, InterpolationError
+from reclass.errors import InfiniteRecursionError, InterpolationError, ResolveError, ResolveErrorList
import unittest
try:
import unittest.mock as mock
@@ -19,6 +21,17 @@
SIMPLE = {'one': 1, 'two': 2, 'three': 3}
SETTINGS = Settings()
+class MockDevice(object):
+ def __init__(self):
+ self._text = ''
+
+ def write(self, s):
+ self._text += s
+ return
+
+ def text(self):
+ return self._text
+
class TestParameters(unittest.TestCase):
def _construct_mocked_params(self, iterable=None, settings=SETTINGS):
@@ -500,5 +513,64 @@
p1.interpolate()
self.assertEqual(error.exception.message, "-> \n Bad references, at gamma\n ${beta}")
+ def test_multiple_resolve_errors(self):
+ p1 = Parameters({'alpha': '${gamma}', 'beta': '${gamma}'}, SETTINGS, '')
+ with self.assertRaises(ResolveErrorList) as error:
+ p1.interpolate()
+ self.assertEqual(error.exception.message, "-> \n Cannot resolve ${gamma}, at alpha\n Cannot resolve ${gamma}, at beta")
+
+ def test_force_single_resolve_error(self):
+ settings = copy.deepcopy(SETTINGS)
+ settings.group_errors = False
+ p1 = Parameters({'alpha': '${gamma}', 'beta': '${gamma}'}, settings, '')
+ with self.assertRaises(ResolveError) as error:
+ p1.interpolate()
+ self.assertEqual(error.exception.message, "-> \n Cannot resolve ${gamma}, at alpha")
+
+ def test_ignore_overwriten_missing_reference(self):
+ settings = copy.deepcopy(SETTINGS)
+ settings.ignore_overwritten_missing_references = True
+ p1 = Parameters({'alpha': '${beta}'}, settings, '')
+ p2 = Parameters({'alpha': '${gamma}'}, settings, '')
+ p3 = Parameters({'gamma': 3}, settings, '')
+ r1 = {'alpha': 3, 'gamma': 3}
+ p1.merge(p2)
+ p1.merge(p3)
+ err1 = "[WARNING] Reference '${beta}' undefined\n"
+ with mock.patch('sys.stderr', new=MockDevice()) as std_err:
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r1)
+ self.assertEqual(std_err.text(), err1)
+
+ def test_ignore_overwriten_missing_reference_last_value(self):
+ # an error should be raised if the last reference to be merged
+ # is missing even if ignore_overwritten_missing_references is true
+ settings = copy.deepcopy(SETTINGS)
+ settings.ignore_overwritten_missing_references = True
+ p1 = Parameters({'alpha': '${gamma}'}, settings, '')
+ p2 = Parameters({'alpha': '${beta}'}, settings, '')
+ p3 = Parameters({'gamma': 3}, settings, '')
+ p1.merge(p2)
+ p1.merge(p3)
+ with self.assertRaises(InterpolationError) as error:
+ p1.interpolate()
+ self.assertEqual(error.exception.message, "-> \n Cannot resolve ${beta}, at alpha")
+
+ def test_ignore_overwriten_missing_reference_dict(self):
+ # setting ignore_overwritten_missing_references to true should
+ # not change the behaviour for dicts
+ settings = copy.deepcopy(SETTINGS)
+ settings.ignore_overwritten_missing_references = True
+ p1 = Parameters({'alpha': '${beta}'}, settings, '')
+ p2 = Parameters({'alpha': '${gamma}'}, settings, '')
+ p3 = Parameters({'gamma': {'one': 1, 'two': 2}}, settings, '')
+ err1 = "[WARNING] Reference '${beta}' undefined\n"
+ p1.merge(p2)
+ p1.merge(p3)
+ with self.assertRaises(InterpolationError) as error, mock.patch('sys.stderr', new=MockDevice()) as std_err:
+ p1.interpolate()
+ self.assertEqual(error.exception.message, "-> \n Cannot resolve ${beta}, at alpha")
+ self.assertEqual(std_err.text(), err1)
+
if __name__ == '__main__':
unittest.main()
diff --git a/reclass/defaults.py b/reclass/defaults.py
index 82f49b2..ac8aa34 100644
--- a/reclass/defaults.py
+++ b/reclass/defaults.py
@@ -15,12 +15,16 @@
OPT_NODES_URI = 'nodes'
OPT_CLASSES_URI = 'classes'
OPT_PRETTY_PRINT = True
+OPT_GROUP_ERRORS = True
OPT_NO_REFS = False
OPT_OUTPUT = 'yaml'
+
OPT_IGNORE_CLASS_NOTFOUND = False
OPT_IGNORE_CLASS_NOTFOUND_REGEXP = ['.*']
OPT_IGNORE_CLASS_NOTFOUND_WARNING = True
+OPT_IGNORE_OVERWRITTEN_MISSING_REFERENCES = True
+
OPT_ALLOW_SCALAR_OVER_DICT = False
OPT_ALLOW_SCALAR_OVER_LIST = False
OPT_ALLOW_LIST_OVER_SCALAR = False
diff --git a/reclass/errors.py b/reclass/errors.py
index 901b196..a96c47b 100644
--- a/reclass/errors.py
+++ b/reclass/errors.py
@@ -182,6 +182,24 @@
msg = 'Cannot resolve {0}'.format(self.reference.join(REFERENCE_SENTINELS)) + self._add_context_and_uri()
return [ msg ]
+class ResolveErrorList(InterpolationError):
+ def __init__(self):
+ super(ResolveErrorList, self).__init__(msg=None)
+ self.resolve_errors = []
+ self._traceback = False
+
+ def add(self, resolve_error):
+ self.resolve_errors.append(resolve_error)
+
+ def have_errors(self):
+ return len(self.resolve_errors) > 0
+
+ def _get_error_message(self):
+ msgs = []
+ for e in self.resolve_errors:
+ msgs.extend(e._get_error_message())
+ return msgs
+
class InvQueryError(InterpolationError):
diff --git a/reclass/settings.py b/reclass/settings.py
index 987e707..a1e203e 100644
--- a/reclass/settings.py
+++ b/reclass/settings.py
@@ -25,7 +25,9 @@
self.ignore_class_notfound_regexp = [ self.ignore_class_notfound_regexp ]
self.ignore_class_notfound_warning = options.get('ignore_class_notfound_warning', OPT_IGNORE_CLASS_NOTFOUND_WARNING)
+ self.ignore_overwritten_missing_references = options.get('ignore_overwritten_missing_references', OPT_IGNORE_OVERWRITTEN_MISSING_REFERENCES)
+ self.group_errors = options.get('group_errors', OPT_GROUP_ERRORS)
self.ref_parser = reclass.values.parser_funcs.get_ref_parser(self.escape_character, self.reference_sentinels, self.export_sentinels)
self.simple_ref_parser = reclass.values.parser_funcs.get_simple_ref_parser(self.escape_character, self.reference_sentinels, self.export_sentinels)
diff --git a/reclass/values/compitem.py b/reclass/values/compitem.py
index 5786934..2134ea8 100644
--- a/reclass/values/compitem.py
+++ b/reclass/values/compitem.py
@@ -49,3 +49,6 @@
def __repr__(self):
return 'CompItem(%r)' % self._items
+
+ def __str__(self):
+ return ''.join([ str(i) for i in self._items ])
diff --git a/reclass/values/refitem.py b/reclass/values/refitem.py
index ebb9708..0ae65e6 100644
--- a/reclass/values/refitem.py
+++ b/reclass/values/refitem.py
@@ -5,10 +5,12 @@
#
from item import Item
+from reclass.defaults import REFERENCE_SENTINELS
from reclass.settings import Settings
from reclass.utils.dictpath import DictPath
from reclass.errors import ResolveError
+
class RefItem(Item):
def __init__(self, items, settings):
@@ -62,3 +64,7 @@
def __repr__(self):
return 'RefItem(%r)' % self._items
+
+ def __str__(self):
+ strings = [ str(i) for i in self._items ]
+ return '{0}{1}{2}'.format(REFERENCE_SENTINELS[0], ''.join(strings), REFERENCE_SENTINELS[1])
diff --git a/reclass/values/scaitem.py b/reclass/values/scaitem.py
index df574d9..6fd3194 100644
--- a/reclass/values/scaitem.py
+++ b/reclass/values/scaitem.py
@@ -37,3 +37,6 @@
def __repr__(self):
return 'ScaItem({0!r})'.format(self._value)
+
+ def __str__(self):
+ return str(self._value)
diff --git a/reclass/values/value.py b/reclass/values/value.py
index cbd5ce2..4ec6051 100644
--- a/reclass/values/value.py
+++ b/reclass/values/value.py
@@ -83,3 +83,6 @@
def __repr__(self):
return 'Value(%r)' % self._item
+
+ def __str__(self):
+ return str(self._item)
diff --git a/reclass/values/valuelist.py b/reclass/values/valuelist.py
index f89a0c3..6201564 100644
--- a/reclass/values/valuelist.py
+++ b/reclass/values/valuelist.py
@@ -5,6 +5,9 @@
#
import copy
+import sys
+
+from reclass.errors import ResolveError
class ValueList(object):
@@ -87,12 +90,22 @@
output = None
deepCopied = False
+ last_error = None
for n, value in enumerate(self._values):
+ try:
+ new = value.render(context, inventory)
+ except ResolveError as e:
+ if self._settings.ignore_overwritten_missing_references and not isinstance(output, (dict, list)) and n != (len(self._values)-1):
+ new = None
+ last_error = e
+ print >>sys.stderr, "[WARNING] Reference '%s' undefined" % (str(value))
+ else:
+ raise e
+
if output is None:
- output = self._values[n].render(context, inventory)
+ output = new
deepCopied = False
else:
- new = value.render(context, inventory)
if isinstance(output, dict) and isinstance(new, dict):
p1 = Parameters(output, self._settings, None)
p2 = Parameters(new, self._settings, None)
@@ -109,6 +122,10 @@
raise TypeError('Cannot merge %s over %s' % (repr(self._values[n]), repr(self._values[n-1])))
else:
output = new
+
+ if isinstance(output, (dict, list)) and last_error is not None:
+ raise last_error
+
return output
def __repr__(self):