Merge pull request #54 from salt-formulas/andrewp-fixed-parameters
add fixed/immutable parameter prefix
diff --git a/README-extentions.rst b/README-extentions.rst
index 61364a4..ec10e48 100644
--- a/README-extentions.rst
+++ b/README-extentions.rst
@@ -124,6 +124,55 @@
three: ${one}
+Constant Parameters
+--------------------------
+
+Parameters can be labeled as constant by using the prefix ``=``
+
+.. code-block:: yaml
+
+ parameters:
+ =one: 1
+
+If in the normal parameter merging a constant parameter would be changed then depending
+on the setting of ``strict_constant_parameters`` either an exception is raised (``strict_constant_parameters`` true)
+or the parameter is left unchanged and no notification or error is given (``strict_constant_parameters`` false)
+
+For example with:
+
+.. code-block:: yaml
+
+ # nodes/node1.yml
+ classes:
+ - first
+ - second
+
+ # classes/first.yml
+ parameters:
+ =one: 1
+
+ # classes/second.yml
+ parameters:
+ one: 2
+
+``reclass.py --nodeinfo node1`` then gives an ''Attempt to change constant value'' error if ``strict_constant_parameters``
+is true or gives:
+
+.. code-block:: yaml
+
+ parameters:
+ alpha:
+ one: 1
+
+if ``strict_constant_parameters`` is false
+
+Default value for ``strict_constant_parameters`` is True
+
+.. code-block:: yaml
+
+ strict_constant_parameters: True
+
+
Nested References
-----------------
@@ -155,11 +204,12 @@
Ignore overwritten missing references
--------------------------
+-------------------------------------
Given the following classes:
.. code-block:: yaml
+
# node1.yml
classes:
- class1
@@ -197,6 +247,7 @@
Instead of failing on the first undefinded reference error all missing reference errors are printed at once.
.. code-block:: yaml
+
reclass --nodeinfo mynode
-> dontpanic
Cannot resolve ${_param:kkk}, at mkkek3:tree:to:fail, in yaml_fs:///test/classes/third.yml
@@ -237,6 +288,7 @@
Classes:
.. code-block:: yaml
+
#/etc/reclass/classes/global.yml
parameters:
_class:
diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py
index 34ccdeb..fa0f379 100644
--- a/reclass/datatypes/parameters.py
+++ b/reclass/datatypes/parameters.py
@@ -25,37 +25,12 @@
from collections import namedtuple
from reclass.utils.dictpath import DictPath
+from reclass.utils.parameterdict import ParameterDict
+from reclass.utils.parameterlist import ParameterList
from reclass.values.value import Value
from reclass.values.valuelist import ValueList
from reclass.errors import InfiniteRecursionError, ResolveError, ResolveErrorList, InterpolationError, BadReferencesError
-class ParameterDict(dict):
- def __init__(self, *args, **kwargs):
- self._uri = kwargs.pop('uri', None)
- dict.__init__(self, *args, **kwargs)
-
- @property
- def uri(self):
- return self._uri
-
- @uri.setter
- def uri(self, uri):
- self._uri = uri
-
-
-class ParameterList(list):
- def __init__(self, *args, **kwargs):
- self._uri = kwargs.pop('uri', None)
- list.__init__(self, *args, **kwargs)
-
- @property
- def uri(self):
- return self._uri
-
- @uri.setter
- def uri(self, uri):
- self._uri = uri
-
class Parameters(object):
'''
@@ -83,22 +58,17 @@
def __init__(self, mapping, settings, uri, parse_strings=True):
self._settings = settings
- self._base = {}
self._uri = uri
+ self._base = ParameterDict(uri=self._uri)
self._unrendered = None
self._escapes_handled = {}
self._inv_queries = []
self._resolve_errors = ResolveErrorList()
self._needs_all_envs = False
- self._keep_overrides = False
self._parse_strings = parse_strings
if mapping is not None:
- # we initialise by merging
- self._keep_overrides = True
+ # initialise by merging
self.merge(mapping)
- self._keep_overrides = False
-
- #delimiter = property(lambda self: self._delimiter)
def __len__(self):
return len(self._base)
@@ -129,25 +99,39 @@
def as_dict(self):
return self._base.copy()
- def _wrap_value(self, value, path):
- if isinstance(value, dict):
- return self._wrap_dict(value, path)
- elif isinstance(value, list):
- return self._wrap_list(value, path)
- elif isinstance(value, (Value, ValueList)):
+ def _wrap_value(self, value):
+ if isinstance(value, (Value, ValueList)):
return value
+ elif isinstance(value, dict):
+ return self._wrap_dict(value)
+ elif isinstance(value, list):
+ return self._wrap_list(value)
else:
try:
return Value(value, self._settings, self._uri, parse_string=self._parse_strings)
except InterpolationError as e:
- e.context = str(path)
+ e.context = DictPath(self._settings.delimiter)
raise
- def _wrap_list(self, source, path):
- return ParameterList([ self._wrap_value(v, path.new_subpath(k)) for (k, v) in enumerate(source) ], uri=self._uri)
+ def _wrap_list(self, source):
+ l = ParameterList(uri=self._uri)
+ for (k, v) in enumerate(source):
+ try:
+ l.append(self._wrap_value(v))
+ except InterpolationError as e:
+ e.context.add_ancestor(str(k))
+ raise
+ return l
- def _wrap_dict(self, source, path):
- return ParameterDict({ k: self._wrap_value(v, path.new_subpath(k)) for (k, v) in iteritems(source) }, uri=self._uri)
+ def _wrap_dict(self, source):
+ d = ParameterDict(uri=self._uri)
+ for (k, v) in iteritems(source):
+ try:
+ d[k] = self._wrap_value(v)
+ except InterpolationError as e:
+ e.context.add_ancestor(str(k))
+ raise
+ return d
def _update_value(self, cur, new):
if isinstance(cur, Value):
@@ -155,9 +139,10 @@
elif isinstance(cur, ValueList):
values = cur
else:
- uri = self._uri
if isinstance(cur, (ParameterDict, ParameterList)):
uri = cur.uri
+ else:
+ uri = self._uri
values = ValueList(Value(cur, self._settings, uri), self._settings)
if isinstance(new, Value):
@@ -165,14 +150,15 @@
elif isinstance(new, ValueList):
values.extend(new)
else:
- uri = self._uri
if isinstance(new, (ParameterDict, ParameterList)):
uri = new.uri
+ else:
+ uri = self._uri
values.append(Value(new, self._settings, uri, parse_string=self._parse_strings))
return values
- def _merge_dict(self, cur, new, path):
+ def _merge_dict(self, cur, new):
"""Merge a dictionary with another dictionary.
Iterate over keys in new. If this is not an initialization merge and
@@ -183,7 +169,6 @@
Args:
cur (dict): Current dictionary
new (dict): Dictionary to be merged
- path (string): Merging path from recursion
initmerge (bool): True if called as part of entity init
Returns:
@@ -191,43 +176,50 @@
"""
- ret = cur
- for (key, newvalue) in iteritems(new):
- if key.startswith(self._settings.dict_key_override_prefix) and not self._keep_overrides:
- if not isinstance(newvalue, Value):
- newvalue = Value(newvalue, self._settings, self._uri, parse_string=self._parse_strings)
- newvalue.overwrite = True
- ret[key.lstrip(self._settings.dict_key_override_prefix)] = newvalue
+ for (key, value) in iteritems(new):
+ if key[0] in self._settings.dict_key_prefixes:
+ newkey = key[1:]
+ if not isinstance(value, Value):
+ value = Value(value, self._settings, self._uri, parse_string=self._parse_strings)
+ if key[0] == self._settings.dict_key_override_prefix:
+ value.overwrite = True
+ elif key[0] == self._settings.dict_key_constant_prefix:
+ value.constant = True
+ value = self._merge_recurse(cur.get(newkey), value)
+ key = newkey
else:
- ret[key] = self._merge_recurse(ret.get(key), newvalue, path.new_subpath(key))
+ value = self._merge_recurse(cur.get(key), value)
+ cur[key] = value
+ cur.uri = new.uri
+ return cur
- return ret
-
- def _merge_recurse(self, cur, new, path=None):
+ def _merge_recurse(self, cur, new):
"""Merge a parameter with another parameter.
- Iterate over keys in new. Call _merge_dict, _extend_list, or
- _update_scalar depending on type. Pass along whether this is an
- initialization merge.
+ Iterate over keys in new. Call _merge_dict, _update_value
+ depending on type.
Args:
- cur (dict): Current dictionary
- new (dict): Dictionary to be merged
- path (string): Merging path from recursion
- initmerge (bool): True if called as part of entity init, defaults
- to False
+ cur: Current parameter
+ new: Parameter to be merged
Returns:
- dict: a merged dictionary
+ merged parameter (Value or ValueList)
"""
- if cur is None:
- return new
- elif isinstance(new, dict) and isinstance(cur, dict):
- return self._merge_dict(cur, new, path)
+ if isinstance(new, dict):
+ if cur is None:
+ cur = ParameterDict(uri=self._uri)
+ if isinstance(cur, dict):
+ return self._merge_dict(cur, new)
+ else:
+ return self._update_value(cur, new)
else:
- return self._update_value(cur, new)
+ if cur is None:
+ return new
+ else:
+ return self._update_value(cur, new)
def merge(self, other):
"""Merge function (public edition).
@@ -245,13 +237,13 @@
self._unrendered = None
if isinstance(other, dict):
- wrapped = self._wrap_dict(other, DictPath(self._settings.delimiter))
+ wrapped = self._wrap_dict(other)
elif isinstance(other, self.__class__):
- wrapped = other._wrap_dict(other._base, DictPath(self._settings.delimiter))
+ wrapped = other._wrap_dict(other._base)
else:
raise TypeError('Cannot merge %s objects into %s' % (type(other),
self.__class__.__name__))
- self._base = self._merge_recurse(self._base, wrapped, DictPath(self._settings.delimiter))
+ self._base = self._merge_recurse(self._base, wrapped)
def _render_simple_container(self, container, key, value, path):
if isinstance(value, ValueList):
diff --git a/reclass/datatypes/tests/test_exports.py b/reclass/datatypes/tests/test_exports.py
index caa0522..8ccd6df 100644
--- a/reclass/datatypes/tests/test_exports.py
+++ b/reclass/datatypes/tests/test_exports.py
@@ -21,7 +21,7 @@
e = Exports({'alpha': { 'one': 1, 'two': 2}}, SETTINGS, '')
d = {'alpha': { 'three': 3, 'four': 4}}
e.overwrite(d)
- e.initialise_interpolation()
+ e.interpolate()
self.assertEqual(e.as_dict(), d)
def test_malformed_invquery(self):
diff --git a/reclass/datatypes/tests/test_parameters.py b/reclass/datatypes/tests/test_parameters.py
index b15f8ce..5959197 100644
--- a/reclass/datatypes/tests/test_parameters.py
+++ b/reclass/datatypes/tests/test_parameters.py
@@ -17,9 +17,10 @@
from reclass.settings import Settings
from reclass.datatypes import Parameters
+from reclass.utils.parameterdict import ParameterDict
from reclass.values.value import Value
from reclass.values.scaitem import ScaItem
-from reclass.errors import InfiniteRecursionError, InterpolationError, ResolveError, ResolveErrorList, TypeMergeError
+from reclass.errors import ChangedConstantError, InfiniteRecursionError, InterpolationError, ResolveError, ResolveErrorList, TypeMergeError
import unittest
try:
@@ -46,7 +47,7 @@
def _construct_mocked_params(self, iterable=None, settings=SETTINGS):
p = Parameters(iterable, settings, '')
self._base = base = p._base
- p._base = mock.MagicMock(spec_set=dict, wraps=base)
+ p._base = mock.MagicMock(spec_set=ParameterDict, wraps=base)
p._base.__repr__ = mock.MagicMock(autospec=dict.__repr__,
return_value=repr(base))
p._base.__getitem__.side_effect = base.__getitem__
@@ -291,6 +292,16 @@
p1.interpolate()
self.assertEqual(p1.as_dict()['key'], None)
+ def test_merge_list_over_dict(self):
+ p1 = Parameters({}, SETTINGS, '')
+ p2 = Parameters({'one': { 'a': { 'b': 'c' } } }, SETTINGS, 'second')
+ p3 = Parameters({'one': { 'a': [ 'b' ] } }, SETTINGS, 'third')
+ with self.assertRaises(TypeMergeError) as e:
+ p1.merge(p2)
+ p1.merge(p3)
+ p1.interpolate()
+ self.assertEqual(e.exception.message, "-> \n Canot merge list over dictionary, at one:a, in second; third")
+
# def test_merge_bare_dict_over_dict(self):
# settings = Settings({'allow_bare_override': True})
# p1 = Parameters(dict(key=SIMPLE), settings, '')
@@ -339,7 +350,7 @@
p = Parameters(dict(dict=base), SETTINGS, '')
p2 = Parameters(dict(dict=mergee), SETTINGS, '')
p.merge(p2)
- p.initialise_interpolation()
+ p.interpolate()
self.assertDictEqual(p.as_dict(), dict(dict=goal))
def test_interpolate_single(self):
@@ -744,6 +755,27 @@
p1.interpolate()
self.assertEqual(p1.as_dict(), r)
+ def test_strict_constant_parameter(self):
+ p1 = Parameters({'one': { 'a': 1} }, SETTINGS, 'first')
+ p2 = Parameters({'one': { '=a': 2} }, SETTINGS, 'second')
+ p3 = Parameters({'one': { 'a': 3} }, SETTINGS, 'third')
+ with self.assertRaises(ChangedConstantError) as e:
+ p1.merge(p2)
+ p1.merge(p3)
+ p1.interpolate()
+ self.assertEqual(e.exception.message, "-> \n Attempt to change constant value, at one:a, in second; third")
+
+ def test_constant_parameter(self):
+ settings = Settings({'strict_constant_parameters': False})
+ p1 = Parameters({'one': { 'a': 1} }, settings, 'first')
+ p2 = Parameters({'one': { '=a': 2} }, settings, 'second')
+ p3 = Parameters({'one': { 'a': 3} }, settings, 'third')
+ r = {'one': { 'a': 2 } }
+ p1.merge(p2)
+ p1.merge(p3)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r)
+
if __name__ == '__main__':
unittest.main()
diff --git a/reclass/defaults.py b/reclass/defaults.py
index 5dbd94b..1e50c0e 100644
--- a/reclass/defaults.py
+++ b/reclass/defaults.py
@@ -29,6 +29,7 @@
OPT_IGNORE_CLASS_NOTFOUND_WARNING = True
OPT_IGNORE_OVERWRITTEN_MISSING_REFERENCES = True
+OPT_STRICT_CONSTANT_PARAMETERS = True
OPT_ALLOW_SCALAR_OVER_DICT = False
OPT_ALLOW_SCALAR_OVER_LIST = False
@@ -50,6 +51,7 @@
EXPORT_SENTINELS = ('$[', ']')
PARAMETER_INTERPOLATION_DELIMITER = ':'
PARAMETER_DICT_KEY_OVERRIDE_PREFIX = '~'
+PARAMETER_DICT_KEY_CONSTANT_PREFIX = '='
ESCAPE_CHARACTER = '\\'
AUTOMATIC_RECLASS_PARAMETERS = True
diff --git a/reclass/errors.py b/reclass/errors.py
index fdd7f38..0c9d48f 100644
--- a/reclass/errors.py
+++ b/reclass/errors.py
@@ -15,6 +15,7 @@
import traceback
from reclass.defaults import REFERENCE_SENTINELS, EXPORT_SENTINELS
+from reclass.utils.dictpath import DictPath
class ReclassException(Exception):
@@ -140,7 +141,7 @@
def _add_context_and_uri(self):
msg = ''
if self.context:
- msg += ', at %s' % self.context
+ msg += ', at %s' % str(self.context)
if self.uri:
msg += ', in %s' % self.uri
return msg
@@ -255,6 +256,7 @@
def _get_error_message(self):
msg = [ 'Parse error: {0}'.format(self._line.join(EXPORT_SENTINELS)) + self._add_context_and_uri() ]
msg.append('{0} at char {1}'.format(self._err, self._col - 1))
+ return msg
class InfiniteRecursionError(InterpolationError):
@@ -304,6 +306,16 @@
return msg
+class ChangedConstantError(InterpolationError):
+
+ def __init__(self, uri):
+ super(ChangedConstantError, self).__init__(msg=None, uri=uri, tbFlag=False)
+
+ def _get_error_message(self):
+ msg = [ 'Attempt to change constant value' + self._add_context_and_uri() ]
+ return msg
+
+
class MappingError(ReclassException):
def __init__(self, msg, rc=posix.EX_DATAERR):
diff --git a/reclass/settings.py b/reclass/settings.py
index 6e35dd2..51c518f 100644
--- a/reclass/settings.py
+++ b/reclass/settings.py
@@ -23,12 +23,15 @@
self.default_environment = options.get('default_environment', DEFAULT_ENVIRONMENT)
self.delimiter = options.get('delimiter', PARAMETER_INTERPOLATION_DELIMITER)
self.dict_key_override_prefix = options.get('dict_key_override_prefix', PARAMETER_DICT_KEY_OVERRIDE_PREFIX)
+ self.dict_key_constant_prefix = options.get('dict_key_constant_prefix', PARAMETER_DICT_KEY_CONSTANT_PREFIX)
+ self.dict_key_prefixes = [ str(self.dict_key_override_prefix), str(self.dict_key_constant_prefix) ]
self.escape_character = options.get('escape_character', ESCAPE_CHARACTER)
self.export_sentinels = options.get('export_sentinels', EXPORT_SENTINELS)
self.inventory_ignore_failed_node = options.get('inventory_ignore_failed_node', OPT_INVENTORY_IGNORE_FAILED_NODE)
self.inventory_ignore_failed_render = options.get('inventory_ignore_failed_render', OPT_INVENTORY_IGNORE_FAILED_RENDER)
self.reference_sentinels = options.get('reference_sentinels', REFERENCE_SENTINELS)
self.ignore_class_notfound = options.get('ignore_class_notfound', OPT_IGNORE_CLASS_NOTFOUND)
+ self.strict_constant_parameters = options.get('strict_constant_parameters', OPT_STRICT_CONSTANT_PARAMETERS)
self.ignore_class_notfound_regexp = options.get('ignore_class_notfound_regexp', OPT_IGNORE_CLASS_NOTFOUND_REGEXP)
if isinstance(self.ignore_class_notfound_regexp, string_types):
@@ -53,6 +56,7 @@
and self.default_environment == other.default_environment \
and self.delimiter == other.delimiter \
and self.dict_key_override_prefix == other.dict_key_override_prefix \
+ and self.dict_key_constant_prefix == other.dict_key_constant_prefix \
and self.escape_character == other.escape_character \
and self.export_sentinels == other.export_sentinels \
and self.inventory_ignore_failed_node == other.inventory_ignore_failed_node \
@@ -60,7 +64,8 @@
and self.reference_sentinels == other.reference_sentinels \
and self.ignore_class_notfound == other.ignore_class_notfound \
and self.ignore_class_notfound_regexp == other.ignore_class_notfound_regexp \
- and self.ignore_class_notfound_warning == other.ignore_class_notfound_warning
+ and self.ignore_class_notfound_warning == other.ignore_class_notfound_warning \
+ and self.strict_constant_parameters == other.strict_constant_parameters
def __copy__(self):
cls = self.__class__
diff --git a/reclass/utils/dictpath.py b/reclass/utils/dictpath.py
index 57971b4..32831cf 100644
--- a/reclass/utils/dictpath.py
+++ b/reclass/utils/dictpath.py
@@ -148,6 +148,9 @@
def add_subpath(self, key):
self._parts.append(key)
+ def add_ancestor(self, key):
+ self._parts.insert(0, key)
+
def is_ancestor_of(self, other):
if len(other._parts) <= len(self._parts):
return False
diff --git a/reclass/utils/parameterdict.py b/reclass/utils/parameterdict.py
new file mode 100644
index 0000000..6319a0b
--- /dev/null
+++ b/reclass/utils/parameterdict.py
@@ -0,0 +1,12 @@
+class ParameterDict(dict):
+ def __init__(self, *args, **kwargs):
+ self._uri = kwargs.pop('uri', None)
+ dict.__init__(self, *args, **kwargs)
+
+ @property
+ def uri(self):
+ return self._uri
+
+ @uri.setter
+ def uri(self, uri):
+ self._uri = uri
diff --git a/reclass/utils/parameterlist.py b/reclass/utils/parameterlist.py
new file mode 100644
index 0000000..f2473df
--- /dev/null
+++ b/reclass/utils/parameterlist.py
@@ -0,0 +1,12 @@
+class ParameterList(list):
+ def __init__(self, *args, **kwargs):
+ self._uri = kwargs.pop('uri', None)
+ list.__init__(self, *args, **kwargs)
+
+ @property
+ def uri(self):
+ return self._uri
+
+ @uri.setter
+ def uri(self, uri):
+ self._uri = uri
diff --git a/reclass/values/value.py b/reclass/values/value.py
index 286407c..4e86274 100644
--- a/reclass/values/value.py
+++ b/reclass/values/value.py
@@ -24,6 +24,7 @@
self._settings = settings
self._uri = uri
self._overwrite = False
+ self._constant = False
if isinstance(value, string_types):
if parse_string:
try:
@@ -49,6 +50,14 @@
self._overwrite = overwrite
@property
+ def constant(self):
+ return self._constant
+
+ @constant.setter
+ def constant(self, constant):
+ self._constant = constant
+
+ @property
def uri(self):
return self._uri
diff --git a/reclass/values/valuelist.py b/reclass/values/valuelist.py
index adffb87..9c1e1fa 100644
--- a/reclass/values/valuelist.py
+++ b/reclass/values/valuelist.py
@@ -13,7 +13,9 @@
import copy
import sys
-from reclass.errors import ResolveError, TypeMergeError
+from reclass.errors import ChangedConstantError, ResolveError, TypeMergeError
+
+
class ValueList(object):
@@ -46,7 +48,7 @@
self._is_complex = False
item_type = self._values[0].item_type()
for v in self._values:
- if v.is_complex() or v.overwrite or v.item_type() != item_type:
+ if v.is_complex() or v.constant or v.overwrite or v.item_type() != item_type:
self._is_complex = True
def has_references(self):
@@ -107,6 +109,7 @@
output = None
deepCopied = False
last_error = None
+ constant = False
for n, value in enumerate(self._values):
try:
new = value.render(context, inventory)
@@ -120,6 +123,12 @@
else:
raise e
+ if constant:
+ if self._settings.strict_constant_parameters:
+ raise ChangedConstantError('{0}; {1}'.format(self._values[n-1].uri, self._values[n].uri))
+ else:
+ continue
+
if output is None or value.overwrite:
output = new
deepCopied = False
@@ -170,6 +179,9 @@
output = new
deepCopied = False
+ if value.constant:
+ constant = True
+
if isinstance(output, (dict, list)) and last_error is not None:
raise last_error