Merge pull request #74 from salt-formulas/develop
Bump version 1.7.0
diff --git a/README-extentions.rst b/README-extensions.rst
similarity index 74%
rename from README-extentions.rst
rename to README-extensions.rst
index 2bc4816..e67e441 100644
--- a/README-extentions.rst
+++ b/README-extensions.rst
@@ -336,6 +336,45 @@
...
+Load classes with relative names
+--------------------------------
+
+Load referenced class from a relative location to the current class.
+To load class from relative location start the class uri with "." or ".." char.
+The only supported reference is to nested tree structure below the current class.
+
+You are allowed to use syntax for relative uri to required class on any place on your model (first class loaded, init.yml, regular class .yml).
+
+The feature is expected to improve flexibility while sharing classes between your models.
+
+Please mpte that you can't use '..' without any calss following. If you want simply up in the sctructure, type in '..init'.
+
+It's a new feature use it with care and mind that using "relative syntax" lower traceability of
+your pillar composition.
+
+Example usage of relative class name using '.' and '..':
+
+.. code-block:: yaml
+
+ #/etc/reclass/classes/component/defaults.yml
+ classes:
+ component:
+ config:
+ a: b
+
+.. code-block:: yaml
+
+ #/etc/reclass/classes/component/init.yml
+ classes:
+ - .defaults
+
+.. code-block:: yaml
+
+ #/etc/reclass/classes/component/configuration/init.yml
+ classes:
+ - ..defaults
+
+
Inventory Queries
-----------------
@@ -538,3 +577,117 @@
...
If the subfolder path starts with the underscore character ``_``, then the subfolder path is NOT added to the node name.
+
+
+Git storage type
+----------------
+
+Reclass node and class yaml files can be read from a remote git repository with the yaml_git storage type. Use nodes_uri and
+classes_uri to define the git repos to use for nodes and classes. These can be the same repo.
+
+For salt masters using ssh connections the private and public keys must be readable by the salt daemon, which requires the
+private key NOT be password protected. For stand alone reclass using ssh connections if the privkey and pubkey options
+are not defined then any in memory key (from ssh-add) will be used.
+
+Salt master reclass config example:
+
+.. code-block:: yaml
+
+ storage_type:yaml_git
+ nodes_uri:
+ # branch to use
+ branch: master
+
+ # cache directory (default: ~/.reclass/git/cache)
+ cache_dir: /var/cache/reclass/git
+
+ # lock directory (default: ~/.reclass/git/lock)
+ lock_dir: /var/cache/reclass/lock
+
+ # private key for ssh connections (no default, but will used keys stored
+ # by ssh-add in memory if privkey and pubkey are not set)
+ privkey: /root/salt_rsa
+ # public key for ssh connections
+ pubkey: /root/salt_rsa.pub
+
+ repo: git+ssh://gitlab@remote.server:salt/nodes.git
+
+ classes_uri:
+ # branch to use or __env__ to use the branch matching the node
+ # environment name
+ branch: __env__
+
+ # cache directory (default: ~/.reclass/git/cache)
+ cache_dir: /var/cache/reclass/git
+
+ # lock directory (default: ~/.reclass/git/lock)
+ lock_dir: /var/cache/reclass/lock
+
+ # private key for ssh connections (no default, but will used keys stored
+ # by ssh-add in memory if privkey and pubkey are not set)
+ privkey: /root/salt_rsa
+ # public key for ssh connections
+ pubkey: /root/salt_rsa.pub
+
+ # branch/env overrides for specific branches
+ env_overrides:
+ # prod env uses master branch
+ - prod:
+ branch: master
+ # use master branch for nodes with no environment defined
+ - none:
+ branch: master
+
+ repo: git+ssh://gitlab@remote.server:salt/site.git
+
+ # root directory of the class hierarcy in git repo
+ # defaults to root directory of git repo if not given
+ root: classes
+
+
+Mixed storage type
+------------------
+
+Use a mixture of storage types.
+
+Salt master reclass config example, which by default uses yaml_git storage but overrides the location for
+classes for the pre-prod environment to use a directory on the local disc:
+
+.. code-block:: yaml
+
+ storage_type: mixed
+ nodes_uri:
+ # storage type to use
+ storage_type: yaml_git
+
+ # yaml_git storage options
+ branch: master
+ cache_dir: /var/cache/reclass/git
+ lock_dir: /var/cache/reclass/lock
+ privkey: /root/salt_rsa
+ pubkey: /root/salt_rsa.pub
+ repo: git+ssh://gitlab@remote.server:salt/nodes.git
+
+ classes_uri:
+ # storage type to use
+ storage_type: yaml_git
+
+ # yaml_git storage options
+ branch: __env__
+ cache_dir: /var/cache/reclass/git
+ lock_dir: /var/cache/reclass/lock
+ privkey: /root/salt_rsa
+ pubkey: /root/salt_rsa.pub
+ repo: git+ssh://gitlab@remote.server:salt/site.git
+ root: classes
+
+ env_overrides:
+ - prod:
+ branch: master
+ - none:
+ branch: master
+ - pre-prod:
+ # override storage type for this environment
+ storage_type: yaml_fs
+ # options for yaml_fs storage type
+ uri: /srv/salt/env/pre-prod/classes
diff --git a/README.rst b/README.rst
index b865e4f..5dcb5a1 100644
--- a/README.rst
+++ b/README.rst
@@ -16,15 +16,16 @@
* Ignore class notfound/regexp option
-.. include:: ./README-extensions.rst
-
-
Documentation
=============
-Documentation covering the original version is in the doc directory.
-See the README-extensions.rst file for documentation on the extentions.
+.. _README-extensions: README-extensions.rst
+Documentation covering the original version is in the doc directory.
+See the `README-extensions`_ file for documentation on the extentions.
+
+
+.. include:: ./README-extensions.rst
Reclass related projects/tools
diff --git a/doc/source/changelog.rst b/doc/source/changelog.rst
index d29e837..c3134c5 100644
--- a/doc/source/changelog.rst
+++ b/doc/source/changelog.rst
@@ -5,6 +5,10 @@
========= ========== ========================================================
Version Date Changes
========= ========== ========================================================
+1.7.0 2018-11-06 * Python code and parser refactoring by a-ovchinnikov
+ * Improvements in yaml_git and mixed setup by Andrew Pickford
+ * Relative paths in class names by Petr Michalec, Martin Polreich and Andrew Pickford
+ * Bug Fixes for recently added features
1.5.6 2018-07-30 * Fix, usage of integers as pillar keys
* Refactoring python codebase by @a-ovchinkonv
* New feature, "compose node name" from node subdirectory structure (by @gburiola)
diff --git a/reclass/__init__.py b/reclass/__init__.py
index fe78810..6aabf3d 100644
--- a/reclass/__init__.py
+++ b/reclass/__init__.py
@@ -19,7 +19,7 @@
storage_class = StorageBackendLoader(storage_type).load()
return MemcacheProxy(storage_class(nodes_uri, classes_uri, compose_node_name, **kwargs))
-def get_path_mangler(storage_type,**kwargs):
+def get_path_mangler(storage_type, **kwargs):
return StorageBackendLoader(storage_type).path_mangler()
def output(data, fmt, pretty_print=False, no_refs=False):
diff --git a/reclass/core.py b/reclass/core.py
index bc89738..3e0ab34 100644
--- a/reclass/core.py
+++ b/reclass/core.py
@@ -23,7 +23,6 @@
from six import iteritems
from reclass.settings import Settings
-from reclass.output.yaml_outputter import ExplicitDumper
from reclass.datatypes import Entity, Classes, Parameters, Exports
from reclass.errors import MappingFormatError, ClassNameResolveError, ClassNotFound, InvQueryClassNameResolveError, InvQueryClassNotFound, InvQueryError, InterpolationError, ResolveError
from reclass.values.parser import Parser
@@ -39,7 +38,8 @@
self._settings = settings
self._input_data = input_data
if self._settings.ignore_class_notfound:
- self._cnf_r = re.compile('|'.join([x for x in self._settings.ignore_class_notfound_regexp]))
+ self._cnf_r = re.compile(
+ '|'.join(self._settings.ignore_class_notfound_regexp))
@staticmethod
def _get_timestamp():
@@ -113,7 +113,10 @@
context = Entity(self._settings, name='empty (@{0})'.format(nodename))
for klass in entity.classes.as_list():
- if klass.count('$') > 0:
+ # class name contain reference
+ num_references = klass.count(self._settings.reference_sentinels[0]) +\
+ klass.count(self._settings.export_sentinels[0])
+ if num_references > 0:
try:
klass = str(self._parser.parse(klass, self._settings).render(merge_base.parameters.as_dict(), {}))
except ResolveError as e:
@@ -151,8 +154,16 @@
def _get_automatic_parameters(self, nodename, environment):
if self._settings.automatic_parameters:
- return Parameters({ '_reclass_': { 'name': { 'full': nodename, 'short': nodename.split('.')[0] },
- 'environment': environment } }, self._settings, '__auto__')
+ pars = {
+ '_reclass_': {
+ 'name': {
+ 'full': nodename,
+ 'short': nodename.split('.')[0]
+ },
+ 'environment': environment
+ }
+ }
+ return Parameters(pars, self._settings, '__auto__')
else:
return Parameters({}, self._settings, '')
@@ -161,13 +172,12 @@
for nodename in self._storage.enumerate_nodes():
try:
node_base = self._storage.get_node(nodename, self._settings)
- if node_base.environment == None:
+ if node_base.environment is None:
node_base.environment = self._settings.default_environment
except yaml.scanner.ScannerError as e:
if self._settings.inventory_ignore_failed_node:
continue
- else:
- raise
+ raise
if all_envs or node_base.environment == environment:
try:
@@ -188,7 +198,7 @@
node.interpolate_single_export(q)
except InterpolationError as e:
e.nodename = nodename
- raise InvQueryError(q.contents(), e, context=p, uri=q.uri)
+ raise InvQueryError(q.contents, e, context=p, uri=q.uri)
inventory[nodename] = node.exports.as_dict()
return inventory
@@ -219,7 +229,8 @@
raise
def _nodeinfo_as_dict(self, nodename, entity):
- ret = {'__reclass__' : {'node': entity.name, 'name': nodename,
+ ret = {'__reclass__' : {'node': entity.name,
+ 'name': nodename,
'uri': entity.uri,
'environment': entity.environment,
'timestamp': Core._get_timestamp()
diff --git a/reclass/datatypes/applications.py b/reclass/datatypes/applications.py
index 90ae54c..4f6ee10 100644
--- a/reclass/datatypes/applications.py
+++ b/reclass/datatypes/applications.py
@@ -28,18 +28,14 @@
def __init__(self, iterable=None,
negation_prefix=DEFAULT_NEGATION_PREFIX):
- self._negation_prefix = negation_prefix
+ self.negation_prefix = negation_prefix
self._offset = len(negation_prefix)
self._negations = []
super(Applications, self).__init__(iterable)
- @property
- def negation_prefix(self):
- return self._negation_prefix
-
def append_if_new(self, item):
self._assert_is_string(item)
- if item.startswith(self._negation_prefix):
+ if item.startswith(self.negation_prefix):
item = item[self._offset:]
self._negations.append(item)
try:
@@ -64,6 +60,6 @@
def __repr__(self):
contents = self._items + \
- ['%s%s' % (self._negation_prefix, i) for i in self._negations]
+ ['%s%s' % (self.negation_prefix, i) for i in self._negations]
return "%s(%r, %r)" % (self.__class__.__name__, contents,
- str(self._negation_prefix))
+ str(self.negation_prefix))
diff --git a/reclass/datatypes/classes.py b/reclass/datatypes/classes.py
index 5270e28..fa9cbcf 100644
--- a/reclass/datatypes/classes.py
+++ b/reclass/datatypes/classes.py
@@ -57,7 +57,7 @@
def _assert_is_string(self, item):
if not isinstance(item, six.string_types):
- raise TypeError('%s instances can only contain strings, '\
+ raise TypeError('%s instances can only contain strings, '
'not %s' % (self.__class__.__name__, type(item)))
def _assert_valid_characters(self, item):
diff --git a/reclass/datatypes/entity.py b/reclass/datatypes/entity.py
index 8133de5..2e0e1e4 100644
--- a/reclass/datatypes/entity.py
+++ b/reclass/datatypes/entity.py
@@ -35,7 +35,6 @@
self._environment = environment
name = property(lambda s: s._name)
- short_name = property(lambda s: s._short_name)
uri = property(lambda s: s._uri)
classes = property(lambda s: s._classes)
applications = property(lambda s: s._applications)
@@ -61,10 +60,10 @@
return received_value
def merge(self, other):
- self._classes.merge_unique(other._classes)
- self._applications.merge_unique(other._applications)
- self._parameters.merge(other._parameters)
- self._exports.merge(other._exports)
+ self._classes.merge_unique(other.classes)
+ self._applications.merge_unique(other.applications)
+ self._parameters.merge(other.parameters)
+ self._exports.merge(other.exports)
self._name = other.name
self._uri = other.uri
self._parameters._uri = other.uri
@@ -91,12 +90,12 @@
def __eq__(self, other):
return isinstance(other, type(self)) \
- and self._applications == other._applications \
- and self._classes == other._classes \
- and self._parameters == other._parameters \
- and self._exports == other._exports \
- and self._name == other._name \
- and self._uri == other._uri
+ and self._applications == other.applications \
+ and self._classes == other.classes \
+ and self._parameters == other.parameters \
+ and self._exports == other.exports \
+ and self._name == other.name \
+ and self._uri == other.uri
def __ne__(self, other):
return not self.__eq__(other)
diff --git a/reclass/datatypes/exports.py b/reclass/datatypes/exports.py
index 04ab200..984a15a 100644
--- a/reclass/datatypes/exports.py
+++ b/reclass/datatypes/exports.py
@@ -23,9 +23,6 @@
def __init__(self, mapping, settings, uri):
super(Exports, self).__init__(mapping, settings, uri)
- def __repr__(self):
- return '%s(%r)' % (self.__class__.__name__, self._base)
-
def delete_key(self, key):
self._base.pop(key, None)
self._unrendered.pop(key, None)
diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py
index ee404ce..bab2a28 100644
--- a/reclass/datatypes/parameters.py
+++ b/reclass/datatypes/parameters.py
@@ -105,24 +105,23 @@
e.context = DictPath(self._settings.delimiter)
raise
+ def _get_wrapped(self, position, value):
+ try:
+ return self._wrap_value(value)
+ except InterpolationError as e:
+ e.context.add_ancestor(str(position))
+ raise
+
def _wrap_list(self, source):
l = ParameterList(uri=self._uri)
for (k, v) in enumerate(source):
- try:
- l.append(self._wrap_value(v))
- except InterpolationError as e:
- e.context.add_ancestor(str(k))
- raise
+ l.append(self._get_wrapped(k, v))
return l
def _wrap_dict(self, source):
d = ParameterDict(uri=self._uri)
for (k, v) in iteritems(source):
- try:
- d[k] = self._wrap_value(v)
- except InterpolationError as e:
- e.context.add_ancestor(str(k))
- raise
+ d[k] = self._get_wrapped(k, v)
return d
def _update_value(self, cur, new):
diff --git a/reclass/datatypes/tests/test_exports.py b/reclass/datatypes/tests/test_exports.py
index a0acce7..16a45cb 100644
--- a/reclass/datatypes/tests/test_exports.py
+++ b/reclass/datatypes/tests/test_exports.py
@@ -120,5 +120,14 @@
p.interpolate(e)
self.assertIn(p.as_dict(), [ r1, r2 ])
+ def test_merging_inv_queries(self):
+ e = {'node1': {'a': 1}, 'node2': {'a': 1}, 'node3': {'a': 2}}
+ p1 = Parameters({'exp': '$[ if exports:a == 1 ]'}, SETTINGS, '')
+ p2 = Parameters({'exp': '$[ if exports:a == 2 ]'}, SETTINGS, '')
+ r = { 'exp': [ 'node1', 'node2', 'node3' ] }
+ p1.merge(p2)
+ p1.interpolate(e)
+ self.assertEqual(p1.as_dict(), r)
+
if __name__ == '__main__':
unittest.main()
diff --git a/reclass/settings.py b/reclass/settings.py
index 3e223cc..b7f5252 100644
--- a/reclass/settings.py
+++ b/reclass/settings.py
@@ -5,69 +5,61 @@
from __future__ import print_function
from __future__ import unicode_literals
-import copy
-import reclass.values.parser_funcs
-from reclass.defaults import *
+import reclass.defaults as defaults
-from six import string_types
+from six import string_types, iteritems
+
class Settings(object):
+ known_opts = {
+ 'allow_scalar_over_dict': defaults.OPT_ALLOW_SCALAR_OVER_DICT,
+ 'allow_scalar_over_list': defaults.OPT_ALLOW_SCALAR_OVER_LIST,
+ 'allow_list_over_scalar': defaults.OPT_ALLOW_LIST_OVER_SCALAR,
+ 'allow_dict_over_scalar': defaults.OPT_ALLOW_DICT_OVER_SCALAR,
+ 'allow_none_override': defaults.OPT_ALLOW_NONE_OVERRIDE,
+ 'automatic_parameters': defaults.AUTOMATIC_RECLASS_PARAMETERS,
+ 'default_environment': defaults.DEFAULT_ENVIRONMENT,
+ 'delimiter': defaults.PARAMETER_INTERPOLATION_DELIMITER,
+ 'dict_key_override_prefix':
+ defaults.PARAMETER_DICT_KEY_OVERRIDE_PREFIX,
+ 'dict_key_constant_prefix':
+ defaults.PARAMETER_DICT_KEY_CONSTANT_PREFIX,
+ 'escape_character': defaults.ESCAPE_CHARACTER,
+ 'export_sentinels': defaults.EXPORT_SENTINELS,
+ 'inventory_ignore_failed_node':
+ defaults.OPT_INVENTORY_IGNORE_FAILED_NODE,
+ 'inventory_ignore_failed_render':
+ defaults.OPT_INVENTORY_IGNORE_FAILED_RENDER,
+ 'reference_sentinels': defaults.REFERENCE_SENTINELS,
+ 'ignore_class_notfound': defaults.OPT_IGNORE_CLASS_NOTFOUND,
+ 'strict_constant_parameters':
+ defaults.OPT_STRICT_CONSTANT_PARAMETERS,
+ 'ignore_class_notfound_regexp':
+ defaults.OPT_IGNORE_CLASS_NOTFOUND_REGEXP,
+ 'ignore_class_notfound_warning':
+ defaults.OPT_IGNORE_CLASS_NOTFOUND_WARNING,
+ 'ignore_overwritten_missing_referencesdefaults.':
+ defaults.OPT_IGNORE_OVERWRITTEN_MISSING_REFERENCES,
+ 'group_errors': defaults.OPT_GROUP_ERRORS,
+ 'compose_node_name': defaults.OPT_COMPOSE_NODE_NAME,
+ }
+
def __init__(self, options={}):
- self.allow_scalar_over_dict = options.get('allow_scalar_over_dict', OPT_ALLOW_SCALAR_OVER_DICT)
- self.allow_scalar_over_list = options.get('allow_scalar_over_list', OPT_ALLOW_SCALAR_OVER_LIST)
- self.allow_list_over_scalar = options.get('allow_list_over_scalar', OPT_ALLOW_LIST_OVER_SCALAR)
- self.allow_dict_over_scalar = options.get('allow_dict_over_scalar', OPT_ALLOW_DICT_OVER_SCALAR)
- self.allow_none_override = options.get('allow_none_override', OPT_ALLOW_NONE_OVERRIDE)
- self.automatic_parameters = options.get('automatic_parameters', AUTOMATIC_RECLASS_PARAMETERS)
- self.default_environment = options.get('default_environment', DEFAULT_ENVIRONMENT)
- self.delimiter = options.get('delimiter', PARAMETER_INTERPOLATION_DELIMITER)
- self.dict_key_override_prefix = options.get('dict_key_override_prefix', PARAMETER_DICT_KEY_OVERRIDE_PREFIX)
- self.dict_key_constant_prefix = options.get('dict_key_constant_prefix', PARAMETER_DICT_KEY_CONSTANT_PREFIX)
- self.dict_key_prefixes = [ str(self.dict_key_override_prefix), str(self.dict_key_constant_prefix) ]
- self.escape_character = options.get('escape_character', ESCAPE_CHARACTER)
- self.export_sentinels = options.get('export_sentinels', EXPORT_SENTINELS)
- self.inventory_ignore_failed_node = options.get('inventory_ignore_failed_node', OPT_INVENTORY_IGNORE_FAILED_NODE)
- self.inventory_ignore_failed_render = options.get('inventory_ignore_failed_render', OPT_INVENTORY_IGNORE_FAILED_RENDER)
- self.reference_sentinels = options.get('reference_sentinels', REFERENCE_SENTINELS)
- self.ignore_class_notfound = options.get('ignore_class_notfound', OPT_IGNORE_CLASS_NOTFOUND)
- self.strict_constant_parameters = options.get('strict_constant_parameters', OPT_STRICT_CONSTANT_PARAMETERS)
- self.compose_node_name = options.get('compose_node_name', OPT_COMPOSE_NODE_NAME)
+ for opt_name, opt_value in iteritems(self.known_opts):
+ setattr(self, opt_name, options.get(opt_name, opt_value))
- self.ignore_class_notfound_regexp = options.get('ignore_class_notfound_regexp', OPT_IGNORE_CLASS_NOTFOUND_REGEXP)
+ self.dict_key_prefixes = [str(self.dict_key_override_prefix),
+ str(self.dict_key_constant_prefix)]
if isinstance(self.ignore_class_notfound_regexp, string_types):
- self.ignore_class_notfound_regexp = [ self.ignore_class_notfound_regexp ]
-
- self.ignore_class_notfound_warning = options.get('ignore_class_notfound_warning', OPT_IGNORE_CLASS_NOTFOUND_WARNING)
- self.ignore_overwritten_missing_references = options.get('ignore_overwritten_missing_references', OPT_IGNORE_OVERWRITTEN_MISSING_REFERENCES)
-
- self.group_errors = options.get('group_errors', OPT_GROUP_ERRORS)
-
- self.ref_parser = reclass.values.parser_funcs.get_ref_parser(self.escape_character, self.reference_sentinels, self.export_sentinels)
- self.simple_ref_parser = reclass.values.parser_funcs.get_simple_ref_parser(self.escape_character, self.reference_sentinels, self.export_sentinels)
+ self.ignore_class_notfound_regexp = [
+ self.ignore_class_notfound_regexp]
def __eq__(self, other):
- return isinstance(other, type(self)) \
- and self.allow_scalar_over_dict == other.allow_scalar_over_dict \
- and self.allow_scalar_over_list == other.allow_scalar_over_list \
- and self.allow_list_over_scalar == other.allow_list_over_scalar \
- and self.allow_dict_over_scalar == other.allow_dict_over_scalar \
- and self.allow_none_override == other.allow_none_override \
- and self.automatic_parameters == other.automatic_parameters \
- and self.default_environment == other.default_environment \
- and self.delimiter == other.delimiter \
- and self.dict_key_override_prefix == other.dict_key_override_prefix \
- and self.dict_key_constant_prefix == other.dict_key_constant_prefix \
- and self.escape_character == other.escape_character \
- and self.export_sentinels == other.export_sentinels \
- and self.inventory_ignore_failed_node == other.inventory_ignore_failed_node \
- and self.inventory_ignore_failed_render == other.inventory_ignore_failed_render \
- and self.reference_sentinels == other.reference_sentinels \
- and self.ignore_class_notfound == other.ignore_class_notfound \
- and self.ignore_class_notfound_regexp == other.ignore_class_notfound_regexp \
- and self.ignore_class_notfound_warning == other.ignore_class_notfound_warning \
- and self.strict_constant_parameters == other.strict_constant_parameters \
- and self.compose_node_name == other.compose_node_name
+ if isinstance(other, type(self)):
+ return all(getattr(self, opt) == getattr(other, opt)
+ for opt in self.known_opts)
+ return False
def __copy__(self):
cls = self.__class__
diff --git a/reclass/storage/__init__.py b/reclass/storage/__init__.py
index 3b46a2a..fe873e3 100644
--- a/reclass/storage/__init__.py
+++ b/reclass/storage/__init__.py
@@ -11,6 +11,7 @@
from __future__ import print_function
from __future__ import unicode_literals
+from reclass.storage.common import NameMangler
class NodeStorageBase(object):
@@ -34,3 +35,14 @@
def path_mangler(self):
msg = "Storage class '{0}' does not implement path_mangler."
raise NotImplementedError(msg.format(self.name))
+
+
+class ExternalNodeStorageBase(NodeStorageBase):
+
+ def __init__(self, name, compose_node_name):
+ super(ExternalNodeStorageBase, self).__init__(name)
+ self.class_name_mangler = NameMangler.classes
+ if compose_node_name:
+ self.node_name_mangler = NameMangler.composed_nodes
+ else:
+ self.node_name_mangler = NameMangler.nodes
diff --git a/reclass/storage/loader.py b/reclass/storage/loader.py
index aab554a..0a66a66 100644
--- a/reclass/storage/loader.py
+++ b/reclass/storage/loader.py
@@ -32,5 +32,6 @@
def path_mangler(self, name='path_mangler'):
function = getattr(self._module, name, None)
if function is None:
- raise AttributeError('Storage backend class {0} does not export "{1}"'.format(self._name, name))
+ raise AttributeError('Storage backend class {0} does not export '
+ '"{1}"'.format(self._name, name))
return function
diff --git a/reclass/storage/mixed/__init__.py b/reclass/storage/mixed/__init__.py
index 6324c74..45262cc 100644
--- a/reclass/storage/mixed/__init__.py
+++ b/reclass/storage/mixed/__init__.py
@@ -14,7 +14,7 @@
import reclass.errors
from reclass import get_storage
-from reclass.storage import NodeStorageBase
+from reclass.storage import ExternalNodeStorageBase
def path_mangler(inventory_base_uri, nodes_uri, classes_uri):
if nodes_uri == classes_uri:
@@ -23,17 +23,17 @@
STORAGE_NAME = 'mixed'
-class ExternalNodeStorage(NodeStorageBase):
+class ExternalNodeStorage(ExternalNodeStorageBase):
MixedUri = collections.namedtuple('MixedURI', 'storage_type options')
- def __init__(self, nodes_uri, classes_uri):
- super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
+ def __init__(self, nodes_uri, classes_uri, compose_node_name):
+ super(ExternalNodeStorage, self).__init__(STORAGE_NAME, compose_node_name)
self._nodes_uri = self._uri(nodes_uri)
- self._nodes_storage = get_storage(self._nodes_uri.storage_type, self._nodes_uri.options, None)
+ self._nodes_storage = get_storage(self._nodes_uri.storage_type, self._nodes_uri.options, None, compose_node_name)
self._classes_default_uri = self._uri(classes_uri)
- self._classes_default_storage = get_storage(self._classes_default_uri.storage_type, None, self._classes_default_uri.options)
+ self._classes_default_storage = get_storage(self._classes_default_uri.storage_type, None, self._classes_default_uri.options, compose_node_name)
self._classes_storage = dict()
if 'env_overrides' in classes_uri:
@@ -42,7 +42,7 @@
uri = copy.deepcopy(classes_uri)
uri.update(options)
uri = self._uri(uri)
- self._classes_storage[env] = get_storage(uri.storage_type, None, uri.options)
+ self._classes_storage[env] = get_storage(uri.storage_type, None, uri.options, compose_node_name)
def _uri(self, uri):
ret = copy.deepcopy(uri)
diff --git a/reclass/storage/yaml_fs/__init__.py b/reclass/storage/yaml_fs/__init__.py
index 7ed3fe4..20e8eec 100644
--- a/reclass/storage/yaml_fs/__init__.py
+++ b/reclass/storage/yaml_fs/__init__.py
@@ -15,8 +15,7 @@
import fnmatch
import yaml
from reclass.output.yaml_outputter import ExplicitDumper
-from reclass.storage import NodeStorageBase
-from reclass.storage.common import NameMangler
+from reclass.storage import ExternalNodeStorageBase
from reclass.storage.yamldata import YamlData
from .directory import Directory
from reclass.datatypes import Entity
@@ -53,22 +52,18 @@
return n, c
-class ExternalNodeStorage(NodeStorageBase):
+class ExternalNodeStorage(ExternalNodeStorageBase):
def __init__(self, nodes_uri, classes_uri, compose_node_name):
- super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
+ super(ExternalNodeStorage, self).__init__(STORAGE_NAME, compose_node_name)
if nodes_uri is not None:
self._nodes_uri = nodes_uri
- if compose_node_name:
- self._nodes = self._enumerate_inventory(nodes_uri, NameMangler.composed_nodes)
- else:
- self._nodes = self._enumerate_inventory(nodes_uri, NameMangler.nodes)
-
+ self._nodes = self._enumerate_inventory(nodes_uri, self.node_name_mangler)
if classes_uri is not None:
self._classes_uri = classes_uri
- self._classes = self._enumerate_inventory(classes_uri, NameMangler.classes)
+ self._classes = self._enumerate_inventory(classes_uri, self.class_name_mangler)
nodes_uri = property(lambda self: self._nodes_uri)
classes_uri = property(lambda self: self._classes_uri)
@@ -101,7 +96,6 @@
try:
relpath = self._nodes[name]
path = os.path.join(self.nodes_uri, relpath)
- name = os.path.splitext(relpath)[0]
except KeyError as e:
raise reclass.errors.NodeNotFound(self.name, name, self.nodes_uri)
entity = YamlData.from_file(path).get_entity(name, settings)
diff --git a/reclass/storage/yaml_git/__init__.py b/reclass/storage/yaml_git/__init__.py
index 38de092..a28079b 100644
--- a/reclass/storage/yaml_git/__init__.py
+++ b/reclass/storage/yaml_git/__init__.py
@@ -9,8 +9,11 @@
import collections
import distutils.version
+import errno
+import fcntl
import fnmatch
import os
+import time
# Squelch warning on centos7 due to upgrading cffi
# see https://github.com/saltstack/salt/pull/39871
@@ -28,8 +31,7 @@
from six import iteritems
import reclass.errors
-from reclass.storage import NodeStorageBase
-from reclass.storage.common import NameMangler
+from reclass.storage import ExternalNodeStorageBase
from reclass.storage.yamldata import YamlData
FILE_EXTENSION = '.yml'
@@ -51,6 +53,7 @@
self.branch = None
self.root = None
self.cache_dir = None
+ self.lock_dir = None
self.pubkey = None
self.privkey = None
self.password = None
@@ -60,6 +63,7 @@
if 'repo' in dictionary: self.repo = dictionary['repo']
if 'branch' in dictionary: self.branch = dictionary['branch']
if 'cache_dir' in dictionary: self.cache_dir = dictionary['cache_dir']
+ if 'lock_dir' in dictionary: self.lock_dir = dictionary['lock_dir']
if 'pubkey' in dictionary: self.pubkey = dictionary['pubkey']
if 'privkey' in dictionary: self.privkey = dictionary['privkey']
if 'password' in dictionary: self.password = dictionary['password']
@@ -73,9 +77,32 @@
return '<{0}: {1} {2} {3}>'.format(self.__class__.__name__, self.repo, self.branch, self.root)
-class GitRepo(object):
+class LockFile():
+ def __init__(self, file):
+ self._file = file
- def __init__(self, uri):
+ def __enter__(self):
+ self._fd = open(self._file, 'w+')
+ start = time.time()
+ while True:
+ if (time.time() - start) > 120:
+ raise IOError('Timeout waiting to lock file: {0}'.format(self._file))
+ try:
+ fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ break
+ except IOError as e:
+ # raise on unrelated IOErrors
+ if e.errno != errno.EAGAIN:
+ raise
+ else:
+ time.sleep(0.1)
+
+ def __exit__(self, type, value, traceback):
+ self._fd.close()
+
+
+class GitRepo(object):
+ def __init__(self, uri, node_name_mangler, class_name_mangler):
if pygit2 is None:
raise errors.MissingModuleError('pygit2')
self.transport, _, self.url = uri.repo.partition('://')
@@ -86,9 +113,18 @@
self.cache_dir = '{0}/{1}/{2}'.format(os.path.expanduser("~"), '.reclass/cache/git', self.name)
else:
self.cache_dir = '{0}/{1}'.format(uri.cache_dir, self.name)
-
- self._init_repo(uri)
- self._fetch()
+ if uri.lock_dir is None:
+ self.lock_file = '{0}/{1}/{2}'.format(os.path.expanduser("~"), '.reclass/cache/lock', self.name)
+ else:
+ self.lock_file = '{0}/{1}'.format(uri.lock_dir, self.name)
+ lock_dir = os.path.dirname(self.lock_file)
+ if not os.path.exists(lock_dir):
+ os.makedirs(lock_dir)
+ self._node_name_mangler = node_name_mangler
+ self._class_name_mangler = class_name_mangler
+ with LockFile(self.lock_file):
+ self._init_repo(uri)
+ self._fetch()
self.branches = self.repo.listall_branches()
self.files = self.files_in_repo()
@@ -98,10 +134,7 @@
else:
os.makedirs(self.cache_dir)
self.repo = pygit2.init_repository(self.cache_dir, bare=True)
-
- if not self.repo.remotes:
self.repo.create_remote('origin', self.url)
-
if 'ssh' in self.transport:
if '@' in self.url:
user, _, _ = self.url.partition('@')
@@ -129,7 +162,6 @@
if self.credentials is not None:
origin.credentials = self.credentials
fetch_results = origin.fetch(**fetch_kwargs)
-
remote_branches = self.repo.listall_branches(pygit2.GIT_BRANCH_REMOTE)
local_branches = self.repo.listall_branches()
for remote_branch_name in remote_branches:
@@ -184,7 +216,8 @@
if fnmatch.fnmatch(file.name, '*{0}'.format(FILE_EXTENSION)):
name = os.path.splitext(file.name)[0]
relpath = os.path.dirname(file.path)
- relpath, name = NameMangler.classes(relpath, name)
+ if callable(self._class_name_mangler):
+ relpath, name = self._class_name_mangler(relpath, name)
if name in ret:
raise reclass.errors.DuplicateNodeNameError(self.name + ' - ' + bname, name, ret[name], path)
else:
@@ -197,16 +230,19 @@
for (name, file) in iteritems(self.files[branch]):
if subdir is None or name.startswith(subdir):
node_name = os.path.splitext(file.name)[0]
+ relpath = os.path.dirname(file.path)
+ if callable(self._node_name_mangler):
+ relpath, node_name = self._node_name_mangler(relpath, node_name)
if node_name in ret:
raise reclass.errors.DuplicateNodeNameError(self.name, name, files[name], path)
else:
ret[node_name] = file
return ret
-class ExternalNodeStorage(NodeStorageBase):
- def __init__(self, nodes_uri, classes_uri):
- super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
+class ExternalNodeStorage(ExternalNodeStorageBase):
+ def __init__(self, nodes_uri, classes_uri, compose_node_name):
+ super(ExternalNodeStorage, self).__init__(STORAGE_NAME, compose_node_name)
self._repos = dict()
if nodes_uri is not None:
@@ -261,7 +297,7 @@
def _load_repo(self, uri):
if uri.repo not in self._repos:
- self._repos[uri.repo] = GitRepo(uri)
+ self._repos[uri.repo] = GitRepo(uri, self.node_name_mangler, self.class_name_mangler)
def _env_to_uri(self, environment):
ret = None
diff --git a/reclass/storage/yamldata.py b/reclass/storage/yamldata.py
index a861154..a38b589 100644
--- a/reclass/storage/yamldata.py
+++ b/reclass/storage/yamldata.py
@@ -53,6 +53,33 @@
def get_data(self):
return self._data
+ def set_absolute_names(self, name, names):
+ new_names = []
+ for n in names:
+ if n[0] == '.':
+ dots = self.count_dots(n)
+ levels_up = (dots * (-1))
+ parent = '.'.join(name.split('.')[0:levels_up])
+ if parent == '':
+ n = n[dots:]
+ else:
+ n = parent + n[dots - 1:]
+ new_names.append(n)
+ return new_names
+
+ def yield_dots(self, value):
+ try:
+ idx = value.index('.')
+ except ValueError:
+ return
+ if idx == 0:
+ yield '.'
+ for dot in self.yield_dots(value[1:]):
+ yield dot
+
+ def count_dots(self, value):
+ return len(list(self.yield_dots(value)))
+
def get_entity(self, name, settings):
#if name is None:
# name = self._uri
@@ -60,6 +87,7 @@
classes = self._data.get('classes')
if classes is None:
classes = []
+ classes = self.set_absolute_names(name, classes)
classes = datatypes.Classes(classes)
applications = self._data.get('applications')
diff --git a/reclass/tests/data/02/classes/four.yml b/reclass/tests/data/02/classes/four.yml
new file mode 100644
index 0000000..1f9873c
--- /dev/null
+++ b/reclass/tests/data/02/classes/four.yml
@@ -0,0 +1,2 @@
+parameters:
+ four_alpha: 3
diff --git a/reclass/tests/data/02/classes/init.yml b/reclass/tests/data/02/classes/init.yml
new file mode 100644
index 0000000..e40b899
--- /dev/null
+++ b/reclass/tests/data/02/classes/init.yml
@@ -0,0 +1,2 @@
+parameters:
+ alpha_init: 5
\ No newline at end of file
diff --git a/reclass/tests/data/02/classes/one/alpha.yml b/reclass/tests/data/02/classes/one/alpha.yml
new file mode 100644
index 0000000..9454cd0
--- /dev/null
+++ b/reclass/tests/data/02/classes/one/alpha.yml
@@ -0,0 +1,13 @@
+classes:
+- .beta
+- two.beta
+- ..four
+- ..two.gamma
+- ..init
+
+parameters:
+ test1: ${one_beta}
+ test2: ${two_beta}
+ test3: ${four_alpha}
+ test4: ${two_gamma}
+ test5: ${alpha_init}
diff --git a/reclass/tests/data/02/classes/one/beta.yml b/reclass/tests/data/02/classes/one/beta.yml
new file mode 100644
index 0000000..f754252
--- /dev/null
+++ b/reclass/tests/data/02/classes/one/beta.yml
@@ -0,0 +1,2 @@
+parameters:
+ one_beta: 1
diff --git a/reclass/tests/data/02/classes/three.yml b/reclass/tests/data/02/classes/three.yml
new file mode 100644
index 0000000..987fde0
--- /dev/null
+++ b/reclass/tests/data/02/classes/three.yml
@@ -0,0 +1,2 @@
+classes:
+- .one.alpha
diff --git a/reclass/tests/data/02/classes/two/beta.yml b/reclass/tests/data/02/classes/two/beta.yml
new file mode 100644
index 0000000..1f578b2
--- /dev/null
+++ b/reclass/tests/data/02/classes/two/beta.yml
@@ -0,0 +1,2 @@
+parameters:
+ two_beta: 2
diff --git a/reclass/tests/data/02/classes/two/gamma.yml b/reclass/tests/data/02/classes/two/gamma.yml
new file mode 100644
index 0000000..a1d71da
--- /dev/null
+++ b/reclass/tests/data/02/classes/two/gamma.yml
@@ -0,0 +1,2 @@
+parameters:
+ two_gamma: 4
diff --git a/reclass/tests/data/02/nodes/relative.yml b/reclass/tests/data/02/nodes/relative.yml
new file mode 100644
index 0000000..1f2bbdc
--- /dev/null
+++ b/reclass/tests/data/02/nodes/relative.yml
@@ -0,0 +1,2 @@
+classes:
+ - one.alpha
diff --git a/reclass/tests/data/02/nodes/top_relative.yml b/reclass/tests/data/02/nodes/top_relative.yml
new file mode 100644
index 0000000..5dae5be
--- /dev/null
+++ b/reclass/tests/data/02/nodes/top_relative.yml
@@ -0,0 +1,2 @@
+classes:
+ - three
diff --git a/reclass/tests/data/03/classes/a.yml b/reclass/tests/data/03/classes/a.yml
new file mode 100644
index 0000000..748a297
--- /dev/null
+++ b/reclass/tests/data/03/classes/a.yml
@@ -0,0 +1,6 @@
+parameters:
+ a: 1
+ alpha:
+ - ${a}
+ beta:
+ a: ${a}
diff --git a/reclass/tests/data/03/classes/b.yml b/reclass/tests/data/03/classes/b.yml
new file mode 100644
index 0000000..cce2609
--- /dev/null
+++ b/reclass/tests/data/03/classes/b.yml
@@ -0,0 +1,6 @@
+parameters:
+ b: 2
+ alpha:
+ - ${b}
+ beta:
+ b: ${b}
diff --git a/reclass/tests/data/03/classes/c.yml b/reclass/tests/data/03/classes/c.yml
new file mode 100644
index 0000000..7441417
--- /dev/null
+++ b/reclass/tests/data/03/classes/c.yml
@@ -0,0 +1,6 @@
+parameters:
+ c: 3
+ alpha:
+ - ${c}
+ beta:
+ c: ${c}
diff --git a/reclass/tests/data/03/classes/d.yml b/reclass/tests/data/03/classes/d.yml
new file mode 100644
index 0000000..e61a1ff
--- /dev/null
+++ b/reclass/tests/data/03/classes/d.yml
@@ -0,0 +1,6 @@
+parameters:
+ d: 4
+ alpha:
+ - ${d}
+ beta:
+ d: ${d}
diff --git a/reclass/tests/data/03/nodes/alpha/one.yml b/reclass/tests/data/03/nodes/alpha/one.yml
new file mode 100644
index 0000000..f2b613d
--- /dev/null
+++ b/reclass/tests/data/03/nodes/alpha/one.yml
@@ -0,0 +1,3 @@
+classes:
+- a
+- b
diff --git a/reclass/tests/data/03/nodes/alpha/two.yml b/reclass/tests/data/03/nodes/alpha/two.yml
new file mode 100644
index 0000000..b020af3
--- /dev/null
+++ b/reclass/tests/data/03/nodes/alpha/two.yml
@@ -0,0 +1,3 @@
+classes:
+- a
+- c
diff --git a/reclass/tests/data/03/nodes/beta/one.yml b/reclass/tests/data/03/nodes/beta/one.yml
new file mode 100644
index 0000000..168a4fb
--- /dev/null
+++ b/reclass/tests/data/03/nodes/beta/one.yml
@@ -0,0 +1,3 @@
+classes:
+- b
+- c
diff --git a/reclass/tests/data/03/nodes/beta/two.yml b/reclass/tests/data/03/nodes/beta/two.yml
new file mode 100644
index 0000000..56c6343
--- /dev/null
+++ b/reclass/tests/data/03/nodes/beta/two.yml
@@ -0,0 +1,3 @@
+classes:
+- c
+- d
diff --git a/reclass/tests/test_core.py b/reclass/tests/test_core.py
index 047bf24..4827177 100644
--- a/reclass/tests/test_core.py
+++ b/reclass/tests/test_core.py
@@ -59,6 +59,33 @@
params = { 'node_test': 'class not found', '_reclass_': { 'environment': 'base', 'name': {'full': 'class_notfound', 'short': 'class_notfound' } } }
self.assertEqual(node['parameters'], params)
+ def test_relative_class_names(self):
+ reclass = self._core('02')
+ node = reclass.nodeinfo('relative')
+ params = { 'test1': 1, 'test2': 2, 'test3': 3, 'test4': 4, 'test5': 5, 'one_beta': 1, 'two_beta': 2, 'four_alpha': 3, 'two_gamma': 4, 'alpha_init': 5, '_reclass_': { 'environment': 'base', 'name': { 'full': 'relative', 'short': 'relative' } } }
+ self.assertEqual(node['parameters'], params)
+
+ def test_top_relative_class_names(self):
+ reclass = self._core('02')
+ node = reclass.nodeinfo('top_relative')
+ params = { 'test1': 1, 'test2': 2, 'test3': 3, 'test4': 4, 'test5': 5, 'one_beta': 1, 'two_beta': 2, 'four_alpha': 3, 'two_gamma': 4, 'alpha_init': 5, '_reclass_': { 'environment': 'base', 'name': { 'full': 'top_relative', 'short': 'top_relative' } } }
+ self.assertEqual(node['parameters'], params)
+
+ def test_compose_node_names(self):
+ reclass = self._core('03', {'compose_node_name': True})
+ alpha_one_node = reclass.nodeinfo('alpha.one')
+ alpha_one_res = {'a': 1, 'alpha': [1, 2], 'beta': {'a': 1, 'b': 2}, 'b': 2, '_reclass_': {'environment': 'base', 'name': {'full': 'alpha.one', 'short': 'alpha'}}}
+ alpha_two_node = reclass.nodeinfo('alpha.two')
+ alpha_two_res = {'a': 1, 'alpha': [1, 3], 'beta': {'a': 1, 'c': 3}, 'c': 3, '_reclass_': {'environment': 'base', 'name': {'full': 'alpha.two', 'short': 'alpha'}}}
+ beta_one_node = reclass.nodeinfo('beta.one')
+ beta_one_res = {'alpha': [2, 3], 'beta': {'c': 3, 'b': 2}, 'b': 2, 'c': 3, '_reclass_': {'environment': 'base', 'name': {'full': 'beta.one', 'short': 'beta'}}}
+ beta_two_node = reclass.nodeinfo('beta.two')
+ beta_two_res = {'alpha': [3, 4], 'c': 3, 'beta': {'c': 3, 'd': 4}, 'd': 4, '_reclass_': {'environment': u'base', 'name': {'full': u'beta.two', 'short': u'beta'}}}
+ self.assertEqual(alpha_one_node['parameters'], alpha_one_res)
+ self.assertEqual(alpha_two_node['parameters'], alpha_two_res)
+ self.assertEqual(beta_one_node['parameters'], beta_one_res)
+ self.assertEqual(beta_two_node['parameters'], beta_two_res)
+
if __name__ == '__main__':
unittest.main()
diff --git a/reclass/values/invitem.py b/reclass/values/invitem.py
index 0179f4f..adb1cb6 100644
--- a/reclass/values/invitem.py
+++ b/reclass/values/invitem.py
@@ -110,8 +110,9 @@
subtests = list(it.compress(expr, it.cycle([1, 1, 1, 0])))
self._els = [EqualityTest(subtests[j:j+3], self._delimiter)
for j in range(0, len(subtests), 3)]
- self.refs = [x.refs for x in self._els]
- self.inv_refs = [x.inv_refs for x in self._els]
+ for x in self._els:
+ self.refs.extend(x.refs)
+ self.inv_refs.extend(x.inv_refs)
try:
self._ops = [self.known_operators[x[1]] for x in expr[3::4]]
except KeyError as e:
@@ -134,6 +135,7 @@
def __init__(self, newitem, settings):
super(InvItem, self).__init__(newitem.render(None, None), settings)
self.needs_all_envs = False
+ self.has_inv_query = True
self.ignore_failed_render = (
self._settings.inventory_ignore_failed_render)
self._parse_expression(self.contents)
@@ -178,16 +180,15 @@
raise ExpressionError(msg % self._expr_type, tbFlag=False)
@property
- def has_inv_query(self):
- return True
-
- @property
def has_references(self):
return len(self._question.refs) > 0
def get_references(self):
return self._question.refs
+ def assembleRefs(self, context):
+ return
+
def get_inv_references(self):
return self.inv_refs
diff --git a/reclass/values/item.py b/reclass/values/item.py
index ee46995..45aeb77 100644
--- a/reclass/values/item.py
+++ b/reclass/values/item.py
@@ -22,6 +22,7 @@
def __init__(self, item, settings):
self._settings = settings
self.contents = item
+ self.has_inv_query = False
def allRefs(self):
return True
@@ -30,10 +31,6 @@
def has_references(self):
return False
- @property
- def has_inv_query(self):
- return False
-
def is_container(self):
return False
@@ -60,6 +57,10 @@
def __init__(self, items, settings):
super(ItemWithReferences, self).__init__(items, settings)
+ try:
+ iter(self.contents)
+ except TypeError:
+ self.contents = [self.contents]
self.assembleRefs()
@property
@@ -81,6 +82,7 @@
if item.allRefs is False:
self.allRefs = False
+
class ContainerItem(Item):
def is_container(self):
diff --git a/reclass/values/parser.py b/reclass/values/parser.py
index 914e825..3f7ac1f 100644
--- a/reclass/values/parser.py
+++ b/reclass/values/parser.py
@@ -16,53 +16,74 @@
from .scaitem import ScaItem
from reclass.errors import ParseError
-from reclass.values.parser_funcs import STR, REF, INV
+from reclass.values.parser_funcs import tags
+import reclass.values.parser_funcs as parsers
+
+import collections
+import six
+
class Parser(object):
+ def __init__(self):
+ self._ref_parser = None
+ self._simple_parser = None
+ self._old_settings = None
+
+ @property
+ def ref_parser(self):
+ if self._ref_parser is None or self._settings != self._old_settings:
+ self._ref_parser = parsers.get_ref_parser(self._settings)
+ self._old_settings = self._settings
+ return self._ref_parser
+
+ @property
+ def simple_ref_parser(self):
+ if self._simple_parser is None or self._settings != self._old_settings:
+ self._simple_parser = parsers.get_simple_ref_parser(self._settings)
+ self._old_settings = self._settings
+ return self._simple_parser
+
def parse(self, value, settings):
- self._settings = settings
- dollars = value.count('$')
- if dollars == 0:
- # speed up: only use pyparsing if there is a $ in the string
- return ScaItem(value, self._settings)
- elif dollars == 1:
- # speed up: try a simple reference
+ def full_parse():
try:
- tokens = self._settings.simple_ref_parser.leaveWhitespace().parseString(value).asList()
- except pp.ParseException:
- # fall back on the full parser
- try:
- tokens = self._settings.ref_parser.leaveWhitespace().parseString(value).asList()
- except pp.ParseException as e:
- raise ParseError(e.msg, e.line, e.col, e.lineno)
- else:
- # use the full parser
- try:
- tokens = self._settings.ref_parser.leaveWhitespace().parseString(value).asList()
+ return self.ref_parser.parseString(value)
except pp.ParseException as e:
raise ParseError(e.msg, e.line, e.col, e.lineno)
+ self._settings = settings
+ sentinel_count = (value.count(settings.reference_sentinels[0]) +
+ value.count(settings.export_sentinels[0]))
+ if sentinel_count == 0:
+ # speed up: only use pyparsing if there are sentinels in the value
+ return ScaItem(value, self._settings)
+ elif sentinel_count == 1: # speed up: try a simple reference
+ try:
+ tokens = self.simple_ref_parser.parseString(value)
+ except pp.ParseException:
+ tokens = full_parse() # fall back on the full parser
+ else:
+ tokens = full_parse() # use the full parser
+
+ tokens = parsers.listify(tokens)
items = self._create_items(tokens)
if len(items) == 1:
return items[0]
- else:
- return CompItem(items, self._settings)
+ return CompItem(items, self._settings)
- _create_dict = { STR: (lambda s, v: ScaItem(v, s._settings)),
- REF: (lambda s, v: s._create_ref(v)),
- INV: (lambda s, v: s._create_inv(v)) }
+ _item_builders = {tags.STR: (lambda s, v: ScaItem(v, s._settings)),
+ tags.REF: (lambda s, v: s._create_ref(v)),
+ tags.INV: (lambda s, v: s._create_inv(v)) }
def _create_items(self, tokens):
- return [ self._create_dict[t](self, v) for t, v in tokens ]
+ return [self._item_builders[t](self, v) for t, v in tokens ]
def _create_ref(self, tokens):
- items = [ self._create_dict[t](self, v) for t, v in tokens ]
+ items = [ self._item_builders[t](self, v) for t, v in tokens ]
return RefItem(items, self._settings)
def _create_inv(self, tokens):
- items = [ ScaItem(v, self._settings) for t, v in tokens ]
+ items = [ScaItem(v, self._settings) for t, v in tokens]
if len(items) == 1:
return InvItem(items[0], self._settings)
- else:
- return InvItem(CompItem(items), self._settings)
+ return InvItem(CompItem(items), self._settings)
diff --git a/reclass/values/parser_funcs.py b/reclass/values/parser_funcs.py
index 50babd0..db34ceb 100644
--- a/reclass/values/parser_funcs.py
+++ b/reclass/values/parser_funcs.py
@@ -8,12 +8,13 @@
from __future__ import print_function
from __future__ import unicode_literals
-import pyparsing as pp
+import collections
+import enum
import functools
+import pyparsing as pp
+import six
-STR = 1
-REF = 2
-INV = 3
+tags = enum.Enum('Tags', ['STR', 'REF', 'INV'])
_OBJ = 'OBJ'
_LOGICAL = 'LOGICAL'
@@ -34,15 +35,29 @@
ALL_ENVS = '+AllEnvs'
+s_end = pp.StringEnd()
+
def _tag_with(tag, transform=lambda x:x):
def inner(tag, string, location, tokens):
token = transform(tokens[0])
tokens[0] = (tag, token)
return functools.partial(inner, tag)
-def get_expression_parser():
+def _asList(x):
+ if isinstance(x, pp.ParseResults):
+ return x.asList()
+ return x
- s_end = pp.StringEnd()
+def listify(w, modifier=_asList):
+ if (isinstance(w, collections.Iterable) and
+ not isinstance(w, six.string_types)):
+ cls = type(w)
+ if cls == pp.ParseResults:
+ cls = list
+ return cls([listify(x) for x in w])
+ return modifier(w)
+
+def get_expression_parser():
sign = pp.Optional(pp.Literal('-'))
number = pp.Word(pp.nums)
dpoint = pp.Literal('.')
@@ -80,12 +95,11 @@
line = options + expr + s_end
return line
-def get_ref_parser(escape_character, reference_sentinels, export_sentinels):
- _ESCAPE = escape_character
+def get_ref_parser(settings):
+ _ESCAPE = settings.escape_character
_DOUBLE_ESCAPE = _ESCAPE + _ESCAPE
- _REF_OPEN = reference_sentinels[0]
- _REF_CLOSE = reference_sentinels[1]
+ _REF_OPEN, _REF_CLOSE = settings.reference_sentinels
_REF_CLOSE_FIRST = _REF_CLOSE[0]
_REF_ESCAPE_OPEN = _ESCAPE + _REF_OPEN
_REF_ESCAPE_CLOSE = _ESCAPE + _REF_CLOSE
@@ -93,8 +107,7 @@
_REF_DOUBLE_ESCAPE_CLOSE = _DOUBLE_ESCAPE + _REF_CLOSE
_REF_EXCLUDES = _ESCAPE + _REF_OPEN + _REF_CLOSE
- _INV_OPEN = export_sentinels[0]
- _INV_CLOSE = export_sentinels[1]
+ _INV_OPEN, _INV_CLOSE = settings.export_sentinels
_INV_CLOSE_FIRST = _INV_CLOSE[0]
_INV_ESCAPE_OPEN = _ESCAPE + _INV_OPEN
_INV_ESCAPE_CLOSE = _ESCAPE + _INV_CLOSE
@@ -119,10 +132,10 @@
ref_escape_close = pp.Literal(_REF_ESCAPE_CLOSE).setParseAction(pp.replaceWith(_REF_CLOSE))
ref_text = pp.CharsNotIn(_REF_EXCLUDES) | pp.CharsNotIn(_REF_CLOSE_FIRST, exact=1)
ref_content = pp.Combine(pp.OneOrMore(ref_not_open + ref_not_close + ref_text))
- ref_string = pp.MatchFirst([double_escape, ref_escape_open, ref_escape_close, ref_content]).setParseAction(_tag_with(STR))
+ ref_string = pp.MatchFirst([double_escape, ref_escape_open, ref_escape_close, ref_content]).setParseAction(_tag_with(tags.STR))
ref_item = pp.Forward()
ref_items = pp.OneOrMore(ref_item)
- reference = (ref_open + pp.Group(ref_items) + ref_close).setParseAction(_tag_with(REF))
+ reference = (ref_open + pp.Group(ref_items) + ref_close).setParseAction(_tag_with(tags.REF))
ref_item << (reference | ref_string)
inv_open = pp.Literal(_INV_OPEN).suppress()
@@ -133,29 +146,33 @@
inv_escape_close = pp.Literal(_INV_ESCAPE_CLOSE).setParseAction(pp.replaceWith(_INV_CLOSE))
inv_text = pp.CharsNotIn(_INV_CLOSE_FIRST)
inv_content = pp.Combine(pp.OneOrMore(inv_not_close + inv_text))
- inv_string = pp.MatchFirst([double_escape, inv_escape_open, inv_escape_close, inv_content]).setParseAction(_tag_with(STR))
+ inv_string = pp.MatchFirst(
+ [double_escape, inv_escape_open, inv_escape_close, inv_content]
+ ).setParseAction(_tag_with(tags.STR))
inv_items = pp.OneOrMore(inv_string)
- export = (inv_open + pp.Group(inv_items) + inv_close).setParseAction(_tag_with(INV))
+ export = (inv_open + pp.Group(inv_items) + inv_close).setParseAction(_tag_with(tags.INV))
text = pp.CharsNotIn(_EXCLUDES) | pp.CharsNotIn('', exact=1)
content = pp.Combine(pp.OneOrMore(ref_not_open + inv_not_open + text))
- string = pp.MatchFirst([double_escape, ref_escape_open, inv_escape_open, content]).setParseAction(_tag_with(STR))
+ string = pp.MatchFirst(
+ [double_escape, ref_escape_open, inv_escape_open, content]
+ ).setParseAction(_tag_with(tags.STR))
item = reference | export | string
- line = pp.OneOrMore(item) + pp.StringEnd()
- return line
+ line = pp.OneOrMore(item) + s_end
+ return line.leaveWhitespace()
-def get_simple_ref_parser(escape_character, reference_sentinels, export_sentinels):
- _ESCAPE = escape_character
- _REF_OPEN = reference_sentinels[0]
- _REF_CLOSE = reference_sentinels[1]
- _INV_OPEN = export_sentinels[0]
- _INV_CLOSE = export_sentinels[1]
- _EXCLUDES = _ESCAPE + _REF_OPEN + _REF_CLOSE + _INV_OPEN + _INV_CLOSE
- string = pp.CharsNotIn(_EXCLUDES).setParseAction(_tag_with(STR))
- ref_open = pp.Literal(_REF_OPEN).suppress()
- ref_close = pp.Literal(_REF_CLOSE).suppress()
- reference = (ref_open + pp.Group(string) + ref_close).setParseAction(_tag_with(REF))
- line = pp.StringStart() + pp.Optional(string) + reference + pp.Optional(string) + pp.StringEnd()
- return line
+def get_simple_ref_parser(settings):
+
+ ESCAPE = settings.escape_character
+ REF_OPEN, REF_CLOSE = settings.reference_sentinels
+ INV_OPEN, INV_CLOSE = settings.export_sentinels
+ EXCLUDES = ESCAPE + REF_OPEN + REF_CLOSE + INV_OPEN + INV_CLOSE
+
+ string = pp.CharsNotIn(EXCLUDES).setParseAction(_tag_with(tags.STR))
+ ref_open = pp.Literal(REF_OPEN).suppress()
+ ref_close = pp.Literal(REF_CLOSE).suppress()
+ reference = (ref_open + pp.Group(string) + ref_close).setParseAction(_tag_with(tags.REF))
+ line = pp.StringStart() + pp.Optional(string) + reference + pp.Optional(string) + s_end
+ return line.leaveWhitespace()
diff --git a/reclass/values/refitem.py b/reclass/values/refitem.py
index df713e1..64bf450 100644
--- a/reclass/values/refitem.py
+++ b/reclass/values/refitem.py
@@ -16,12 +16,14 @@
def assembleRefs(self, context={}):
super(RefItem, self).assembleRefs(context)
try:
- strings = [str(i.render(context, None)) for i in self.contents]
- value = "".join(strings)
- self._refs.append(value)
+ self._refs.append(self._flatten_contents(context))
except ResolveError as e:
self.allRefs = False
+ def _flatten_contents(self, context, inventory=None):
+ result = [str(i.render(context, inventory)) for i in self.contents]
+ return "".join(result)
+
def _resolve(self, ref, context):
path = DictPath(self._settings.delimiter, ref)
try:
@@ -30,11 +32,10 @@
raise ResolveError(ref)
def render(self, context, inventory):
- if len(self.contents) == 1:
- return self._resolve(self.contents[0].render(context, inventory),
- context)
- strings = [str(i.render(context, inventory)) for i in self.contents]
- return self._resolve("".join(strings), context)
+ #strings = [str(i.render(context, inventory)) for i in self.contents]
+ #return self._resolve("".join(strings), context)
+ return self._resolve(self._flatten_contents(context, inventory),
+ context)
def __str__(self):
strings = [str(i) for i in self.contents]
diff --git a/reclass/values/tests/test_compitem.py b/reclass/values/tests/test_compitem.py
index 71a6f0e..c3ee690 100644
--- a/reclass/values/tests/test_compitem.py
+++ b/reclass/values/tests/test_compitem.py
@@ -71,6 +71,14 @@
self.assertTrue(composite.has_references)
self.assertEquals(composite.get_references(), expected_refs)
+ def test_string_representation(self):
+ composite = CompItem(Value(1, SETTINGS, ''), SETTINGS)
+ expected = '1'
+
+ result = str(composite)
+
+ self.assertEquals(result, expected)
+
def test_render_single_item(self):
val1 = Value('${foo}', SETTINGS, '')
@@ -106,20 +114,6 @@
self.assertEquals(result, composite2)
- def test_merge_over_merge_list_not_allowed(self):
- val1 = Value(None, SETTINGS, '')
- listitem = ListItem([1], SETTINGS)
- composite = CompItem([val1], SETTINGS)
-
- self.assertRaises(RuntimeError, composite.merge_over, listitem)
-
- def test_merge_dict_dict_not_allowed(self):
- val1 = Value(None, SETTINGS, '')
- dictitem = DictItem({'foo': 'bar'}, SETTINGS)
- composite = CompItem([val1], SETTINGS)
-
- self.assertRaises(RuntimeError, composite.merge_over, dictitem)
-
def test_merge_other_types_not_allowed(self):
other = type('Other', (object,), {'type': 34})
val1 = Value(None, SETTINGS, '')
diff --git a/reclass/values/tests/test_item.py b/reclass/values/tests/test_item.py
new file mode 100644
index 0000000..4b91f6e
--- /dev/null
+++ b/reclass/values/tests/test_item.py
@@ -0,0 +1,48 @@
+from reclass.settings import Settings
+from reclass.values.value import Value
+from reclass.values.compitem import CompItem
+from reclass.values.scaitem import ScaItem
+from reclass.values.valuelist import ValueList
+from reclass.values.listitem import ListItem
+from reclass.values.dictitem import DictItem
+from reclass.values.item import ContainerItem
+from reclass.values.item import ItemWithReferences
+import unittest
+from mock import MagicMock
+
+SETTINGS = Settings()
+
+
+class TestItemWithReferences(unittest.TestCase):
+
+ def test_assembleRef_allrefs(self):
+ phonyitem = MagicMock()
+ phonyitem.has_references = True
+ phonyitem.get_references = lambda *x: [1]
+
+ iwr = ItemWithReferences([phonyitem], {})
+
+ self.assertEquals(iwr.get_references(), [1])
+ self.assertTrue(iwr.allRefs)
+
+ def test_assembleRef_partial(self):
+ phonyitem = MagicMock()
+ phonyitem.has_references = True
+ phonyitem.allRefs = False
+ phonyitem.get_references = lambda *x: [1]
+
+ iwr = ItemWithReferences([phonyitem], {})
+
+ self.assertEquals(iwr.get_references(), [1])
+ self.assertFalse(iwr.allRefs)
+
+
+class TestContainerItem(unittest.TestCase):
+
+ def test_render(self):
+ container = ContainerItem('foo', SETTINGS)
+
+ self.assertEquals(container.render(None, None), 'foo')
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/reclass/values/tests/test_listitem.py b/reclass/values/tests/test_listitem.py
new file mode 100644
index 0000000..618b779
--- /dev/null
+++ b/reclass/values/tests/test_listitem.py
@@ -0,0 +1,31 @@
+from reclass.settings import Settings
+from reclass.values.value import Value
+from reclass.values.compitem import CompItem
+from reclass.values.scaitem import ScaItem
+from reclass.values.valuelist import ValueList
+from reclass.values.listitem import ListItem
+from reclass.values.dictitem import DictItem
+import unittest
+
+SETTINGS = Settings()
+
+class TestListItem(unittest.TestCase):
+
+ def test_merge_over_merge_list(self):
+ listitem1 = ListItem([1], SETTINGS)
+ listitem2 = ListItem([2], SETTINGS)
+ expected = ListItem([1, 2], SETTINGS)
+
+ result = listitem2.merge_over(listitem1)
+
+ self.assertEquals(result.contents, expected.contents)
+
+ def test_merge_other_types_not_allowed(self):
+ other = type('Other', (object,), {'type': 34})
+ val1 = Value(None, SETTINGS, '')
+ listitem = ListItem(val1, SETTINGS)
+
+ self.assertRaises(RuntimeError, listitem.merge_over, other)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/reclass/values/tests/test_parser_functions.py b/reclass/values/tests/test_parser_functions.py
new file mode 100644
index 0000000..a660c76
--- /dev/null
+++ b/reclass/values/tests/test_parser_functions.py
@@ -0,0 +1,116 @@
+from reclass import settings
+from reclass.values import parser_funcs as pf
+import unittest
+import ddt
+
+
+SETTINGS = settings.Settings()
+
+# Test cases for parsers. Each test case is a two-tuple of input string and
+# expected output. NOTE: default values for sentinels are used here to avoid
+# cluttering up the code.
+test_pairs_simple = (
+ # Basic test cases.
+ ('${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ # Basic combinations.
+ ('bar${foo}', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ ('bar${foo}baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ ('${foo}baz', [(pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ # Whitespace preservation cases.
+ ('bar ${foo}', [(pf.tags.STR, 'bar '),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ ('bar ${foo baz}', [(pf.tags.STR, 'bar '),
+ (pf.tags.REF, [(pf.tags.STR, 'foo baz')])]),
+ ('bar${foo} baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz')]),
+ (' bar${foo} baz ', [(pf.tags.STR, ' bar'),
+ (pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz ')]),
+)
+
+# Simple parser test cases are also included in this test grouop.
+test_pairs_full = (
+ # Single elements sanity.
+ ('foo', [(pf.tags.STR, 'foo')]),
+ ('$foo', [(pf.tags.STR, '$foo')]),
+ ('{foo}', [(pf.tags.STR, '{foo}')]),
+ ('[foo]', [(pf.tags.STR, '[foo]')]),
+ ('$(foo)', [(pf.tags.STR, '$(foo)')]),
+ ('$[foo]', [(pf.tags.INV, [(pf.tags.STR, 'foo')])]),
+
+ # Escape sequences.
+ # NOTE: these sequences apparently are not working as expected.
+ #(r'\\\\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ #(r'\\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+ #(r'\${foo}', [(pf.tags.REF, [(pf.tags.STR, 'foo')])]),
+
+ # Basic combinations.
+ ('bar$[foo]', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')])]),
+ ('bar$[foo]baz', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+ ('$[foo]baz', [(pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, 'baz')]),
+
+ # Whitespace preservation in various positions.
+ (' foo ', [(pf.tags.STR, ' foo ')]),
+ ('foo bar', [(pf.tags.STR, 'foo bar')]),
+ ('bar $[foo baz]', [(pf.tags.STR, 'bar '),
+ (pf.tags.INV, [(pf.tags.STR, 'foo baz')])]),
+ ('bar$[foo] baz ', [(pf.tags.STR, 'bar'),
+ (pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.STR, ' baz ')]),
+
+ # Nested references and inventory items.
+ ('${foo}${bar}',[(pf.tags.REF, [(pf.tags.STR, 'foo')]),
+ (pf.tags.REF, [(pf.tags.STR, 'bar')])]),
+ ('${foo${bar}}',[(pf.tags.REF, [(pf.tags.STR, 'foo'),
+ (pf.tags.REF, [(pf.tags.STR, 'bar')])])]),
+ ('$[foo]$[bar]',[(pf.tags.INV, [(pf.tags.STR, 'foo')]),
+ (pf.tags.INV, [(pf.tags.STR, 'bar')])]),
+ # NOTE: the cases below do not work as expected, which is probably a bug.
+ # Any nesting in INV creates a string.
+ #('${$[foo]}', [(pf.tags.REF, [(pf.tags.INV, [(pf.tags.STR, 'foo')])])]),
+ #('$[${foo}]', [(pf.tags.INV, [(pf.tags.REF, [(pf.tags.STR, 'foo')])])]),
+ #('$[foo$[bar]]',[(pf.tags.INV, [(pf.tags.STR, 'foo'),
+ # (pf.tags.INV, [(pf.tags.STR, 'bar')])])]),
+
+) + test_pairs_simple
+
+
+@ddt.ddt
+class TestRefParser(unittest.TestCase):
+
+ @ddt.data(*test_pairs_full)
+ def test_standard_reference_parser(self, data):
+ instring, expected = data
+ parser = pf.get_ref_parser(SETTINGS)
+
+ result = pf.listify(parser.parseString(instring).asList())
+
+ self.assertEquals(expected, result)
+
+
+@ddt.ddt
+class TestSimpleRefParser(unittest.TestCase):
+
+ @ddt.data(*test_pairs_simple)
+ def test_standard_reference_parser(self, data):
+ # NOTE: simple reference parser can parse references only. It fails
+ # on inventory items.
+ instring, expected = data
+ parser = pf.get_simple_ref_parser(SETTINGS)
+
+ result = pf.listify(parser.parseString(instring).asList())
+
+ self.assertEquals(expected, result)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/reclass/values/tests/test_refitem.py b/reclass/values/tests/test_refitem.py
new file mode 100644
index 0000000..6581478
--- /dev/null
+++ b/reclass/values/tests/test_refitem.py
@@ -0,0 +1,57 @@
+from reclass import errors
+
+from reclass.settings import Settings
+from reclass.values.value import Value
+from reclass.values.compitem import CompItem
+from reclass.values.scaitem import ScaItem
+from reclass.values.valuelist import ValueList
+from reclass.values.listitem import ListItem
+from reclass.values.dictitem import DictItem
+from reclass.values.refitem import RefItem
+import unittest
+from mock import MagicMock
+
+SETTINGS = Settings()
+
+class TestRefItem(unittest.TestCase):
+
+ def test_assembleRefs_ok(self):
+ phonyitem = MagicMock()
+ phonyitem.render = lambda x, k: 'bar'
+ phonyitem.has_references = True
+ phonyitem.get_references = lambda *x: ['foo']
+
+ iwr = RefItem([phonyitem], {})
+
+ self.assertEquals(iwr.get_references(), ['foo', 'bar'])
+ self.assertTrue(iwr.allRefs)
+
+ def test_assembleRefs_failedrefs(self):
+ phonyitem = MagicMock()
+ phonyitem.render.side_effect = errors.ResolveError('foo')
+ phonyitem.has_references = True
+ phonyitem.get_references = lambda *x: ['foo']
+
+ iwr = RefItem([phonyitem], {})
+
+ self.assertEquals(iwr.get_references(), ['foo'])
+ self.assertFalse(iwr.allRefs)
+
+ def test__resolve_ok(self):
+ reference = RefItem('', Settings({'delimiter': ':'}))
+
+ result = reference._resolve('foo:bar', {'foo':{'bar': 1}})
+
+ self.assertEquals(result, 1)
+
+ def test__resolve_fails(self):
+ refitem = RefItem('', Settings({'delimiter': ':'}))
+ context = {'foo':{'bar': 1}}
+ reference = 'foo:baz'
+
+ self.assertRaises(errors.ResolveError, refitem._resolve, reference,
+ context)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/reclass/values/tests/test_scaitem.py b/reclass/values/tests/test_scaitem.py
new file mode 100644
index 0000000..b6d038d
--- /dev/null
+++ b/reclass/values/tests/test_scaitem.py
@@ -0,0 +1,38 @@
+from reclass.settings import Settings
+from reclass.values.value import Value
+from reclass.values.compitem import CompItem
+from reclass.values.scaitem import ScaItem
+from reclass.values.valuelist import ValueList
+from reclass.values.listitem import ListItem
+from reclass.values.dictitem import DictItem
+import unittest
+
+SETTINGS = Settings()
+
+class TestScaItem(unittest.TestCase):
+
+ def test_merge_over_merge_scalar(self):
+ scalar1 = ScaItem([1], SETTINGS)
+ scalar2 = ScaItem([2], SETTINGS)
+
+ result = scalar2.merge_over(scalar1)
+
+ self.assertEquals(result.contents, scalar2.contents)
+
+ def test_merge_over_merge_composite(self):
+ scalar1 = CompItem(Value(1, SETTINGS, ''), SETTINGS)
+ scalar2 = ScaItem([2], SETTINGS)
+
+ result = scalar2.merge_over(scalar1)
+
+ self.assertEquals(result.contents, scalar2.contents)
+
+ def test_merge_other_types_not_allowed(self):
+ other = type('Other', (object,), {'type': 34})
+ val1 = Value(None, SETTINGS, '')
+ scalar = ScaItem(val1, SETTINGS)
+
+ self.assertRaises(RuntimeError, scalar.merge_over, other)
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/reclass/values/value.py b/reclass/values/value.py
index affd944..451617e 100644
--- a/reclass/values/value.py
+++ b/reclass/values/value.py
@@ -22,15 +22,15 @@
def __init__(self, value, settings, uri, parse_string=True):
self._settings = settings
- self._uri = uri
+ self.uri = uri
self.overwrite = False
- self._constant = False
+ self.constant = False
if isinstance(value, string_types):
if parse_string:
try:
self._item = self._parser.parse(value, self._settings)
except InterpolationError as e:
- e.uri = self._uri
+ e.uri = self.uri
raise
else:
self._item = ScaItem(value, self._settings)
@@ -41,18 +41,6 @@
else:
self._item = ScaItem(value, self._settings)
- @property
- def uri(self):
- return self._uri
-
- @property
- def constant(self):
- return self._constant
-
- @constant.setter
- def constant(self, constant):
- self._constant = constant
-
def item_type(self):
return self._item.type
@@ -78,8 +66,7 @@
def needs_all_envs(self):
if self._item.has_inv_query:
return self._item.needs_all_envs
- else:
- return False
+ return False
def ignore_failed_render(self):
return self._item.ignore_failed_render
@@ -102,7 +89,7 @@
try:
return self._item.render(context, inventory)
except InterpolationError as e:
- e.uri = self._uri
+ e.uri = self.uri
raise
@property
diff --git a/reclass/values/valuelist.py b/reclass/values/valuelist.py
index a56395b..e8c3a0c 100644
--- a/reclass/values/valuelist.py
+++ b/reclass/values/valuelist.py
@@ -22,9 +22,9 @@
self.allRefs = True
self._values = [value]
self._inv_refs = []
- self._has_inv_query = False
+ self.has_inv_query = False
self.ignore_failed_render = False
- self._is_complex = False
+ self.is_complex = False
self._update()
@property
@@ -42,40 +42,32 @@
def _update(self):
self.assembleRefs()
self._check_for_inv_query()
- self._is_complex = False
+ self.is_complex = False
item_type = self._values[0].item_type()
for v in self._values:
if v.is_complex or v.constant or v.overwrite or v.item_type() != item_type:
- self._is_complex = True
+ self.is_complex = True
@property
def has_references(self):
return len(self._refs) > 0
- @property
- def has_inv_query(self):
- return self._has_inv_query
-
def get_inv_references(self):
return self._inv_refs
- @property
- def is_complex(self):
- return self._is_complex
-
def get_references(self):
return self._refs
def _check_for_inv_query(self):
- self._has_inv_query = False
+ self.has_inv_query = False
self.ignore_failed_render = True
for value in self._values:
if value.has_inv_query:
- self._inv_refs.extend(value.get_inv_references)
- self._has_inv_query = True
- if vale.ignore_failed_render() is False:
+ self._inv_refs.extend(value.get_inv_references())
+ self.has_inv_query = True
+ if value.ignore_failed_render() is False:
self.ignore_failed_render = False
- if self._has_inv_query is False:
+ if self.has_inv_query is False:
self.ignore_failed_render = False
def assembleRefs(self, context={}):
@@ -88,6 +80,13 @@
if value.allRefs is False:
self.allRefs = False
+ @property
+ def needs_all_envs(self):
+ for value in self._values:
+ if value.needs_all_envs:
+ return True
+ return False
+
def merge(self):
output = None
for n, value in enumerate(self._values):
diff --git a/reclass/version.py b/reclass/version.py
index 6d7d7eb..5a40c2e 100644
--- a/reclass/version.py
+++ b/reclass/version.py
@@ -12,12 +12,14 @@
from __future__ import unicode_literals
RECLASS_NAME = 'reclass'
-DESCRIPTION = 'merge data by recursive descent down an ancestry hierarchy (forked extended version)'
-VERSION = '1.5.6'
+DESCRIPTION = ('merge data by recursive descent down an ancestry hierarchy '
+ '(forked extended version)')
+VERSION = '1.6.0'
AUTHOR = 'martin f. krafft / Andrew Pickford / salt-formulas community'
AUTHOR_EMAIL = 'salt-formulas@freelists.org'
MAINTAINER = 'salt-formulas community'
MAINTAINER_EMAIL = 'salt-formulas@freelists.org'
-COPYRIGHT = 'Copyright © 2007–14 martin f. krafft, extensions © 2017 Andrew Pickford, extensions © salt-formulas community'
+COPYRIGHT = ('Copyright © 2007–14 martin f. krafft, extensions © 2017 Andrew'
+ ' Pickford, extensions © salt-formulas community')
LICENCE = 'Artistic Licence 2.0'
URL = 'https://github.com/salt-formulas/reclass'
diff --git a/requirements.txt b/requirements.txt
index 5b3aadd..5f6aed1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,3 +2,4 @@
pyyaml
six
enum34
+ddt
diff --git a/setup.py b/setup.py
index 884be88..ab23207 100644
--- a/setup.py
+++ b/setup.py
@@ -42,7 +42,7 @@
url = URL,
packages = find_packages(exclude=['*tests']), #FIXME validate this
entry_points = { 'console_scripts': console_scripts },
- install_requires = ['pyparsing', 'pyyaml', 'six', 'enum34'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
+ install_requires = ['pyparsing', 'pyyaml', 'six', 'enum34', 'ddt'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
classifiers=[
'Development Status :: 4 - Beta',
diff --git a/test/model/extensions/classes/defaults.yml b/test/model/extensions/classes/defaults.yml
new file mode 100644
index 0000000..5d17c2b
--- /dev/null
+++ b/test/model/extensions/classes/defaults.yml
@@ -0,0 +1,4 @@
+
+parameters:
+ config:
+ defaults: True
diff --git a/test/model/extensions/classes/relative/init.yml b/test/model/extensions/classes/relative/init.yml
new file mode 100644
index 0000000..117e4fa
--- /dev/null
+++ b/test/model/extensions/classes/relative/init.yml
@@ -0,0 +1,3 @@
+
+classes:
+ - .nested
diff --git a/test/model/extensions/classes/relative/nested/common.yml b/test/model/extensions/classes/relative/nested/common.yml
new file mode 100644
index 0000000..28cc0b2
--- /dev/null
+++ b/test/model/extensions/classes/relative/nested/common.yml
@@ -0,0 +1,5 @@
+
+parameters:
+ nested:
+ deep:
+ common: to be overriden
diff --git a/test/model/extensions/classes/relative/nested/deep/common.yml b/test/model/extensions/classes/relative/nested/deep/common.yml
new file mode 100644
index 0000000..b77a24c
--- /dev/null
+++ b/test/model/extensions/classes/relative/nested/deep/common.yml
@@ -0,0 +1,5 @@
+
+parameters:
+ nested:
+ deep:
+ common: False
diff --git a/test/model/extensions/classes/relative/nested/deep/init.yml b/test/model/extensions/classes/relative/nested/deep/init.yml
new file mode 100644
index 0000000..cd12d10
--- /dev/null
+++ b/test/model/extensions/classes/relative/nested/deep/init.yml
@@ -0,0 +1,9 @@
+
+classes:
+ - .common
+
+parameters:
+ nested:
+ deep:
+ init: True
+ common: True
diff --git a/test/model/extensions/classes/relative/nested/dive/session.yml b/test/model/extensions/classes/relative/nested/dive/session.yml
new file mode 100644
index 0000000..9abd1ee
--- /dev/null
+++ b/test/model/extensions/classes/relative/nested/dive/session.yml
@@ -0,0 +1,5 @@
+
+parameters:
+ nested:
+ deep:
+ session: True
diff --git a/test/model/extensions/classes/relative/nested/init.yml b/test/model/extensions/classes/relative/nested/init.yml
new file mode 100644
index 0000000..9f02383
--- /dev/null
+++ b/test/model/extensions/classes/relative/nested/init.yml
@@ -0,0 +1,10 @@
+
+classes:
+ - .common
+ - .deep
+ - .dive.session
+
+parameters:
+ nested:
+ deep:
+ init: True
diff --git a/test/model/extensions/classes/second.yml b/test/model/extensions/classes/second.yml
index a9babd3..929d746 100644
--- a/test/model/extensions/classes/second.yml
+++ b/test/model/extensions/classes/second.yml
@@ -1,5 +1,6 @@
classes:
- first
+- relative
parameters:
will:
diff --git a/test/model/extensions/classes/third.yml b/test/model/extensions/classes/third.yml
index 20a937c..a5157cf 100644
--- a/test/model/extensions/classes/third.yml
+++ b/test/model/extensions/classes/third.yml
@@ -1,6 +1,7 @@
classes:
- missing.class
- second
+- .defaults
parameters:
_param:
diff --git a/test/model/extensions/nodes/reclass.yml b/test/model/extensions/nodes/reclass.yml
index 94b7519..5d5b3ec 100644
--- a/test/model/extensions/nodes/reclass.yml
+++ b/test/model/extensions/nodes/reclass.yml
@@ -1,3 +1,3 @@
classes:
-- third
+- .third