Merge branch 'develop' into new-global-classes
diff --git a/.gitignore b/.gitignore
index 5503138..162c415 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,3 +6,4 @@
/build
/dist
/.coverage
+.kitchen
diff --git a/.kitchen-verify.sh b/.kitchen-verify.sh
new file mode 100755
index 0000000..fb3adde
--- /dev/null
+++ b/.kitchen-verify.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+#set -x
+
+# setup
+source /*.env
+INVENTORY_BASE_URI=/tmp/kitchen/test/model/$MODEL
+RECLASS=/tmp/kitchen
+
+# prereq
+python -m ensurepip --default-pip
+pip install pipenv
+
+# env
+cd $RECLASS
+pipenv --venv || pipenv install --python ${PYVER}
+test -e /etc/reclsss || mkdir /etc/reclass
+cp -avf $INVENTORY_BASE_URI/reclass-config* /etc/reclass
+
+# verify
+for n in $(ls $INVENTORY_BASE_URI/nodes/*|sort); do
+ pipenv run python${PYVER} ./reclass.py --inventory-base-uri=$INVENTORY_BASE_URI --nodeinfo $(basename $n .yml)
+done
diff --git a/.kitchen.yml b/.kitchen.yml
new file mode 100644
index 0000000..45be629
--- /dev/null
+++ b/.kitchen.yml
@@ -0,0 +1,41 @@
+---
+driver:
+ name: docker
+ priviledged: false
+ use_sudo: false
+ volume:
+ - <%= ENV['PWD'] %>:/tmp/kitchen
+
+
+provisioner:
+ name: shell
+ script: .kitchen-verify.sh
+
+
+verifier:
+ name: inspec
+
+<%- pyver = ENV['PYTHON_VERSION'] || '2.7' %>
+
+platforms:
+ <% `find test/model -maxdepth 1 -mindepth 1 -type d |sort -u`.split().each do |model| %>
+ <% model=model.split('/')[2] %>
+ - name: <%= model %>
+ driver_config:
+ image: python:<%= pyver %>
+ platform: ubuntu
+ hostname: reclass
+ provision_command:
+ #FIXME, setup reclass env (prereq, configs, upload models)
+ #- apt-get install -y rsync
+ - echo "
+ export LC_ALL=C.UTF-8;\n
+ export LANG=C.UTF-8;\n
+ export PYVER=<%= pyver %>;\n
+ export MODEL=<%= model %>;\n
+ " > /kitchen.env
+ <% end %>
+
+suites:
+ - name: model
+
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..b060639
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,116 @@
+sudo: required
+language: python
+dist: trusty
+cache: pip
+python:
+- '2.7'
+- '3.6'
+service:
+- docker
+
+#apt:
+ #update: true
+
+#stages:
+#- name: test
+#- name: coverage
+#- name: models
+#- name: build
+# if: fork = false
+#- name: publish
+# if: tag =~ ^v.* and fork = false and branch = 'master'
+
+env:
+ global:
+ - PACKAGENAME="reclass"
+
+install: &pyinst
+- pip install pyparsing
+- pip install PyYAML
+# To test example models with kitchen:
+- |
+ test -e Gemfile || cat <<EOF > Gemfile
+ source 'https://rubygems.org'
+ gem 'rake'
+ gem 'test-kitchen'
+ gem 'kitchen-docker'
+ gem 'kitchen-inspec'
+ gem 'inspec'
+- bundle install
+
+script:
+- python setup.py install
+- find . reclass -name 'test_*.py' | sort | xargs -n1 -i% bash -c "echo %; python %"
+# To test example models with kitchen:
+- export PYTHON_VERSION=$TRAVIS_PYTHON_VERSION
+- kitchen list
+- kitchen test
+
+# NOTE: travis stage builds, below saved for future reference
+#jobs:
+# include:
+# - stage: test
+# script: &unittest
+# - python setup.py install
+# - find . reclass -name 'test_*.py' | sort | xargs -n1 -i% bash -c "echo %; python %"
+#
+# - stage: coverage
+# install: *pyinst
+# script:
+# - python3 -m pytest --cov=. --cov-report=term-missing:skip-covered
+# - coverage xml
+# #- coveralls
+# #- |
+# #[ ! -z "${CODACY_PROJECT_TOKEN}" ] && python-codacy-coverage -r coverage.xml || echo "Codacy coverage NOT exported"
+#
+# - stage: lint
+# script:
+# - python3 -m flake8
+#
+# - stage: models
+# install: &kitchen
+# - pip install PyYAML
+# - pip install virtualenv
+# - |
+# test -e Gemfile || cat <<EOF > Gemfile
+# source 'https://rubygems.org'
+# gem 'rake'
+# gem 'test-kitchen'
+# gem 'kitchen-docker'
+# gem 'kitchen-inspec'
+# gem 'inspec'
+# - bundle install
+# script:
+# - export PYTHON_VERSION=$TRAVIS_PYTHON_VERSION
+# - kitchen list
+# #FIXME- kitchen test
+#
+# - stage: build
+# install: *pyinst
+# script: []
+#
+# - stage: publish
+# install:
+# - "/bin/true"
+# script:
+# - "/bin/true"
+# deploy:
+# provider: pypi
+# user: epcim
+# password:
+# secure: TBD
+# on:
+# tags: true
+# repo: salt-formulas/reclass
+# branch: master
+# #FIXME, $TRAVIS_PYTHON_VERSION == '2.7'
+
+notifications:
+ webhooks:
+ on_success: change # options: [always|never|change] default: always
+ on_failure: never
+ on_start: never
+ on_cancel: never
+ on_error: never
+ email: true
+
diff --git a/Pipfile b/Pipfile
new file mode 100644
index 0000000..525e7cc
--- /dev/null
+++ b/Pipfile
@@ -0,0 +1,17 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[dev-packages]
+
+[packages]
+pyparsing = "*"
+PyYAML = "*"
+six = "*"
+pyyaml = "*"
+# FIXME, issues with compile phase
+#"pygit2" = "*"
+
+[requires]
+python_version = "2.7"
diff --git a/README-extentions.rst b/README-extentions.rst
index b443d01..a18a37a 100644
--- a/README-extentions.rst
+++ b/README-extentions.rst
@@ -208,7 +208,6 @@
group_errors: True
-
Use global classes
------------------
@@ -231,6 +230,82 @@
global_class_regexp = ['.*global']
+Use references in class names
+-----------------------------
+
+Allows to use references in the class names.
+
+References pointed to in class names cannot themselves reference another key, they should be simple strings.
+
+To avoid pitfalls do not over-engineer your class references. They should be used only for core conditions and only for them.
+A short example: `- system.wrodpress.db.${_class:database_backend}`.
+
+Best practices:
+- use references in class names always load your global class specification prior the reference is used.
+- structure your class references under parameters under one key (for example `_class`).
+- use class references as a kind of "context" or "global" available options you always know what they are set.
+
+Class referencing for existing reclass users. Frequently when constructing your models you had to load or not load some
+classes based on your setup. In most cases this lead to fork of a model or introducing kind of template generator (like cookiecutter) to
+create a model based on the base "context" or "global" variables. Class referencing is a simple way how to avoid
+"pre-processors" like this and if/else conditions around class section.
+
+
+Assuming following class setup:
+
+* node is loading `third.yml` class only
+
+
+Classes:
+
+.. code-block:: yaml
+ #/etc/reclass/classes/global.yml
+ parameters:
+ _class:
+ env:
+ override: 'env.dev'
+ lab:
+ name: default
+
+ #/etc/reclass/classes/lab/env/dev.yml
+ parameters:
+ lab:
+ name: dev
+
+ #/etc/reclass/classes/second.yml
+ classes:
+ - global
+ - lab.${_class:env:override}
+
+ #/etc/reclass/classes/third.yml
+ classes:
+ - global
+ - second
+
+
+Reclass --nodeinfo then returns:
+
+.. code-block:: yaml
+
+ ...
+ ...
+ applications: []
+ environment: base
+ exports: {}
+ classes:
+ - global
+ - lab.${_class:env:override}
+ - second
+ parameters:
+ _class:
+ env:
+ override: env.dev
+ lab:
+ name: dev
+ ...
+ ...
+
+
Inventory Queries
-----------------
diff --git a/README.rst b/README.rst
index 99de5f6..b865e4f 100644
--- a/README.rst
+++ b/README.rst
@@ -24,3 +24,26 @@
Documentation covering the original version is in the doc directory.
See the README-extensions.rst file for documentation on the extentions.
+
+
+
+Reclass related projects/tools
+==============================
+
+Queries:
+
+* yg, yaml grep with 'jq' syntax - https://gist.github.com/epcim/f1c5b748fa7c942de50677aef04f29f8, (https://asciinema.org/a/84173)
+* reclass-graph - https://github.com/tomkukral/reclass-graph
+
+Introspection, manupulation:
+
+* reclass-tools, for manipulating reclass models - https://github.com/dis-xcom/reclass_tools
+
+YAML merge tools:
+
+* spruce, general purpose YAML & JSON merging tool - https://github.com/geofffranks/spruce
+
+Other:
+
+* saltclass, new pillar/master_tops module for salt with the behaviour of reclass - https://github.com/saltstack/salt/pull/42349
+
diff --git a/reclass.py b/reclass.py
index a0d8eb8..9c71a10 100755
--- a/reclass.py
+++ b/reclass.py
@@ -6,6 +6,10 @@
# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import reclass.cli
reclass.cli.main()
diff --git a/reclass/__init__.py b/reclass/__init__.py
index a79c8e1..3739b5e 100644
--- a/reclass/__init__.py
+++ b/reclass/__init__.py
@@ -6,11 +6,14 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
-from reclass.output import OutputLoader
-from reclass.storage.loader import StorageBackendLoader
-from reclass.storage.memcache_proxy import MemcacheProxy
-
+from .output import OutputLoader
+from .storage.loader import StorageBackendLoader
+from .storage.memcache_proxy import MemcacheProxy
def get_storage(storage_type, nodes_uri, classes_uri, **kwargs):
storage_class = StorageBackendLoader(storage_type).load()
diff --git a/reclass/adapters/__init__.py b/reclass/adapters/__init__.py
index 8a17572..06edb64 100755
--- a/reclass/adapters/__init__.py
+++ b/reclass/adapters/__init__.py
@@ -5,4 +5,8 @@
#
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
-#
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/adapters/ansible.py b/reclass/adapters/ansible.py
index f6e9af3..abf7df2 100755
--- a/reclass/adapters/ansible.py
+++ b/reclass/adapters/ansible.py
@@ -14,6 +14,11 @@
# The ansible adapter has received little testing and may not work at all now.
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
import os, sys, posix, optparse
from six import iteritems
@@ -89,9 +94,9 @@
data = groups
- print output(data, options.output, options.pretty_print, options.no_refs)
+ print(output(data, options.output, options.pretty_print, options.no_refs))
- except ReclassException, e:
+ except ReclassException as e:
e.exit_with_message(sys.stderr)
sys.exit(posix.EX_OK)
diff --git a/reclass/adapters/salt.py b/reclass/adapters/salt.py
index 31179ff..ce4e792 100755
--- a/reclass/adapters/salt.py
+++ b/reclass/adapters/salt.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import os, sys, posix
@@ -122,9 +126,9 @@
class_mappings=class_mappings,
**defaults)
- print output(data, options.output, options.pretty_print, options.no_refs)
+ print(output(data, options.output, options.pretty_print, options.no_refs))
- except ReclassException, e:
+ except ReclassException as e:
e.exit_with_message(sys.stderr)
sys.exit(posix.EX_OK)
diff --git a/reclass/cli.py b/reclass/cli.py
index d1b22b8..44694c5 100644
--- a/reclass/cli.py
+++ b/reclass/cli.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import sys, os, posix
@@ -38,9 +42,9 @@
else:
data = reclass.inventory()
- print output(data, options.output, options.pretty_print, options.no_refs)
+ print(output(data, options.output, options.pretty_print, options.no_refs))
- except ReclassException, e:
+ except ReclassException as e:
e.exit_with_message(sys.stderr)
sys.exit(posix.EX_OK)
diff --git a/reclass/config.py b/reclass/config.py
index e9bb43b..1a6ba81 100644
--- a/reclass/config.py
+++ b/reclass/config.py
@@ -6,13 +6,17 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import yaml, os, optparse, posix, sys
-import errors
-from defaults import *
-from constants import MODE_NODEINFO, MODE_INVENTORY
-from reclass import get_path_mangler
+from . import errors, get_path_mangler
+from .defaults import *
+from .constants import MODE_NODEINFO, MODE_INVENTORY
+
def make_db_options_group(parser, defaults={}):
ret = optparse.OptionGroup(parser, 'Database options',
@@ -171,7 +175,7 @@
def vvv(msg):
- #print >>sys.stderr, msg
+ #print(msg, file=sys.stderr)
pass
@@ -180,8 +184,8 @@
for d in dirs:
f = os.path.join(d, filename)
if os.access(f, os.R_OK):
- vvv('Using config file: {0}'.format(f))
- return yaml.safe_load(file(f))
+ vvv('Using config file: {0}'.format(str(f)))
+ return yaml.safe_load(open(f))
elif os.path.isfile(f):
raise PermissionsError('cannot read %s' % f)
return {}
diff --git a/reclass/constants.py b/reclass/constants.py
index f69fa8c..58f7769 100644
--- a/reclass/constants.py
+++ b/reclass/constants.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
class _Constant(object):
diff --git a/reclass/core.py b/reclass/core.py
index 3dcacac..a90b9be 100644
--- a/reclass/core.py
+++ b/reclass/core.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import copy
import time
@@ -21,16 +25,14 @@
from reclass.settings import Settings
from reclass.output.yaml_outputter import ExplicitDumper
from reclass.datatypes import Entity, Classes, Parameters, Exports
-from reclass.errors import MappingFormatError, ClassNotFound, InvQueryClassNotFound, InvQueryError, InterpolationError
+from reclass.errors import MappingFormatError, ClassNameResolveError, ClassNotFound, InvQueryClassNameResolveError, InvQueryClassNotFound, InvQueryError, InterpolationError, ResolveError
from reclass.values.parser import Parser
-try:
- basestring
-except NameError:
- basestring = str
class Core(object):
+ _parser = Parser()
+
def __init__(self, storage, class_mappings, settings, input_data=None):
self._storage = storage
self._class_mappings = class_mappings
@@ -99,7 +101,7 @@
p = Parameters(self._input_data, self._settings)
return Entity(self._settings, parameters=p, name='input data')
- def _recurse_entity(self, entity, merge_base=None, seen=None, nodename=None, environment=None):
+ def _recurse_entity(self, entity, merge_base=None, context=None, seen=None, nodename=None, environment=None):
# values/parser in order to interpolate references in classes
_parser = Parser()
@@ -113,9 +115,22 @@
if merge_base is None:
merge_base = Entity(self._settings, name='empty (@{0})'.format(nodename))
+ if context is None:
+ context = Entity(self._settings, name='empty (@{0})'.format(nodename))
+
for klass in entity.classes.as_list():
- if merge_base is not None:
- klass=str(_parser.parse(klass, self._settings).render(merge_base.parameters.as_dict(), {}))
+
+ #if merge_base is not None:
+ # klass=str(_parser.parse(klass, self._settings).render(merge_base.parameters.as_dict(), {}))
+
+ if klass.count('$') > 0:
+ try:
+ klass = str(self._parser.parse(klass, self._settings).render(merge_base.parameters.as_dict(), {}))
+ except ResolveError as e:
+ try:
+ klass = str(self._parser.parse(klass, self._settings).render(context.parameters.as_dict(), {}))
+ except ResolveError as e:
+ raise ClassNameResolveError(klass, nodename, entity.uri)
# class not seen or class on a list of global classes (always (re)loaded)
if klass not in seen or self._gcl_r.match(klass):
try:
@@ -125,13 +140,13 @@
if self._cnf_r.match(klass):
if self._settings.ignore_class_notfound_warning:
# TODO, add logging handler
- print >>sys.stderr, "[WARNING] Reclass class not found: '%s'. Skipped!" % klass
+ print("[WARNING] Reclass class not found: '%s'. Skipped!" % klass, file=sys.stderr)
continue
e.nodename = nodename
e.uri = entity.uri
raise
- descent = self._recurse_entity(class_entity, seen=seen,
+ descent = self._recurse_entity(class_entity, context=merge_base, seen=seen,
nodename=nodename, environment=environment)
# on every iteration, we merge the result of the recursive
# descent into what we have so far…
@@ -146,7 +161,7 @@
def _get_automatic_parameters(self, nodename, environment):
if self._settings.automatic_parameters:
- return Parameters({ '_reclass_': { 'name': { 'full': nodename, 'short': str.split(nodename, '.')[0] },
+ return Parameters({ '_reclass_': { 'name': { 'full': nodename, 'short': nodename.split('.')[0] },
'environment': environment } }, self._settings, '__auto__')
else:
return Parameters({}, self._settings, '')
@@ -169,6 +184,8 @@
node = self._node_entity(nodename)
except ClassNotFound as e:
raise InvQueryClassNotFound(e)
+ except ClassNameResolveError as e:
+ raise InvQueryClassNameResolveError(e)
if queries is None:
try:
node.interpolate_exports()
@@ -196,8 +213,8 @@
seen = {}
merge_base = self._recurse_entity(base_entity, seen=seen, nodename=nodename,
environment=node_entity.environment)
- return self._recurse_entity(node_entity, merge_base, seen=seen, nodename=nodename,
- environment=node_entity.environment)
+ return self._recurse_entity(node_entity, merge_base=merge_base, context=merge_base, seen=seen,
+ nodename=nodename, environment=node_entity.environment)
def _nodeinfo(self, nodename, inventory):
try:
diff --git a/reclass/datatypes/__init__.py b/reclass/datatypes/__init__.py
index 48c4a8b..78c110b 100644
--- a/reclass/datatypes/__init__.py
+++ b/reclass/datatypes/__init__.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from .applications import Applications
from .classes import Classes
from .entity import Entity
diff --git a/reclass/datatypes/applications.py b/reclass/datatypes/applications.py
index 3c7afce..8c6ed15 100644
--- a/reclass/datatypes/applications.py
+++ b/reclass/datatypes/applications.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from .classes import Classes
@@ -61,4 +66,4 @@
contents = self._items + \
['%s%s' % (self._negation_prefix, i) for i in self._negations]
return "%s(%r, %r)" % (self.__class__.__name__, contents,
- self._negation_prefix)
+ str(self._negation_prefix))
diff --git a/reclass/datatypes/classes.py b/reclass/datatypes/classes.py
index 090ed70..33d9b93 100644
--- a/reclass/datatypes/classes.py
+++ b/reclass/datatypes/classes.py
@@ -6,6 +6,15 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+#try:
+# from types import StringTypes
+#except ImportError:
+# StringTypes = (str, )
import six
import os
@@ -52,6 +61,7 @@
self.append_if_new(i)
def _assert_is_string(self, item):
+ #if not isinstance(item, StringTypes):
if not isinstance(item, six.string_types):
raise TypeError('%s instances can only contain strings, '\
'not %s' % (self.__class__.__name__, type(item)))
diff --git a/reclass/datatypes/entity.py b/reclass/datatypes/entity.py
index b43ac72..38360b5 100644
--- a/reclass/datatypes/entity.py
+++ b/reclass/datatypes/entity.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from .classes import Classes
from .applications import Applications
from .exports import Exports
diff --git a/reclass/datatypes/exports.py b/reclass/datatypes/exports.py
index 971befa..7f21295 100644
--- a/reclass/datatypes/exports.py
+++ b/reclass/datatypes/exports.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass (http://github.com/madduck/reclass)
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import copy
diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py
index f9f116c..bedad88 100644
--- a/reclass/datatypes/parameters.py
+++ b/reclass/datatypes/parameters.py
@@ -7,6 +7,16 @@
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+#try:
+# from types import StringTypes
+#except ImportError:
+# StringTypes = (str, )
+
import copy
import sys
import types
@@ -44,7 +54,7 @@
functionality and does not try to be a really mapping object.
'''
- def __init__(self, mapping, settings, uri, merge_initialise = True):
+ def __init__(self, mapping, settings, uri, parse_strings=True):
self._settings = settings
self._base = {}
self._uri = uri
@@ -54,14 +64,12 @@
self._resolve_errors = ResolveErrorList()
self._needs_all_envs = False
self._keep_overrides = False
+ self._parse_strings = parse_strings
if mapping is not None:
- if merge_initialise:
- # we initialise by merging
- self._keep_overrides = True
- self.merge(mapping)
- self._keep_overrides = False
- else:
- self._base = copy.deepcopy(mapping)
+ # we initialise by merging
+ self._keep_overrides = True
+ self.merge(mapping)
+ self._keep_overrides = False
#delimiter = property(lambda self: self._delimiter)
@@ -103,7 +111,7 @@
return value
else:
try:
- return Value(value, self._settings, self._uri)
+ return Value(value, self._settings, self._uri, parse_string=self._parse_strings)
except InterpolationError as e:
e.context = str(path)
raise
@@ -132,7 +140,8 @@
elif isinstance(new, ValueList):
values.extend(new)
else:
- values.append(Value(new, self._settings, self._uri))
+ values.append(Value(new, self._settings, self._uri, parse_string=self._parse_strings))
+
return values
def _merge_dict(self, cur, new, path):
@@ -157,6 +166,9 @@
ret = cur
for (key, newvalue) in iteritems(new):
if key.startswith(self._settings.dict_key_override_prefix) and not self._keep_overrides:
+ if not isinstance(newvalue, Value):
+ newvalue = Value(newvalue, self._settings, self._uri, parse_string=self._parse_strings)
+ newvalue.overwrite = True
ret[key.lstrip(self._settings.dict_key_override_prefix)] = newvalue
else:
ret[key] = self._merge_recurse(ret.get(key), newvalue, path.new_subpath(key))
@@ -189,7 +201,7 @@
else:
return self._update_value(cur, new)
- def merge(self, other, wrap=True):
+ def merge(self, other):
"""Merge function (public edition).
Call _merge_recurse on self with either another Parameter object or a
@@ -205,15 +217,9 @@
self._unrendered = None
if isinstance(other, dict):
- if wrap:
- wrapped = self._wrap_dict(other, DictPath(self._settings.delimiter))
- else:
- wrapped = copy.deepcopy(other)
+ wrapped = self._wrap_dict(other, DictPath(self._settings.delimiter))
elif isinstance(other, self.__class__):
- if wrap:
- wrapped = self._wrap_dict(other._base, DictPath(self._settings.delimiter))
- else:
- wrapped = copy.deepcopy(other._base)
+ wrapped = self._wrap_dict(other._base, DictPath(self._settings.delimiter))
else:
raise TypeError('Cannot merge %s objects into %s' % (type(other),
self.__class__.__name__))
diff --git a/reclass/datatypes/tests/__init__.py b/reclass/datatypes/tests/__init__.py
index e69de29..9aaaf25 100644
--- a/reclass/datatypes/tests/__init__.py
+++ b/reclass/datatypes/tests/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/datatypes/tests/test_applications.py b/reclass/datatypes/tests/test_applications.py
index 6ae07cc..5c896f0 100644
--- a/reclass/datatypes/tests/test_applications.py
+++ b/reclass/datatypes/tests/test_applications.py
@@ -6,8 +6,14 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from reclass.datatypes import Applications, Classes
import unittest
+
try:
import unittest.mock as mock
except ImportError:
diff --git a/reclass/datatypes/tests/test_classes.py b/reclass/datatypes/tests/test_classes.py
index 33d179f..8d396e0 100644
--- a/reclass/datatypes/tests/test_classes.py
+++ b/reclass/datatypes/tests/test_classes.py
@@ -6,9 +6,15 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from reclass.datatypes import Classes
from reclass.datatypes.classes import INVALID_CHARACTERS_FOR_CLASSNAMES
import unittest
+
try:
import unittest.mock as mock
except ImportError:
diff --git a/reclass/datatypes/tests/test_entity.py b/reclass/datatypes/tests/test_entity.py
index f398d51..b09e904 100644
--- a/reclass/datatypes/tests/test_entity.py
+++ b/reclass/datatypes/tests/test_entity.py
@@ -6,11 +6,16 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.settings import Settings
from reclass.datatypes import Entity, Classes, Parameters, Applications, Exports
from reclass.errors import ResolveError
import unittest
+
try:
import unittest.mock as mock
except ImportError:
diff --git a/reclass/datatypes/tests/test_exports.py b/reclass/datatypes/tests/test_exports.py
index 33eccbe..6a6dcde 100644
--- a/reclass/datatypes/tests/test_exports.py
+++ b/reclass/datatypes/tests/test_exports.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass (http://github.com/madduck/reclass)
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.settings import Settings
from reclass.datatypes import Exports, Parameters
diff --git a/reclass/datatypes/tests/test_parameters.py b/reclass/datatypes/tests/test_parameters.py
index b5dc243..fb4a11b 100644
--- a/reclass/datatypes/tests/test_parameters.py
+++ b/reclass/datatypes/tests/test_parameters.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import copy
@@ -15,6 +19,7 @@
from reclass.datatypes import Parameters
from reclass.errors import InfiniteRecursionError, InterpolationError, ResolveError, ResolveErrorList
import unittest
+
try:
import unittest.mock as mock
except ImportError:
@@ -643,5 +648,38 @@
p1.interpolate()
self.assertEqual(p1.as_dict(), r)
+ def test_complex_overwrites_1(self):
+ # find a better name for this test
+ p1 = Parameters({ 'test': { 'dict': { 'a': '${values:one}', 'b': '${values:two}' } },
+ 'values': { 'one': 1, 'two': 2, 'three': { 'x': 'X', 'y': 'Y' } } }, SETTINGS, '')
+ p2 = Parameters({ 'test': { 'dict': { 'c': '${values:two}' } } }, SETTINGS, '')
+ p3 = Parameters({ 'test': { 'dict': { '~b': '${values:three}' } } }, SETTINGS, '')
+ r = {'test': {'dict': {'a': 1, 'b': {'x': 'X', 'y': 'Y'}, 'c': 2}}, 'values': {'one': 1, 'three': {'x': 'X', 'y': 'Y'}, 'two': 2} }
+ p2.merge(p3)
+ p1.merge(p2)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r)
+
+ def test_escaped_string_overwrites(self):
+ p1 = Parameters({ 'test': '\${not_a_ref}' }, SETTINGS, '')
+ p2 = Parameters({ 'test': '\${also_not_a_ref}' }, SETTINGS, '')
+ r = { 'test': '${also_not_a_ref}' }
+ p1.merge(p2)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r)
+
+ def test_escaped_string_in_ref_dict_overwrite(self):
+ p1 = Parameters({'a': { 'one': '\${not_a_ref}' }, 'b': { 'two': '\${also_not_a_ref}' }}, SETTINGS, '')
+ p2 = Parameters({'c': '${a}'}, SETTINGS, '')
+ p3 = Parameters({'c': '${b}'}, SETTINGS, '')
+ p4 = Parameters({'c': { 'one': '\${again_not_a_ref}' } }, SETTINGS, '')
+ r = {'a': {'one': '${not_a_ref}'}, 'b': {'two': '${also_not_a_ref}'}, 'c': {'one': '${again_not_a_ref}', 'two': '${also_not_a_ref}'}}
+ p1.merge(p2)
+ p1.merge(p3)
+ p1.merge(p4)
+ p1.interpolate()
+ self.assertEqual(p1.as_dict(), r)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/reclass/defaults.py b/reclass/defaults.py
index ccd3db9..19592c6 100644
--- a/reclass/defaults.py
+++ b/reclass/defaults.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
import os, sys
from .version import RECLASS_NAME
diff --git a/reclass/errors.py b/reclass/errors.py
index a96c47b..800a2f8 100644
--- a/reclass/errors.py
+++ b/reclass/errors.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import posix, sys
import traceback
@@ -39,14 +43,12 @@
def exit_with_message(self, out=sys.stderr):
if self._full_traceback:
t, v, tb = sys.exc_info()
- print >>out, 'Full Traceback:'
+ print('Full Traceback', file=out)
for l in traceback.format_tb(tb):
- print >>out, l,
- print >>out
+ print(l, file=out)
if self._traceback:
- print >>out, self._traceback
- print >>out, self.message
- print >>out
+ print(self._traceback, file=out)
+ print(self.message, file=out)
sys.exit(self.rc)
@@ -158,6 +160,17 @@
return msg
+class ClassNameResolveError(InterpolationError):
+ def __init__(self, classname, nodename, uri):
+ super(ClassNameResolveError, self).__init__(msg=None, uri=uri, nodename=nodename)
+ self.name = classname
+
+ def _get_error_message(self):
+ msg = [ 'In {0}'.format(self.uri),
+ 'Class name {0} not resolvable'.format(self.name) ]
+ return msg
+
+
class InvQueryClassNotFound(InterpolationError):
def __init__(self, classNotFoundError, nodename=''):
@@ -172,6 +185,19 @@
return msg
+class InvQueryClassNameResolveError(InterpolationError):
+ def __init__(self, classNameResolveError, nodename=''):
+ super(InvQueryClassNameResolveError, self).__init__(msg=None, nodename=nodename)
+ self.classNameResolveError = classNameResolveError
+ self._traceback = self.classNameResolveError._traceback
+
+ def _get_error_message(self):
+ msg = [ 'Inventory Queries:',
+ '-> {0}'.format(self.classNameResolveError.nodename) ]
+ msg.append(self.classNameResolveError._get_error_message())
+ return msg
+
+
class ResolveError(InterpolationError):
def __init__(self, reference, uri=None, context=None):
diff --git a/reclass/output/__init__.py b/reclass/output/__init__.py
index 42fdb0b..000952c 100644
--- a/reclass/output/__init__.py
+++ b/reclass/output/__init__.py
@@ -6,7 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
class OutputterBase(object):
@@ -14,7 +17,7 @@
pass
def dump(self, data, pretty_print=False):
- raise NotImplementedError('dump() method not implemented.')
+ raise NotImplementedError("dump() method not implemented.")
class OutputLoader(object):
@@ -24,7 +27,7 @@
try:
self._module = __import__(self._name, globals(), locals(), self._name)
except ImportError:
- raise NotImplementedError
+ raise NotImplementedError()
def load(self, attr='Outputter'):
klass = getattr(self._module, attr, None)
diff --git a/reclass/output/json_outputter.py b/reclass/output/json_outputter.py
index 8c79039..5d4cfd4 100644
--- a/reclass/output/json_outputter.py
+++ b/reclass/output/json_outputter.py
@@ -6,9 +6,15 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from reclass.output import OutputterBase
import json
+
class Outputter(OutputterBase):
def dump(self, data, pretty_print=False, no_refs=False):
diff --git a/reclass/output/yaml_outputter.py b/reclass/output/yaml_outputter.py
index 9a0d098..05519c6 100644
--- a/reclass/output/yaml_outputter.py
+++ b/reclass/output/yaml_outputter.py
@@ -6,18 +6,27 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from reclass.output import OutputterBase
import yaml
+_SafeDumper = yaml.CSafeDumper if yaml.__with_libyaml__ else yaml.SafeDumper
+
+
class Outputter(OutputterBase):
def dump(self, data, pretty_print=False, no_refs=False):
if (no_refs):
return yaml.dump(data, default_flow_style=not pretty_print, Dumper=ExplicitDumper)
else:
- return yaml.dump(data, default_flow_style=not pretty_print)
+ return yaml.dump(data, default_flow_style=not pretty_print, Dumper=_SafeDumper)
-class ExplicitDumper(yaml.SafeDumper):
+
+class ExplicitDumper(_SafeDumper):
"""
A dumper that will never emit aliases.
"""
diff --git a/reclass/settings.py b/reclass/settings.py
index a33b742..af6c76d 100644
--- a/reclass/settings.py
+++ b/reclass/settings.py
@@ -1,11 +1,15 @@
+# -*- coding: utf-8
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
import copy
import reclass.values.parser_funcs
from reclass.defaults import *
-try:
- basestring
-except NameError:
- basestring = str
+from six import string_types
class Settings(object):
@@ -27,7 +31,7 @@
self.ignore_class_notfound = options.get('ignore_class_notfound', OPT_IGNORE_CLASS_NOTFOUND)
self.ignore_class_notfound_regexp = options.get('ignore_class_notfound_regexp', OPT_IGNORE_CLASS_NOTFOUND_REGEXP)
- if isinstance(self.ignore_class_notfound_regexp, basestring):
+ if isinstance(self.ignore_class_notfound_regexp, string_types):
self.ignore_class_notfound_regexp = [ self.ignore_class_notfound_regexp ]
self.ignore_class_notfound_warning = options.get('ignore_class_notfound_warning', OPT_IGNORE_CLASS_NOTFOUND_WARNING)
diff --git a/reclass/storage/__init__.py b/reclass/storage/__init__.py
index f49ac16..3b46a2a 100644
--- a/reclass/storage/__init__.py
+++ b/reclass/storage/__init__.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
class NodeStorageBase(object):
diff --git a/reclass/storage/common.py b/reclass/storage/common.py
index 6a77fc8..7de71d0 100644
--- a/reclass/storage/common.py
+++ b/reclass/storage/common.py
@@ -1,3 +1,9 @@
+# -*- coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
import os
class NameMangler:
diff --git a/reclass/storage/loader.py b/reclass/storage/loader.py
index 10ca74c..aab554a 100644
--- a/reclass/storage/loader.py
+++ b/reclass/storage/loader.py
@@ -6,16 +6,20 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
-import importlib
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from importlib import import_module
class StorageBackendLoader(object):
def __init__(self, storage_name):
- self._name = 'reclass.storage.' + storage_name
+ self._name = str('reclass.storage.' + storage_name)
try:
- self._module = importlib.import_module(self._name)
- except ImportError:
+ self._module = import_module(self._name)
+ except ImportError as e:
raise NotImplementedError
def load(self, klassname='ExternalNodeStorage'):
diff --git a/reclass/storage/memcache_proxy.py b/reclass/storage/memcache_proxy.py
index 8c5e441..cd90fdd 100644
--- a/reclass/storage/memcache_proxy.py
+++ b/reclass/storage/memcache_proxy.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.storage import NodeStorageBase
diff --git a/reclass/storage/mixed/__init__.py b/reclass/storage/mixed/__init__.py
index 990c931..6324c74 100644
--- a/reclass/storage/mixed/__init__.py
+++ b/reclass/storage/mixed/__init__.py
@@ -2,6 +2,10 @@
# -*- coding: utf-8 -*-
#
# This file is part of reclass
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import collections
import copy
diff --git a/reclass/storage/tests/__init__.py b/reclass/storage/tests/__init__.py
index e69de29..9aaaf25 100644
--- a/reclass/storage/tests/__init__.py
+++ b/reclass/storage/tests/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/storage/tests/test_loader.py b/reclass/storage/tests/test_loader.py
index 6bef87f..12cdec3 100644
--- a/reclass/storage/tests/test_loader.py
+++ b/reclass/storage/tests/test_loader.py
@@ -6,10 +6,16 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from reclass.storage.loader import StorageBackendLoader
import unittest
+
class TestLoader(unittest.TestCase):
def test_load(self):
diff --git a/reclass/storage/tests/test_memcache_proxy.py b/reclass/storage/tests/test_memcache_proxy.py
index a47c29d..24acf20 100644
--- a/reclass/storage/tests/test_memcache_proxy.py
+++ b/reclass/storage/tests/test_memcache_proxy.py
@@ -6,17 +6,23 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.settings import Settings
from reclass.storage.memcache_proxy import MemcacheProxy
from reclass.storage import NodeStorageBase
import unittest
+
try:
import unittest.mock as mock
except ImportError:
import mock
+
class TestMemcacheProxy(unittest.TestCase):
def setUp(self):
diff --git a/reclass/storage/tests/test_yamldata.py b/reclass/storage/tests/test_yamldata.py
index d8129ce..5c48db6 100644
--- a/reclass/storage/tests/test_yamldata.py
+++ b/reclass/storage/tests/test_yamldata.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass (http://github.com/madduck/reclass)
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.storage.yamldata import YamlData
diff --git a/reclass/storage/yaml_fs/__init__.py b/reclass/storage/yaml_fs/__init__.py
index 83f3666..a102f31 100644
--- a/reclass/storage/yaml_fs/__init__.py
+++ b/reclass/storage/yaml_fs/__init__.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
import os, sys
import fnmatch
import yaml
@@ -21,7 +26,7 @@
STORAGE_NAME = 'yaml_fs'
def vvv(msg):
- #print >>sys.stderr, msg
+ #print(msg, file=sys.stderr)
pass
def path_mangler(inventory_base_uri, nodes_uri, classes_uri):
diff --git a/reclass/storage/yaml_fs/directory.py b/reclass/storage/yaml_fs/directory.py
index 614e1c3..a8916b3 100644
--- a/reclass/storage/yaml_fs/directory.py
+++ b/reclass/storage/yaml_fs/directory.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
import os
from reclass.errors import NotFoundError
@@ -13,7 +18,7 @@
FILE_EXTENSION = '.yml'
def vvv(msg):
- #print >>sys.stderr, msg
+ #print(msg, file=sys.stderr)
pass
diff --git a/reclass/storage/yaml_git/__init__.py b/reclass/storage/yaml_git/__init__.py
index 86c1247..c26ef77 100644
--- a/reclass/storage/yaml_git/__init__.py
+++ b/reclass/storage/yaml_git/__init__.py
@@ -2,6 +2,10 @@
# -*- coding: utf-8 -*-
#
# This file is part of reclass
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import collections
import distutils.version
diff --git a/reclass/storage/yamldata.py b/reclass/storage/yamldata.py
index ac3dcb9..a861154 100644
--- a/reclass/storage/yamldata.py
+++ b/reclass/storage/yamldata.py
@@ -6,11 +6,18 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from reclass import datatypes
import yaml
import os
from reclass.errors import NotFoundError
+_SafeLoader = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader
+
class YamlData(object):
@classmethod
@@ -23,7 +30,7 @@
raise NotFoundError('Cannot open: %s' % abs_path)
y = cls('yaml_fs://{0}'.format(abs_path))
with open(abs_path) as fp:
- data = yaml.safe_load(fp)
+ data = yaml.load(fp, Loader=_SafeLoader)
if data is not None:
y._data = data
return y
@@ -32,7 +39,7 @@
def from_string(cls, string, uri):
''' Initialise yaml data from a string '''
y = cls(uri)
- data = yaml.safe_load(string)
+ data = yaml.load(string, Loader=_SafeLoader)
if data is not None:
y._data = data
return y
diff --git a/reclass/tests/__init__.py b/reclass/tests/__init__.py
index e69de29..9aaaf25 100644
--- a/reclass/tests/__init__.py
+++ b/reclass/tests/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/tests/test_core.py b/reclass/tests/test_core.py
index 9225756..679d6ca 100644
--- a/reclass/tests/test_core.py
+++ b/reclass/tests/test_core.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass (http://github.com/madduck/reclass)
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import os
diff --git a/reclass/utils/__init__.py b/reclass/utils/__init__.py
index e69de29..9aaaf25 100644
--- a/reclass/utils/__init__.py
+++ b/reclass/utils/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/utils/dictpath.py b/reclass/utils/dictpath.py
index dfb8b32..39b9572 100644
--- a/reclass/utils/dictpath.py
+++ b/reclass/utils/dictpath.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import six
import re
diff --git a/reclass/utils/tests/__init__.py b/reclass/utils/tests/__init__.py
index e69de29..9aaaf25 100644
--- a/reclass/utils/tests/__init__.py
+++ b/reclass/utils/tests/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/utils/tests/test_dictpath.py b/reclass/utils/tests/test_dictpath.py
index 972dc91..6fbb6b7 100644
--- a/reclass/utils/tests/test_dictpath.py
+++ b/reclass/utils/tests/test_dictpath.py
@@ -6,6 +6,11 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
from reclass.utils.dictpath import DictPath
import unittest
@@ -64,7 +69,7 @@
delim = '%'
s = 'a:b\:b:c'
p = DictPath(delim, s)
- self.assertEqual('%r' % p, 'DictPath(%r, %r)' % (delim, s))
+ self.assertEqual('%r' % p, "DictPath(%r, %r)" % (delim, str(s)))
def test_str(self):
s = 'a:b\:b:c'
diff --git a/reclass/values/__init__.py b/reclass/values/__init__.py
index e69de29..9aaaf25 100644
--- a/reclass/values/__init__.py
+++ b/reclass/values/__init__.py
@@ -0,0 +1,5 @@
+# -*- coding: utf-8
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/values/compitem.py b/reclass/values/compitem.py
index 765b323..7928e6f 100644
--- a/reclass/values/compitem.py
+++ b/reclass/values/compitem.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.settings import Settings
from .item import Item
@@ -39,6 +43,21 @@
def get_references(self):
return self._refs
+ def merge_over(self, item):
+ if item.type == Item.SCALAR or item.type == Item.COMPOSITE:
+ return self
+ elif item.type == Item.LIST:
+ if self._settings.allow_scalar_over_list or (self._settings.allow_none_override and self._value in [None, 'none', 'None']):
+ return self
+ else:
+ raise TypeError('allow scalar over list = False: cannot merge %s over %s' % (repr(self), repr(item)))
+ elif item.type == Item.DICTIONARY:
+ if self._settings.allow_scalar_over_dict or (self._settings.allow_none_override and self._value in [None, 'none', 'None']):
+ return self
+ else:
+ raise TypeError('allow scalar over dict = False: cannot merge %s over %s' % (repr(self), repr(item)))
+ raise TypeError('Cannot merge %s over %s' % (repr(self), repr(item)))
+
def render(self, context, inventory):
# Preserve type if only one item
if len(self._items) == 1:
diff --git a/reclass/values/dictitem.py b/reclass/values/dictitem.py
index 555bd8f..76cefe2 100644
--- a/reclass/values/dictitem.py
+++ b/reclass/values/dictitem.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.settings import Settings
from .item import Item
diff --git a/reclass/values/invitem.py b/reclass/values/invitem.py
index 970321b..37a35cf 100644
--- a/reclass/values/invitem.py
+++ b/reclass/values/invitem.py
@@ -3,11 +3,15 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import copy
import pyparsing as pp
-from six import iteritems
+from six import iteritems, string_types
from .item import Item
from reclass.settings import Settings
@@ -90,7 +94,7 @@
raise ResolveError(str(path))
def _get_vars(self, var, export, parameter, value):
- if isinstance(var, str):
+ if isinstance(var, string_types):
path = DictPath(self._delimiter, var)
if path.path[0].lower() == 'exports':
export = path
diff --git a/reclass/values/item.py b/reclass/values/item.py
index 57fd0e3..bc507f4 100644
--- a/reclass/values/item.py
+++ b/reclass/values/item.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.utils.dictpath import DictPath
diff --git a/reclass/values/listitem.py b/reclass/values/listitem.py
index 1829e32..8f1a21d 100644
--- a/reclass/values/listitem.py
+++ b/reclass/values/listitem.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from .item import Item
from reclass.settings import Settings
diff --git a/reclass/values/parser.py b/reclass/values/parser.py
index a8adcf0..914e825 100644
--- a/reclass/values/parser.py
+++ b/reclass/values/parser.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import pyparsing as pp
diff --git a/reclass/values/parser_funcs.py b/reclass/values/parser_funcs.py
index bd5a1ba..46db7cc 100644
--- a/reclass/values/parser_funcs.py
+++ b/reclass/values/parser_funcs.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import pyparsing as pp
diff --git a/reclass/values/refitem.py b/reclass/values/refitem.py
index 3e3341c..d24eeee 100644
--- a/reclass/values/refitem.py
+++ b/reclass/values/refitem.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from .item import Item
from reclass.defaults import REFERENCE_SENTINELS
diff --git a/reclass/values/scaitem.py b/reclass/values/scaitem.py
index 9de5681..f3057cb 100644
--- a/reclass/values/scaitem.py
+++ b/reclass/values/scaitem.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.settings import Settings
from .item import Item
@@ -18,7 +22,7 @@
return self._value
def merge_over(self, item):
- if item.type == Item.SCALAR:
+ if item.type == Item.SCALAR or item.type == Item.COMPOSITE:
return self
elif item.type == Item.LIST:
if self._settings.allow_scalar_over_list or (self._settings.allow_none_override and self._value is None):
diff --git a/reclass/values/tests/__init__.py b/reclass/values/tests/__init__.py
index e69de29..16d1248 100644
--- a/reclass/values/tests/__init__.py
+++ b/reclass/values/tests/__init__.py
@@ -0,0 +1,7 @@
+#
+# -*- coding: utf-8
+#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
diff --git a/reclass/values/tests/test_value.py b/reclass/values/tests/test_value.py
index 84403d3..4853340 100644
--- a/reclass/values/tests/test_value.py
+++ b/reclass/values/tests/test_value.py
@@ -6,6 +6,10 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import pyparsing as pp
diff --git a/reclass/values/value.py b/reclass/values/value.py
index 74ef272..f87a9b4 100644
--- a/reclass/values/value.py
+++ b/reclass/values/value.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from .parser import Parser
from .dictitem import DictItem
@@ -10,19 +14,25 @@
from .scaitem import ScaItem
from reclass.errors import InterpolationError
+from six import string_types
+
class Value(object):
_parser = Parser()
- def __init__(self, value, settings, uri):
+ def __init__(self, value, settings, uri, parse_string=True):
self._settings = settings
self._uri = uri
- if isinstance(value, str):
- try:
- self._item = self._parser.parse(value, self._settings)
- except InterpolationError as e:
- e.uri = self._uri
- raise
+ self._overwrite = False
+ if isinstance(value, string_types):
+ if parse_string:
+ try:
+ self._item = self._parser.parse(value, self._settings)
+ except InterpolationError as e:
+ e.uri = self._uri
+ raise
+ else:
+ self._item = ScaItem(value, self._settings)
elif isinstance(value, list):
self._item = ListItem(value, self._settings)
elif isinstance(value, dict):
@@ -30,6 +40,14 @@
else:
self._item = ScaItem(value, self._settings)
+ @property
+ def overwrite(self):
+ return self._overwrite
+
+ @overwrite.setter
+ def overwrite(self, overwrite):
+ self._overwrite = overwrite
+
def uri(self):
return self._uri
diff --git a/reclass/values/valuelist.py b/reclass/values/valuelist.py
index fc0eaad..8b8dd51 100644
--- a/reclass/values/valuelist.py
+++ b/reclass/values/valuelist.py
@@ -3,6 +3,10 @@
#
# This file is part of reclass
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from __future__ import print_function
@@ -35,6 +39,9 @@
self.assembleRefs()
self._check_for_inv_query()
+ def uri(self):
+ return '; '.join([ x.uri() for x in self._values ])
+
def has_references(self):
return len(self._refs) > 0
@@ -104,14 +111,14 @@
else:
raise e
- if output is None:
+ if output is None or value.overwrite:
output = new
deepCopied = False
else:
if isinstance(output, dict) and isinstance(new, dict):
- p1 = Parameters(output, self._settings, None, merge_initialise = False)
- p2 = Parameters(new, self._settings, None, merge_initialise = False)
- p1.merge(p2, wrap=False)
+ p1 = Parameters(output, self._settings, None, parse_strings=False)
+ p2 = Parameters(new, self._settings, None, parse_strings=False)
+ p1.merge(p2)
output = p1.as_dict()
continue
elif isinstance(output, list) and isinstance(new, list):
@@ -124,6 +131,7 @@
raise TypeError('Cannot merge %s over %s' % (repr(self._values[n]), repr(self._values[n-1])))
else:
output = new
+ deepCopied = False
if isinstance(output, (dict, list)) and last_error is not None:
raise last_error
diff --git a/reclass/version.py b/reclass/version.py
index 90c2cb7..ccf6e96 100644
--- a/reclass/version.py
+++ b/reclass/version.py
@@ -6,9 +6,14 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
RECLASS_NAME = 'reclass'
DESCRIPTION = 'merge data by recursive descent down an ancestry hierarchy (forked extended version)'
-VERSION = '1.5.2'
+VERSION = '1.5.4'
AUTHOR = 'martin f. krafft / Andrew Pickford / salt-formulas community'
AUTHOR_EMAIL = 'salt-formulas@freelists.org'
MAINTAINER = 'salt-formulas community'
diff --git a/run_tests.py b/run_tests.py
index 1506945..aeccb57 100755
--- a/run_tests.py
+++ b/run_tests.py
@@ -6,6 +6,10 @@
# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
import unittest
tests = unittest.TestLoader().discover('reclass')
diff --git a/setup.py b/setup.py
index 5b4b8b6..789b0fd 100644
--- a/setup.py
+++ b/setup.py
@@ -6,6 +6,10 @@
# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
from reclass.version import *
from setuptools import setup, find_packages
diff --git a/test/model/default/classes/first.yml b/test/model/default/classes/first.yml
new file mode 100644
index 0000000..9b72a26
--- /dev/null
+++ b/test/model/default/classes/first.yml
@@ -0,0 +1,32 @@
+parameters:
+ _param:
+ some: param
+ colour: red
+ lab:
+ name: test
+ label: first
+ colour:
+ escaped: \${_param:colour}
+ doubleescaped: \\${_param:colour}
+ unescaped: ${_param:colour}
+ colours:
+ red:
+ name: red
+ blue:
+ name: blue
+ one:
+ a: 1
+ b: 2
+ two:
+ c: 3
+ d: 4
+ three:
+ e: 5
+ list_to_override:
+ - one
+ - two
+ dict_to_override:
+ one: 1
+ two: 2
+
+
diff --git a/test/model/default/classes/lab/env/dev.yml b/test/model/default/classes/lab/env/dev.yml
new file mode 100644
index 0000000..0cce363
--- /dev/null
+++ b/test/model/default/classes/lab/env/dev.yml
@@ -0,0 +1,4 @@
+
+parameters:
+ lab:
+ name: dev
diff --git a/test/model/default/classes/second.yml b/test/model/default/classes/second.yml
new file mode 100644
index 0000000..dab50c7
--- /dev/null
+++ b/test/model/default/classes/second.yml
@@ -0,0 +1,9 @@
+classes:
+- first
+
+parameters:
+ will:
+ warn:
+ at:
+ second: ${_param:notfound}
+ three: ${one}
diff --git a/test/model/default/classes/third.yml b/test/model/default/classes/third.yml
new file mode 100644
index 0000000..135acd4
--- /dev/null
+++ b/test/model/default/classes/third.yml
@@ -0,0 +1,18 @@
+classes:
+- second
+
+parameters:
+ _param:
+ notfound: exist
+ myparam: ${_param:some}
+ will:
+ not:
+ fail:
+ at:
+ tree: ${_param:notfound}
+ three: ${two}
+ empty:
+ list: []
+ dict: {}
+ ~list_to_override: ${empty:list}
+ ~dict_to_override: ${empty:dict}
diff --git a/test/model/default/nodes/reclass.yml b/test/model/default/nodes/reclass.yml
new file mode 100644
index 0000000..94b7519
--- /dev/null
+++ b/test/model/default/nodes/reclass.yml
@@ -0,0 +1,3 @@
+
+classes:
+- third
diff --git a/test/model/default/reclass-config.yml b/test/model/default/reclass-config.yml
new file mode 100644
index 0000000..9d8f30f
--- /dev/null
+++ b/test/model/default/reclass-config.yml
@@ -0,0 +1 @@
+storage_type: yaml_fs
diff --git a/test/model/extensions/classes/first.yml b/test/model/extensions/classes/first.yml
new file mode 100644
index 0000000..96ece27
--- /dev/null
+++ b/test/model/extensions/classes/first.yml
@@ -0,0 +1,6 @@
+parameters:
+ _param:
+ some: param
+ lab:
+ name: test
+ label: first
diff --git a/test/model/extensions/classes/lab/env/dev.yml b/test/model/extensions/classes/lab/env/dev.yml
new file mode 100644
index 0000000..0cce363
--- /dev/null
+++ b/test/model/extensions/classes/lab/env/dev.yml
@@ -0,0 +1,4 @@
+
+parameters:
+ lab:
+ name: dev
diff --git a/test/model/extensions/classes/second.yml b/test/model/extensions/classes/second.yml
new file mode 100644
index 0000000..a9babd3
--- /dev/null
+++ b/test/model/extensions/classes/second.yml
@@ -0,0 +1,8 @@
+classes:
+- first
+
+parameters:
+ will:
+ warn:
+ at:
+ second: ${_param:notfound}
diff --git a/test/model/extensions/classes/third.yml b/test/model/extensions/classes/third.yml
new file mode 100644
index 0000000..20a937c
--- /dev/null
+++ b/test/model/extensions/classes/third.yml
@@ -0,0 +1,13 @@
+classes:
+- missing.class
+- second
+
+parameters:
+ _param:
+ notfound: exist
+ myparam: ${_param:some}
+ will:
+ not:
+ fail:
+ at:
+ tree: ${_param:notfound}
diff --git a/test/model/extensions/nodes/reclass.yml b/test/model/extensions/nodes/reclass.yml
new file mode 100644
index 0000000..94b7519
--- /dev/null
+++ b/test/model/extensions/nodes/reclass.yml
@@ -0,0 +1,3 @@
+
+classes:
+- third
diff --git a/test/model/extensions/reclass-config.yml b/test/model/extensions/reclass-config.yml
new file mode 100644
index 0000000..6e2f101
--- /dev/null
+++ b/test/model/extensions/reclass-config.yml
@@ -0,0 +1,3 @@
+storage_type: yaml_fs
+ignore_class_notfound: True
+ignore_class_regexp: ['.*']