Merge pull request #25 from salt-formulas/python3-new-adrian
New Python3 branch (by Adrian Chifor)
diff --git a/.gitignore b/.gitignore
index c5e9682..5503138 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
*.py[co]
.*.sw?
+.DS_Store
/reclass-config.yml
/reclass.egg-info
/build
diff --git a/reclass/__init__.py b/reclass/__init__.py
index adb421e..a79c8e1 100644
--- a/reclass/__init__.py
+++ b/reclass/__init__.py
@@ -11,6 +11,7 @@
from reclass.storage.loader import StorageBackendLoader
from reclass.storage.memcache_proxy import MemcacheProxy
+
def get_storage(storage_type, nodes_uri, classes_uri, **kwargs):
storage_class = StorageBackendLoader(storage_type).load()
return MemcacheProxy(storage_class(nodes_uri, classes_uri, **kwargs))
diff --git a/reclass/adapters/ansible.py b/reclass/adapters/ansible.py
index 1887245..f6e9af3 100755
--- a/reclass/adapters/ansible.py
+++ b/reclass/adapters/ansible.py
@@ -16,6 +16,8 @@
import os, sys, posix, optparse
+from six import iteritems
+
from reclass import get_storage, output
from reclass.core import Core
from reclass.errors import ReclassException
@@ -81,7 +83,7 @@
apps = data['applications']
if options.applications_postfix:
postfix = options.applications_postfix
- groups.update([(k + postfix, v) for k,v in apps.iteritems()])
+ groups.update([(k + postfix, v) for (k, v) in iteritems(apps)])
else:
groups.update(apps)
diff --git a/reclass/adapters/salt.py b/reclass/adapters/salt.py
index 54adf5a..31179ff 100755
--- a/reclass/adapters/salt.py
+++ b/reclass/adapters/salt.py
@@ -9,6 +9,8 @@
import os, sys, posix
+from six import iteritems
+
from reclass import get_storage, output, get_path_mangler
from reclass.core import Core
from reclass.errors import ReclassException
@@ -68,7 +70,7 @@
else:
data = reclass.inventory()
nodes = {}
- for node_id, node_data in data['nodes'].iteritems():
+ for (node_id, node_data) in iteritems(data['nodes']):
env = node_data['environment']
if env not in nodes:
nodes[env] = {}
diff --git a/reclass/core.py b/reclass/core.py
index 23851a5..92e7c25 100644
--- a/reclass/core.py
+++ b/reclass/core.py
@@ -15,12 +15,20 @@
import string
import sys
import yaml
+
+from six import iteritems
+
from reclass.settings import Settings
from reclass.output.yaml_outputter import ExplicitDumper
from reclass.datatypes import Entity, Classes, Parameters, Exports
from reclass.errors import MappingFormatError, ClassNotFound, InvQueryClassNotFound, InvQueryError, InterpolationError
from reclass.values.parser import Parser
+try:
+ basestring
+except NameError:
+ basestring = str
+
class Core(object):
def __init__(self, storage, class_mappings, settings, input_data=None):
@@ -55,7 +63,7 @@
regexp = True
try:
key = lexer.get_token()
- except ValueError, e:
+ except ValueError as e:
raise MappingFormatError('Error in mapping "{0}": missing closing '
'quote (or slash)'.format(instr))
if regexp:
@@ -135,7 +143,7 @@
def _get_automatic_parameters(self, nodename, environment):
if self._settings.automatic_parameters:
- return Parameters({ '_reclass_': { 'name': { 'full': nodename, 'short': string.split(nodename, '.')[0] },
+ return Parameters({ '_reclass_': { 'name': { 'full': nodename, 'short': str.split(nodename, '.')[0] },
'environment': environment } }, self._settings, '__auto__')
else:
return Parameters({}, self._settings, '')
@@ -227,7 +235,7 @@
nodes = {}
applications = {}
classes = {}
- for f, nodeinfo in entities.iteritems():
+ for (f, nodeinfo) in iteritems(entities):
d = nodes[f] = self._nodeinfo_as_dict(f, nodeinfo)
for a in d['applications']:
if a in applications:
diff --git a/reclass/datatypes/classes.py b/reclass/datatypes/classes.py
index b8793a2..090ed70 100644
--- a/reclass/datatypes/classes.py
+++ b/reclass/datatypes/classes.py
@@ -7,12 +7,13 @@
# Released under the terms of the Artistic Licence 2.0
#
-import types
+import six
import os
from reclass.errors import InvalidClassnameError
INVALID_CHARACTERS_FOR_CLASSNAMES = ' ' + os.sep
+
class Classes(object):
'''
A very limited ordered set of strings with O(n) uniqueness constraints. It
@@ -51,7 +52,7 @@
self.append_if_new(i)
def _assert_is_string(self, item):
- if not isinstance(item, types.StringTypes):
+ if not isinstance(item, six.string_types):
raise TypeError('%s instances can only contain strings, '\
'not %s' % (self.__class__.__name__, type(item)))
diff --git a/reclass/datatypes/exports.py b/reclass/datatypes/exports.py
index 62ea03f..971befa 100644
--- a/reclass/datatypes/exports.py
+++ b/reclass/datatypes/exports.py
@@ -6,6 +6,8 @@
import copy
+from six import iteritems, next
+
from .parameters import Parameters
from reclass.errors import ResolveError
from reclass.values.value import Value
@@ -25,12 +27,12 @@
self._unrendered.pop(key, None)
def overwrite(self, other):
- overdict = {'~' + key: value for key, value in other.iteritems()}
+ overdict = {'~' + key: value for (key, value) in iteritems(other)}
self.merge(overdict)
def interpolate_from_external(self, external):
while len(self._unrendered) > 0:
- path, v = self._unrendered.iteritems().next()
+ path, v = next(iteritems(self._unrendered))
value = path.get_value(self._base)
if isinstance(value, (Value, ValueList)):
external._interpolate_references(path, value, None)
@@ -51,7 +53,7 @@
required = self._get_required_paths(mainpath)
while len(required) > 0:
while len(required) > 0:
- path, v = required.iteritems().next()
+ path, v = next(iteritems(required))
value = path.get_value(self._base)
if isinstance(value, (Value, ValueList)):
try:
diff --git a/reclass/datatypes/parameters.py b/reclass/datatypes/parameters.py
index ac15925..c96a67d 100644
--- a/reclass/datatypes/parameters.py
+++ b/reclass/datatypes/parameters.py
@@ -10,12 +10,16 @@
import copy
import sys
import types
+
+from six import iteritems, next
+
from collections import namedtuple
from reclass.utils.dictpath import DictPath
from reclass.values.value import Value
from reclass.values.valuelist import ValueList
from reclass.errors import InfiniteRecursionError, ResolveError, ResolveErrorList, InterpolationError, ParseError, BadReferencesError
+
class Parameters(object):
'''
A class to hold nested dictionaries with the following specialities:
@@ -108,7 +112,7 @@
return [ self._wrap_value(v, path.new_subpath(k)) for (k, v) in enumerate(source) ]
def _wrap_dict(self, source, path):
- return { k: self._wrap_value(v, path.new_subpath(k)) for k, v in source.iteritems() }
+ return { k: self._wrap_value(v, path.new_subpath(k)) for (k, v) in iteritems(source) }
def _update_value(self, cur, new):
if isinstance(cur, Value):
@@ -147,7 +151,7 @@
"""
ret = cur
- for key, newvalue in new.iteritems():
+ for (key, newvalue) in iteritems(new):
if key.startswith(self._settings.dict_key_override_prefix) and not self._keep_overrides:
ret[key.lstrip(self._settings.dict_key_override_prefix)] = newvalue
else:
@@ -243,7 +247,7 @@
container[key] = value.render(None, None)
def _render_simple_dict(self, dictionary, path):
- for key, value in dictionary.iteritems():
+ for (key, value) in iteritems(dictionary):
self._render_simple_container(dictionary, key, value, path)
def _render_simple_list(self, item_list, path):
@@ -256,7 +260,7 @@
# we could use a view here, but this is simple enough:
# _interpolate_inner removes references from the refs hash after
# processing them, so we cannot just iterate the dict
- path, v = self._unrendered.iteritems().next()
+ path, v = next(iteritems(self._unrendered))
self._interpolate_inner(path, inventory)
if self._resolve_errors.have_errors():
raise self._resolve_errors
diff --git a/reclass/datatypes/tests/test_applications.py b/reclass/datatypes/tests/test_applications.py
index 307a430..6ae07cc 100644
--- a/reclass/datatypes/tests/test_applications.py
+++ b/reclass/datatypes/tests/test_applications.py
@@ -63,7 +63,7 @@
l = ['a', '~b', 'a', '~d']
a = Applications(l)
is_negation = lambda x: x.startswith(a.negation_prefix)
- GOAL = filter(lambda x: not is_negation(x), set(l)) + filter(is_negation, l)
+ GOAL = list(filter(lambda x: not is_negation(x), set(l))) + list(filter(is_negation, l))
self.assertEqual('%r' % a, "%s(%r, '~')" % (a.__class__.__name__, GOAL))
if __name__ == '__main__':
diff --git a/reclass/datatypes/tests/test_parameters.py b/reclass/datatypes/tests/test_parameters.py
index 5c91893..b5dc243 100644
--- a/reclass/datatypes/tests/test_parameters.py
+++ b/reclass/datatypes/tests/test_parameters.py
@@ -9,6 +9,8 @@
import copy
+from six import iteritems
+
from reclass.settings import Settings
from reclass.datatypes import Parameters
from reclass.errors import InfiniteRecursionError, InterpolationError, ResolveError, ResolveErrorList
@@ -132,7 +134,7 @@
p2, b2 = self._construct_mocked_params(mergee)
p1.merge(p2)
p1.initialise_interpolation()
- for key, value in mergee.iteritems():
+ for (key, value) in iteritems(mergee):
# check that each key, value in mergee resulted in a get call and
# a __setitem__ call against b1 (the merge target)
self.assertIn(mock.call(key), b1.get.call_args_list)
diff --git a/reclass/defaults.py b/reclass/defaults.py
index a07877d..408307d 100644
--- a/reclass/defaults.py
+++ b/reclass/defaults.py
@@ -7,7 +7,7 @@
# Released under the terms of the Artistic Licence 2.0
#
import os, sys
-from version import RECLASS_NAME
+from .version import RECLASS_NAME
# defaults for the command-line options
OPT_STORAGE_TYPE = 'yaml_fs'
diff --git a/reclass/output/__init__.py b/reclass/output/__init__.py
index 58cd101..42fdb0b 100644
--- a/reclass/output/__init__.py
+++ b/reclass/output/__init__.py
@@ -6,13 +6,15 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+
+
class OutputterBase(object):
def __init__(self):
pass
def dump(self, data, pretty_print=False):
- raise NotImplementedError, "dump() method not yet implemented"
+ raise NotImplementedError('dump() method not implemented.')
class OutputLoader(object):
@@ -27,6 +29,5 @@
def load(self, attr='Outputter'):
klass = getattr(self._module, attr, None)
if klass is None:
- raise AttributeError, \
- 'Outputter class {0} does not export "{1}"'.format(self._name, klass)
+ raise AttributeError('Outputter class {0} does not export "{1}"'.format(self._name, klass))
return klass
diff --git a/reclass/settings.py b/reclass/settings.py
index 44c58d8..e3fc26e 100644
--- a/reclass/settings.py
+++ b/reclass/settings.py
@@ -2,6 +2,11 @@
import reclass.values.parser_funcs
from reclass.defaults import *
+try:
+ basestring
+except NameError:
+ basestring = str
+
class Settings(object):
def __init__(self, options={}):
diff --git a/reclass/storage/loader.py b/reclass/storage/loader.py
index 77fdecb..10ca74c 100644
--- a/reclass/storage/loader.py
+++ b/reclass/storage/loader.py
@@ -6,13 +6,15 @@
# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
# Released under the terms of the Artistic Licence 2.0
#
+import importlib
+
class StorageBackendLoader(object):
def __init__(self, storage_name):
self._name = 'reclass.storage.' + storage_name
try:
- self._module = __import__(self._name, globals(), locals(), self._name)
+ self._module = importlib.import_module(self._name)
except ImportError:
raise NotImplementedError
@@ -21,7 +23,6 @@
if klass is None:
raise AttributeError('Storage backend class {0} does not export '
'"{1}"'.format(self._name, klassname))
-
return klass
def path_mangler(self, name='path_mangler'):
diff --git a/reclass/storage/memcache_proxy.py b/reclass/storage/memcache_proxy.py
index 405ea8e..8c5e441 100644
--- a/reclass/storage/memcache_proxy.py
+++ b/reclass/storage/memcache_proxy.py
@@ -35,7 +35,7 @@
return self._real_storage.get_node(name, settings)
try:
return self._nodes_cache[name]
- except KeyError, e:
+ except KeyError as e:
ret = self._real_storage.get_node(name, settings)
self._nodes_cache[name] = ret
return ret
@@ -45,7 +45,7 @@
return self._real_storage.get_class(name, environment, settings)
try:
return self._classes_cache[environment][name]
- except KeyError, e:
+ except KeyError as e:
if environment not in self._classes_cache:
self._classes_cache[environment] = dict()
ret = self._real_storage.get_class(name, environment, settings)
diff --git a/reclass/storage/mixed/__init__.py b/reclass/storage/mixed/__init__.py
index 4651e00..990c931 100644
--- a/reclass/storage/mixed/__init__.py
+++ b/reclass/storage/mixed/__init__.py
@@ -6,6 +6,8 @@
import collections
import copy
+from six import iteritems
+
import reclass.errors
from reclass import get_storage
from reclass.storage import NodeStorageBase
@@ -32,7 +34,7 @@
self._classes_storage = dict()
if 'env_overrides' in classes_uri:
for override in classes_uri['env_overrides']:
- for env, options in override.iteritems():
+ for (env, options) in iteritems(override):
uri = copy.deepcopy(classes_uri)
uri.update(options)
uri = self._uri(uri)
diff --git a/reclass/storage/yaml_fs/__init__.py b/reclass/storage/yaml_fs/__init__.py
index b92cbfe..83f3666 100644
--- a/reclass/storage/yaml_fs/__init__.py
+++ b/reclass/storage/yaml_fs/__init__.py
@@ -93,7 +93,7 @@
relpath = self._nodes[name]
path = os.path.join(self.nodes_uri, relpath)
name = os.path.splitext(relpath)[0]
- except KeyError, e:
+ except KeyError as e:
raise reclass.errors.NodeNotFound(self.name, name, self.nodes_uri)
entity = YamlData.from_file(path).get_entity(name, settings)
return entity
@@ -102,7 +102,7 @@
vvv('GET CLASS {0}'.format(name))
try:
path = os.path.join(self.classes_uri, self._classes[name])
- except KeyError, e:
+ except KeyError as e:
raise reclass.errors.ClassNotFound(self.name, name, self.classes_uri)
entity = YamlData.from_file(path).get_entity(name, settings)
return entity
diff --git a/reclass/storage/yaml_fs/directory.py b/reclass/storage/yaml_fs/directory.py
index 03302b7..614e1c3 100644
--- a/reclass/storage/yaml_fs/directory.py
+++ b/reclass/storage/yaml_fs/directory.py
@@ -7,16 +7,16 @@
# Released under the terms of the Artistic Licence 2.0
#
import os
-import sys
from reclass.errors import NotFoundError
-SKIPDIRS = ( 'CVS', 'SCCS' )
+SKIPDIRS = ('CVS', 'SCCS')
FILE_EXTENSION = '.yml'
def vvv(msg):
#print >>sys.stderr, msg
pass
+
class Directory(object):
def __init__(self, path, fileclass=None):
@@ -39,7 +39,8 @@
files = property(lambda self: self._files)
def walk(self, register_fn=None):
- if not callable(register_fn): register_fn = self._register_files
+ if not callable(register_fn):
+ register_fn = self._register_files
def _error(exc):
raise(exc)
diff --git a/reclass/storage/yaml_git/__init__.py b/reclass/storage/yaml_git/__init__.py
index f4cb287..86c1247 100644
--- a/reclass/storage/yaml_git/__init__.py
+++ b/reclass/storage/yaml_git/__init__.py
@@ -15,6 +15,8 @@
warnings.simplefilter('ignore')
import pygit2
+from six import iteritems
+
import reclass.errors
from reclass.storage import NodeStorageBase
from reclass.storage.common import NameMangler
@@ -180,7 +182,7 @@
def nodes(self, branch, subdir):
ret = {}
- for name, file in self.files[branch].iteritems():
+ for (name, file) in iteritems(self.files[branch]):
if subdir is None or name.startswith(subdir):
node_name = os.path.splitext(file.name)[0]
if node_name in ret:
@@ -209,7 +211,7 @@
self._classes_uri = []
if 'env_overrides' in classes_uri:
for override in classes_uri['env_overrides']:
- for env, options in override.iteritems():
+ for (env, options) in iteritems(override):
uri = GitURI(self._classes_default_uri)
uri.update({ 'branch': env })
uri.update(options)
diff --git a/reclass/storage/yamldata.py b/reclass/storage/yamldata.py
index 0dda2b7..ac3dcb9 100644
--- a/reclass/storage/yamldata.py
+++ b/reclass/storage/yamldata.py
@@ -22,11 +22,10 @@
if not os.access(abs_path, os.R_OK):
raise NotFoundError('Cannot open: %s' % abs_path)
y = cls('yaml_fs://{0}'.format(abs_path))
- fp = file(abs_path)
- data = yaml.safe_load(fp)
- if data is not None:
- y._data = data
- fp.close()
+ with open(abs_path) as fp:
+ data = yaml.safe_load(fp)
+ if data is not None:
+ y._data = data
return y
@classmethod
diff --git a/reclass/utils/dictpath.py b/reclass/utils/dictpath.py
index aec6722..dfb8b32 100644
--- a/reclass/utils/dictpath.py
+++ b/reclass/utils/dictpath.py
@@ -7,7 +7,8 @@
# Released under the terms of the Artistic Licence 2.0
#
-import types, re
+import six
+import re
class DictPath(object):
'''
@@ -61,7 +62,7 @@
else:
if isinstance(contents, list):
self._parts = contents
- elif isinstance(contents, types.StringTypes):
+ elif isinstance(contents, six.string_types):
self._parts = self._split_string(contents)
elif isinstance(contents, tuple):
self._parts = list(contents)
@@ -76,7 +77,7 @@
return self._delim.join(str(i) for i in self._parts)
def __eq__(self, other):
- if isinstance(other, types.StringTypes):
+ if isinstance(other, six.string_types):
other = DictPath(self._delim, other)
return self._parts == other._parts \
diff --git a/reclass/values/compitem.py b/reclass/values/compitem.py
index 2134ea8..765b323 100644
--- a/reclass/values/compitem.py
+++ b/reclass/values/compitem.py
@@ -5,7 +5,7 @@
#
from reclass.settings import Settings
-from item import Item
+from .item import Item
class CompItem(Item):
diff --git a/reclass/values/dictitem.py b/reclass/values/dictitem.py
index d778fe2..555bd8f 100644
--- a/reclass/values/dictitem.py
+++ b/reclass/values/dictitem.py
@@ -5,7 +5,7 @@
#
from reclass.settings import Settings
-from item import Item
+from .item import Item
class DictItem(Item):
diff --git a/reclass/values/invitem.py b/reclass/values/invitem.py
index 84ea39d..970321b 100644
--- a/reclass/values/invitem.py
+++ b/reclass/values/invitem.py
@@ -7,7 +7,9 @@
import copy
import pyparsing as pp
-from item import Item
+from six import iteritems
+
+from .item import Item
from reclass.settings import Settings
from reclass.utils.dictpath import DictPath
from reclass.errors import ExpressionError, ParseError, ResolveError
@@ -301,7 +303,7 @@
def _value_expression(self, inventory):
results = {}
- for node, items in inventory.iteritems():
+ for (node, items) in iteritems(inventory):
if self._value_path.exists_in(items):
results[node] = copy.deepcopy(self._resolve(self._value_path, items))
return results
@@ -311,14 +313,14 @@
ExpressionError('Failed to render %s' % str(self), tbFlag=False)
results = {}
- for node, items in inventory.iteritems():
+ for (node, items) in iteritems(inventory):
if self._question.value(context, items) and self._value_path.exists_in(items):
results[node] = copy.deepcopy(self._resolve(self._value_path, items))
return results
def _list_test_expression(self, context, inventory):
results = []
- for node, items in inventory.iteritems():
+ for (node, items) in iteritems(inventory):
if self._question.value(context, items):
results.append(node)
return results
diff --git a/reclass/values/listitem.py b/reclass/values/listitem.py
index c7f29d0..1829e32 100644
--- a/reclass/values/listitem.py
+++ b/reclass/values/listitem.py
@@ -4,7 +4,7 @@
# This file is part of reclass
#
-from item import Item
+from .item import Item
from reclass.settings import Settings
class ListItem(Item):
diff --git a/reclass/values/parser.py b/reclass/values/parser.py
index bdd881d..a8adcf0 100644
--- a/reclass/values/parser.py
+++ b/reclass/values/parser.py
@@ -6,10 +6,10 @@
import pyparsing as pp
-from compitem import CompItem
-from invitem import InvItem
-from refitem import RefItem
-from scaitem import ScaItem
+from .compitem import CompItem
+from .invitem import InvItem
+from .refitem import RefItem
+from .scaitem import ScaItem
from reclass.errors import ParseError
from reclass.values.parser_funcs import STR, REF, INV
diff --git a/reclass/values/refitem.py b/reclass/values/refitem.py
index 0ae65e6..3e3341c 100644
--- a/reclass/values/refitem.py
+++ b/reclass/values/refitem.py
@@ -4,7 +4,7 @@
# This file is part of reclass
#
-from item import Item
+from .item import Item
from reclass.defaults import REFERENCE_SENTINELS
from reclass.settings import Settings
from reclass.utils.dictpath import DictPath
diff --git a/reclass/values/scaitem.py b/reclass/values/scaitem.py
index 6bd65dc..9de5681 100644
--- a/reclass/values/scaitem.py
+++ b/reclass/values/scaitem.py
@@ -5,7 +5,7 @@
#
from reclass.settings import Settings
-from item import Item
+from .item import Item
class ScaItem(Item):
diff --git a/reclass/values/value.py b/reclass/values/value.py
index 4ec6051..74ef272 100644
--- a/reclass/values/value.py
+++ b/reclass/values/value.py
@@ -4,10 +4,10 @@
# This file is part of reclass
#
-from parser import Parser
-from dictitem import DictItem
-from listitem import ListItem
-from scaitem import ScaItem
+from .parser import Parser
+from .dictitem import DictItem
+from .listitem import ListItem
+from .scaitem import ScaItem
from reclass.errors import InterpolationError
class Value(object):
diff --git a/reclass/values/valuelist.py b/reclass/values/valuelist.py
index 46d8ec7..fc0eaad 100644
--- a/reclass/values/valuelist.py
+++ b/reclass/values/valuelist.py
@@ -4,6 +4,8 @@
# This file is part of reclass
#
+from __future__ import print_function
+
import copy
import sys
@@ -98,7 +100,7 @@
if self._settings.ignore_overwritten_missing_references and not isinstance(output, (dict, list)) and n != (len(self._values)-1):
new = None
last_error = e
- print >>sys.stderr, "[WARNING] Reference '%s' undefined" % (str(value))
+ print("[WARNING] Reference '%s' undefined" % str(value), file=sys.stderr)
else:
raise e
diff --git a/requirements.txt b/requirements.txt
index ea72e95..66f0f4b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
pyparsing
pyyaml
pygit2
+six
diff --git a/setup.py b/setup.py
index 2fb77ae..401236e 100644
--- a/setup.py
+++ b/setup.py
@@ -38,7 +38,7 @@
url = URL,
packages = find_packages(exclude=['*tests']), #FIXME validate this
entry_point = { 'console_scripts': console_scripts },
- install_requires = ['pyparsing', 'pyyaml'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
+ install_requires = ['pyparsing', 'pyyaml', 'six'], #FIXME pygit2 (require libffi-dev, libgit2-dev 0.26.x )
classifiers=[
'Development Status :: 4 - Beta',