Factor out all reclass logic from storage classes
Signed-off-by: martin f. krafft <madduck@madduck.net>
diff --git a/reclass/__init__.py b/reclass/__init__.py
index 8bb06e1..bfad8e9 100644
--- a/reclass/__init__.py
+++ b/reclass/__init__.py
@@ -10,24 +10,9 @@
from output import OutputLoader
from storage.loader import StorageBackendLoader
-def get_storage(storage_type, nodes_uri, classes_uri, class_mappings):
+def get_storage(storage_type, nodes_uri, classes_uri):
storage_class = StorageBackendLoader(storage_type).load()
- return storage_class(nodes_uri, classes_uri, class_mappings)
-
-
-def get_nodeinfo(storage_type, inventory_base_uri, nodes_uri, classes_uri,
- nodename, class_mappings):
- storage = get_storage(storage_type, nodes_uri, classes_uri,
- class_mappings)
- # TODO: template interpolation
- return storage.nodeinfo(nodename)
-
-
-def get_inventory(storage_type, inventory_base_uri, nodes_uri, classes_uri,
- class_mappings):
- storage = get_storage(storage_type, nodes_uri, classes_uri,
- class_mappings)
- return storage.inventory()
+ return storage_class(nodes_uri, classes_uri)
def output(data, fmt, pretty_print=False):
diff --git a/reclass/adapters/ansible.py b/reclass/adapters/ansible.py
index 74861d3..cbf5f17 100755
--- a/reclass/adapters/ansible.py
+++ b/reclass/adapters/ansible.py
@@ -13,7 +13,8 @@
import os, sys, posix, optparse
-from reclass import get_nodeinfo, get_inventory, output
+from reclass import get_storage, output
+from reclass.core import Core
from reclass.errors import ReclassException
from reclass.config import find_and_read_configfile, get_options
from reclass.version import *
@@ -52,13 +53,14 @@
nodeinfo_help='output host_vars for the given host',
add_options_cb=add_ansible_options_group,
defaults=defaults)
+
+ storage = get_storage(options.storage_type, options.nodes_uri,
+ options.classes_uri)
class_mappings = defaults.get('class_mappings')
+ reclass = Core(storage, class_mappings)
if options.mode == MODE_NODEINFO:
- data = get_nodeinfo(options.storage_type,
- options.inventory_base_uri, options.nodes_uri,
- options.classes_uri, options.hostname,
- class_mappings)
+ data = reclass.nodeinfo(options.hostname)
# Massage and shift the data like Ansible wants it
data['parameters']['__reclass__'] = data['__reclass__']
for i in ('classes', 'applications'):
@@ -66,10 +68,7 @@
data = data['parameters']
else:
- data = get_inventory(options.storage_type,
- options.inventory_base_uri,
- options.nodes_uri, options.classes_uri,
- class_mappings)
+ data = reclass.inventory()
# Ansible inventory is only the list of groups. Groups are the set
# of classes plus the set of applications with the postfix added:
groups = data['classes']
diff --git a/reclass/adapters/salt.py b/reclass/adapters/salt.py
index 5a8a772..6fd7224 100755
--- a/reclass/adapters/salt.py
+++ b/reclass/adapters/salt.py
@@ -9,9 +9,11 @@
import os, sys, posix
-from reclass import get_nodeinfo, get_inventory, output
+from reclass import get_storage, output
+from reclass.core import Core
from reclass.errors import ReclassException
-from reclass.config import find_and_read_configfile, get_options
+from reclass.config import find_and_read_configfile, get_options, \
+ path_mangler
from reclass.constants import MODE_NODEINFO
from reclass.defaults import *
from reclass.version import *
@@ -23,8 +25,12 @@
classes_uri=OPT_CLASSES_URI,
class_mappings=None):
- data = get_nodeinfo(storage_type, inventory_base_uri, nodes_uri,
- classes_uri, minion_id, class_mappings)
+ nodes_uri, classes_uri = path_mangler(inventory_base_uri,
+ nodes_uri, classes_uri)
+ storage = get_storage(storage_type, nodes_uri, classes_uri)
+ reclass = Core(storage, class_mappings)
+
+ data = reclass.nodeinfo(minion_id)
params = data.get('parameters', {})
params['__reclass__'] = {}
params['__reclass__']['applications'] = data['applications']
@@ -39,19 +45,22 @@
env = 'base'
# TODO: node environments
+ nodes_uri, classes_uri = path_mangler(inventory_base_uri,
+ nodes_uri, classes_uri)
+ storage = get_storage(storage_type, nodes_uri, classes_uri)
+ reclass = Core(storage, class_mappings)
# if the minion_id is not None, then return just the applications for the
# specific minion, otherwise return the entire top data (which we need for
# CLI invocations of the adapter):
if minion_id is not None:
- data = get_nodeinfo(storage_type, inventory_base_uri, nodes_uri,
- classes_uri, minion_id, class_mappings)
+ data = reclass.nodeinfo(storage_type, inventory_base_uri, nodes_uri,
+ classes_uri, minion_id, class_mappings)
applications = data.get('applications', [])
return {env: applications}
else:
- data = get_inventory(storage_type, inventory_base_uri, nodes_uri,
- classes_uri, class_mappings)
+ data = reclass.inventory()
nodes = {}
for node_id, node_data in data['nodes'].iteritems():
nodes[node_id] = node_data['applications']
diff --git a/reclass/cli.py b/reclass/cli.py
index 19e943c..a07404a 100644
--- a/reclass/cli.py
+++ b/reclass/cli.py
@@ -9,7 +9,8 @@
import sys, os, posix
-from reclass import get_nodeinfo, get_inventory, output
+from reclass import get_storage, output
+from reclass.core import Core
from reclass.config import find_and_read_configfile, get_options
from reclass.errors import ReclassException
from reclass.defaults import *
@@ -24,17 +25,17 @@
defaults.update(find_and_read_configfile())
options = get_options(RECLASS_NAME, VERSION, DESCRIPTION,
defaults=defaults)
+
+ storage = get_storage(options.storage_type, options.nodes_uri,
+ options.classes_uri)
class_mappings = defaults.get('class_mappings')
+ reclass = Core(storage, class_mappings)
+
if options.mode == MODE_NODEINFO:
- data = get_nodeinfo(options.storage_type,
- options.inventory_base_uri, options.nodes_uri,
- options.classes_uri, options.nodename,
- class_mappings)
+ data = reclass.nodeinfo(options.nodename)
+
else:
- data = get_inventory(options.storage_type,
- options.inventory_base_uri,
- options.nodes_uri, options.classes_uri,
- class_mappings)
+ data = reclass.inventory()
print output(data, options.output, options.pretty_print)
diff --git a/reclass/core.py b/reclass/core.py
new file mode 100644
index 0000000..07b2c3a
--- /dev/null
+++ b/reclass/core.py
@@ -0,0 +1,153 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–14 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+
+import time
+#import types
+import re
+#import sys
+import fnmatch
+import shlex
+from reclass.datatypes import Entity, Classes
+from reclass.errors import MappingFormatError, ClassNotFound
+
+class Core(object):
+
+ def __init__(self, storage, class_mappings):
+ self._storage = storage
+ self._class_mappings = class_mappings
+
+ @staticmethod
+ def _get_timestamp():
+ return time.strftime('%c')
+
+ @staticmethod
+ def _match_regexp(key, nodename):
+ return re.search(key, nodename)
+
+ @staticmethod
+ def _match_glob(key, nodename):
+ return fnmatch.fnmatchcase(nodename, key)
+
+ @staticmethod
+ def _shlex_split(instr):
+ lexer = shlex.shlex(instr, posix=True)
+ lexer.whitespace_split = True
+ lexer.commenters = ''
+ regexp = False
+ if instr[0] == '/':
+ lexer.quotes += '/'
+ lexer.escapedquotes += '/'
+ regexp = True
+ try:
+ key = lexer.get_token()
+ except ValueError, e:
+ raise MappingFormatError('Error in mapping "{0}": missing closing '
+ 'quote (or slash)'.format(instr))
+ if regexp:
+ key = '/{0}/'.format(key)
+ return key, list(lexer)
+
+ def _populate_with_class_mappings(self, nodename):
+ if not self._class_mappings:
+ return Entity(name='empty')
+ c = Classes()
+ for mapping in self._class_mappings:
+ matched = False
+ key, klasses = Core._shlex_split(mapping)
+ if key[0] == ('/'):
+ matched = Core._match_regexp(key[1:-1], nodename)
+ if matched:
+ for klass in klasses:
+ c.append_if_new(matched.expand(klass))
+
+ else:
+ if Core._match_glob(key, nodename):
+ for klass in klasses:
+ c.append_if_new(klass)
+
+ return Entity(classes=c,
+ name='class mappings for node {0}'.format(nodename))
+
+ def _recurse_entity(self, entity, merge_base=None, seen=None, nodename=None):
+ if seen is None:
+ seen = {}
+
+ if merge_base is None:
+ merge_base = Entity(name='empty (@{0})'.format(nodename))
+
+ for klass in entity.classes.as_list():
+ if klass not in seen:
+ try:
+ class_entity = self._storage.get_class(klass)
+ except ClassNotFound, e:
+ e.set_nodename(nodename)
+ raise e
+
+ descent = self._recurse_entity(class_entity, seen=seen,
+ nodename=nodename)
+ # on every iteration, we merge the result of the recursive
+ # descent into what we have so far…
+ merge_base.merge(descent)
+ seen[klass] = True
+
+ # … and finally, we merge what we have at this level into the
+ # result of the iteration, so that elements at the current level
+ # overwrite stuff defined by parents
+ merge_base.merge(entity)
+ return merge_base
+
+ def _nodeinfo(self, nodename):
+ node_entity = self._storage.get_node(nodename)
+ base_entity = self._populate_with_class_mappings(node_entity.name)
+ seen = {}
+ merge_base = self._recurse_entity(base_entity, seen=seen,
+ nodename=base_entity.name)
+ ret = self._recurse_entity(node_entity, merge_base, seen=seen,
+ nodename=node_entity.name)
+ ret.interpolate()
+ return ret
+
+ def _nodeinfo_as_dict(self, nodename, entity):
+ ret = {'__reclass__' : {'node': entity.name, 'name': nodename,
+ 'uri': entity.uri,
+ 'timestamp': Core._get_timestamp()
+ },
+ }
+ ret.update(entity.as_dict())
+ return ret
+
+ def nodeinfo(self, nodename):
+ return self._nodeinfo_as_dict(nodename, self._nodeinfo(nodename))
+
+ def inventory(self):
+ entities = {}
+ for n in self._storage.enumerate_nodes():
+ entities[n] = self._nodeinfo(n)
+
+ nodes = {}
+ applications = {}
+ classes = {}
+ for f, nodeinfo in entities.iteritems():
+ d = nodes[f] = self._nodeinfo_as_dict(f, nodeinfo)
+ for a in d['applications']:
+ if a in applications:
+ applications[a].append(f)
+ else:
+ applications[a] = [f]
+ for c in d['classes']:
+ if c in classes:
+ classes[c].append(f)
+ else:
+ classes[c] = [f]
+
+ return {'__reclass__' : {'timestamp': Core._get_timestamp()},
+ 'nodes': nodes,
+ 'classes': classes,
+ 'applications': applications
+ }
diff --git a/reclass/storage/__init__.py b/reclass/storage/__init__.py
index 754c223..8ae2408 100644
--- a/reclass/storage/__init__.py
+++ b/reclass/storage/__init__.py
@@ -7,167 +7,21 @@
# Released under the terms of the Artistic Licence 2.0
#
-import time
-import types
-import re
-import sys
-import fnmatch
-import shlex
-from reclass.datatypes import Entity, Classes
-from reclass.errors import MappingFormatError
-
-def _get_timestamp():
- return time.strftime('%c')
-
-def vvv(msg):
- print >>sys.stderr, msg
- pass
-
class NodeStorageBase(object):
- def __init__(self, nodes_uri, classes_uri, class_mappings):
- self._nodes_uri = nodes_uri
- self._classes_uri = classes_uri
- self._classes_cache = {}
- self._class_mappings = class_mappings
+ def __init__(self, name):
+ self._name = name
- nodes_uri = property(lambda self: self._nodes_uri)
- classes_uri = property(lambda self: self._classes_uri)
- class_mappings = property(lambda self: self._class_mappings)
+ name = property(lambda self: self._name)
- def _match_regexp(self, key, nodename):
- return re.search(key, nodename)
+ def get_node(self, name, merge_base=None):
+ msg = "Storage class '{0}' does not implement node entity retrieval."
+ raise NotImplementedError(msg.format(self.name))
- def _match_glob(self, key, nodename):
- return fnmatch.fnmatchcase(nodename, key)
+ def get_class(self, name):
+ msg = "Storage class '{0}' does not implement class entity retrieval."
+ raise NotImplementedError(msg.format(self.name))
- def _shlex_split(self, instr):
- lexer = shlex.shlex(instr, posix=True)
- lexer.whitespace_split = True
- lexer.commenters = ''
- regexp = False
- if instr[0] == '/':
- lexer.quotes += '/'
- lexer.escapedquotes += '/'
- regexp = True
- try:
- key = lexer.get_token()
- except ValueError, e:
- raise MappingFormatError('Error in mapping "{0}": missing closing '
- 'quote (or slash)'.format(instr))
- if regexp:
- key = '/{0}/'.format(key)
- return key, list(lexer)
-
- def _populate_with_class_mappings(self, nodename):
- if not self.class_mappings:
- return Entity(name='empty')
- c = Classes()
- for mapping in self.class_mappings:
- matched = False
- key, klasses = self._shlex_split(mapping)
- if key[0] == ('/'):
- matched = self._match_regexp(key[1:-1], nodename)
- if matched:
- for klass in klasses:
- c.append_if_new(matched.expand(klass))
-
- else:
- if self._match_glob(key, nodename):
- for klass in klasses:
- c.append_if_new(klass)
-
- return Entity(classes=c,
- name='class mappings for node {0}'.format(nodename))
-
- def _get_storage_name(self):
- raise NotImplementedError, "Storage class does not have a name"
-
- def _get_node(self, name, merge_base=None):
- raise NotImplementedError, "Storage class not implement node entity retrieval"
-
- def _get_class(self, name):
- raise NotImplementedError, "Storage class not implement class entity retrieval"
-
- def _recurse_entity(self, entity, merge_base=None, seen=None, nodename=None):
- if seen is None:
- seen = {}
-
- if merge_base is None:
- merge_base = Entity(name='empty (@{0})'.format(nodename))
-
- for klass in entity.classes.as_list():
- if klass not in seen:
- try:
- class_entity = self._classes_cache[klass]
- except KeyError, e:
- class_entity = self._get_class(klass, nodename)
- self._classes_cache[klass] = class_entity
-
- descent = self._recurse_entity(class_entity, seen=seen,
- nodename=nodename)
- # on every iteration, we merge the result of the recursive
- # descent into what we have so far…
- merge_base.merge(descent)
- seen[klass] = True
-
- # … and finally, we merge what we have at this level into the
- # result of the iteration, so that elements at the current level
- # overwrite stuff defined by parents
- merge_base.merge(entity)
- return merge_base
-
- def _nodeinfo(self, nodename):
- node_entity = self._get_node(nodename)
- base_entity = self._populate_with_class_mappings(node_entity.name)
- seen = {}
- merge_base = self._recurse_entity(base_entity, seen=seen,
- nodename=base_entity.name)
- ret = self._recurse_entity(node_entity, merge_base, seen=seen,
- nodename=node_entity.name)
- ret.interpolate()
- return ret
-
- def _nodeinfo_as_dict(self, nodename, entity):
- ret = {'__reclass__' : {'node': entity.name, 'name': nodename,
- 'uri': entity.uri,
- 'timestamp': _get_timestamp()
- },
- }
- ret.update(entity.as_dict())
- return ret
-
- def nodeinfo(self, nodename):
- return self._nodeinfo_as_dict(nodename, self._nodeinfo(nodename))
-
- def _list_inventory(self):
- raise NotImplementedError, "Storage class does not implement inventory listing"
-
- def inventory(self):
- entities = self._list_inventory()
-
- nodes = {}
- applications = {}
- classes = {}
- for f, nodeinfo in entities.iteritems():
- d = nodes[f] = self._nodeinfo_as_dict(f, nodeinfo)
- for a in d['applications']:
- if a in applications:
- applications[a].append(f)
- else:
- applications[a] = [f]
- for c in d['classes']:
- if c in classes:
- classes[c].append(f)
- else:
- classes[c] = [f]
-
- return {'__reclass__' : {'timestamp': _get_timestamp()},
- 'nodes': nodes,
- 'classes': classes,
- 'applications': applications
- }
-
-
-
-
+ def enumerate_nodes(self):
+ msg = "Storage class '{0}' does not implement node enumeration."
+ raise NotImplementedError(msg.format(self.name))
diff --git a/reclass/storage/yaml_fs/__init__.py b/reclass/storage/yaml_fs/__init__.py
index 740d230..c163f15 100644
--- a/reclass/storage/yaml_fs/__init__.py
+++ b/reclass/storage/yaml_fs/__init__.py
@@ -15,6 +15,7 @@
import reclass.errors
FILE_EXTENSION = '.yml'
+STORAGE_NAME = 'yaml_fs'
def vvv(msg):
#print >>sys.stderr, msg
@@ -22,19 +23,20 @@
class ExternalNodeStorage(NodeStorageBase):
- def __init__(self, nodes_uri, classes_uri, class_mappings):
- super(ExternalNodeStorage, self).__init__(nodes_uri, classes_uri,
- class_mappings)
+ def __init__(self, nodes_uri, classes_uri):
+ super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
def _handle_node_duplicates(name, uri1, uri2):
raise reclass.errors.DuplicateNodeNameError(self._get_storage_name(),
name, uri1, uri2)
+ self._nodes_uri = nodes_uri
self._nodes = self._enumerate_inventory(nodes_uri,
duplicate_handler=_handle_node_duplicates)
+ self._classes_uri = classes_uri
self._classes = self._enumerate_inventory(classes_uri)
- def _get_storage_name(self):
- return 'yaml_fs'
+ nodes_uri = property(lambda self: self._nodes_uri)
+ classes_uri = property(lambda self: self._classes_uri)
def _enumerate_inventory(self, basedir, duplicate_handler=None):
ret = {}
@@ -52,31 +54,25 @@
d.walk(register_fn)
return ret
- def _get_node(self, name):
+ def get_node(self, name):
vvv('GET NODE {0}'.format(name))
try:
relpath = self._nodes[name]
path = os.path.join(self.nodes_uri, relpath)
name = os.path.splitext(relpath)[0]
except KeyError, e:
- raise reclass.errors.NodeNotFound(self._get_storage_name(),
- name, self.nodes_uri)
+ raise reclass.errors.NodeNotFound(self.name, name, self.nodes_uri)
entity = YamlFile(path).get_entity(name)
return entity
- def _get_class(self, name, nodename=None):
+ def get_class(self, name, nodename=None):
vvv('GET CLASS {0}'.format(name))
try:
path = os.path.join(self.classes_uri, self._classes[name])
except KeyError, e:
- raise reclass.errors.ClassNotFound(self._get_storage_name(),
- name, self.classes_uri,
- nodename)
+ raise reclass.errors.ClassNotFound(self.name, name, self.classes_uri)
entity = YamlFile(path).get_entity()
return entity
- def _list_inventory(self):
- entities = {}
- for n in self._nodes.iterkeys():
- entities[n] = self._nodeinfo(n)
- return entities
+ def enumerate_nodes(self):
+ return self._nodes.keys()