inital support for a git storage type
diff --git a/reclass/__init__.py b/reclass/__init__.py
index c86b880..83b962f 100644
--- a/reclass/__init__.py
+++ b/reclass/__init__.py
@@ -15,6 +15,8 @@
storage_class = StorageBackendLoader(storage_type).load()
return MemcacheProxy(storage_class(nodes_uri, classes_uri, **kwargs))
+def get_path_mangler(storage_type,**kwargs):
+ return StorageBackendLoader(storage_type).path_mangler()
def output(data, fmt, pretty_print=False, no_refs=False):
output_class = OutputLoader(fmt).load()
diff --git a/reclass/config.py b/reclass/config.py
index 0f3b023..6043b41 100644
--- a/reclass/config.py
+++ b/reclass/config.py
@@ -11,7 +11,8 @@
import errors
from defaults import *
-from constants import MODE_NODEINFO, MODE_INVENTORY
+from constants import MODE_NODEINFO, MODE_INVENTORY
+from reclass import get_path_mangler
def make_db_options_group(parser, defaults={}):
ret = optparse.OptionGroup(parser, 'Database options',
@@ -131,30 +132,6 @@
return parser, option_checker
-def path_mangler(inventory_base_uri, nodes_uri, classes_uri):
-
- if inventory_base_uri is None:
- # if inventory_base is not given, default to current directory
- inventory_base_uri = os.getcwd()
-
- nodes_uri = nodes_uri or 'nodes'
- classes_uri = classes_uri or 'classes'
-
- def _path_mangler_inner(path):
- ret = os.path.join(inventory_base_uri, path)
- ret = os.path.expanduser(ret)
- return os.path.abspath(ret)
-
- n, c = map(_path_mangler_inner, (nodes_uri, classes_uri))
- if n == c:
- raise errors.DuplicateUriError(n, c)
- common = os.path.commonprefix((n, c))
- if common == n or common == c:
- raise errors.UriOverlapError(n, c)
-
- return n, c
-
-
def get_options(name, version, description,
inventory_shortopt='-i',
inventory_longopt='--inventory',
@@ -178,8 +155,8 @@
options, args = parser.parse_args()
checker(options, args)
- options.nodes_uri, options.classes_uri = \
- path_mangler(options.inventory_base_uri, options.nodes_uri, options.classes_uri)
+ path_mangler = get_path_mangler(options.storage_type)
+ options.nodes_uri, options.classes_uri = path_mangler(options.inventory_base_uri, options.nodes_uri, options.classes_uri)
return options
diff --git a/reclass/storage/__init__.py b/reclass/storage/__init__.py
index 001fdce..3990b91 100644
--- a/reclass/storage/__init__.py
+++ b/reclass/storage/__init__.py
@@ -22,14 +22,10 @@
msg = "Storage class '{0}' does not implement class entity retrieval."
raise NotImplementedError(msg.format(self.name))
- def get_exports(self):
- msg = "Storage class '{0}' does not implement get_exports."
- raise NotImplementedError(msg.format(self.name))
-
- def put_exports(self, new):
- msg = "Storage class '{0}' does not implement put_exports."
- raise NotImplementedError(msg.format(self.name))
-
def enumerate_nodes(self):
msg = "Storage class '{0}' does not implement node enumeration."
raise NotImplementedError(msg.format(self.name))
+
+ def path_mangler(self):
+ msg = "Storage class '{0}' does not implement path_mangler."
+ raise NotImplementedError(msg.format(self.name))
diff --git a/reclass/storage/common.py b/reclass/storage/common.py
new file mode 100644
index 0000000..6a77fc8
--- /dev/null
+++ b/reclass/storage/common.py
@@ -0,0 +1,22 @@
+import os
+
+class NameMangler:
+ @staticmethod
+ def nodes(relpath, name):
+ # nodes are identified just by their basename, so
+ # no mangling required
+ return relpath, name
+
+ @staticmethod
+ def classes(relpath, name):
+ if relpath == '.' or relpath == '':
+ # './' is converted to None
+ return None, name
+ parts = relpath.split(os.path.sep)
+ if name != 'init':
+ # "init" is the directory index, so only append the basename
+ # to the path parts for all other filenames. This has the
+ # effect that data in file "foo/init.yml" will be registered
+ # as data for class "foo", not "foo.init"
+ parts.append(name)
+ return relpath, '.'.join(parts)
diff --git a/reclass/storage/git_fs/__init__.py b/reclass/storage/git_fs/__init__.py
new file mode 100644
index 0000000..3c2ad07
--- /dev/null
+++ b/reclass/storage/git_fs/__init__.py
@@ -0,0 +1,109 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+
+import collections
+import fnmatch
+import os
+import pygit2
+
+import reclass.errors
+from reclass.storage import NodeStorageBase
+from reclass.storage.common import NameMangler
+from reclass.storage.yamldata import YamlData
+
+FILE_EXTENSION = '.yml'
+STORAGE_NAME = 'git_fs'
+
+def path_mangler(inventory_base_uri, nodes_uri, classes_uri):
+ if nodes_uri == classes_uri:
+ raise errors.DuplicateUriError(nodes_uri, classes_uri)
+ return nodes_uri, classes_uri
+
+
+def list_files_in_branch(repo, branch):
+ def _files_in_tree(tree, path):
+ files = []
+ for entry in tree:
+ if entry.filemode == pygit2.GIT_FILEMODE_TREE:
+ subtree = repo.get(entry.id)
+ if path == '':
+ subpath = entry.name
+ else:
+ subpath = '/'.join([path, entry.name])
+ files.extend(_files_in_tree(subtree, subpath))
+ else:
+ if path == '':
+ relpath = entry.name
+ else:
+ relpath = '/'.join([path, entry.name])
+ files.append(GitMD(entry.name, relpath, entry.id))
+ return files
+
+ tree = repo.revparse_single('master').tree
+ return _files_in_tree(tree, '')
+
+
+GitMD = collections.namedtuple("GitMD", ["name", "path", "id"], verbose=False, rename=False)
+
+
+class ExternalNodeStorage(NodeStorageBase):
+
+ def __init__(self, nodes_uri, classes_uri, default_environment=None):
+ super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
+
+ self._nodes_uri = nodes_uri
+ self._nodes_repo = pygit2.Repository(self._nodes_uri)
+ self._nodes = self._enumerate_nodes()
+
+ self._classes_uri = classes_uri
+ self._classes_repo = pygit2.Repository(self._classes_uri)
+ self._classes_branches = self._classes_repo.listall_branches()
+ self._classes = self._enumerate_classes()
+ self._default_environment = default_environment
+
+ nodes_uri = property(lambda self: self._nodes_uri)
+ classes_uri = property(lambda self: self._classes_uri)
+
+ def get_node(self, name):
+ blob = self._nodes_repo.get(self._nodes[name].id)
+ entity = YamlData.from_string(blob.data, 'git_fs://{0}:master/{1}'.format(self._nodes_uri, self._nodes[name].path)).get_entity(name, self._default_environment)
+ return entity
+
+ def get_class(self, name, nodename=None, branch='master'):
+ blob = self._classes_repo.get(self._classes[branch][name].id)
+ entity = YamlData.from_string(blob.data, 'git_fs://{0}:{1}/{2}'.format(self._classes_uri, branch, self._classes[branch][name].path)).get_entity(name, self._default_environment)
+ return entity
+
+ def enumerate_nodes(self):
+ return self._nodes.keys()
+
+ def _enumerate_nodes(self):
+ ret = {}
+ files = list_files_in_branch(self._nodes_repo, 'master')
+ for file in files:
+ if fnmatch.fnmatch(file.name, '*{0}'.format(FILE_EXTENSION)):
+ name = os.path.splitext(file.name)[0]
+ if name in ret:
+ raise reclass.errors.DuplicateNodeNameError(self.name, name, ret[name], path)
+ else:
+ ret[name] = file
+ return ret
+
+ def _enumerate_classes(self):
+ ret = {}
+ for bname in self._classes_branches:
+ branch = {}
+ files = list_files_in_branch(self._classes_repo, bname)
+ for file in files:
+ if fnmatch.fnmatch(file.name, '*{0}'.format(FILE_EXTENSION)):
+ name = os.path.splitext(file.name)[0]
+ relpath = os.path.dirname(file.path)
+ relpath, name = NameMangler.classes(relpath, name)
+ if name in ret:
+ raise reclass.errors.DuplicateNodeNameError(self.name, name, ret[name], path)
+ else:
+ branch[name] = file
+ ret[bname] = branch
+ return ret
diff --git a/reclass/storage/loader.py b/reclass/storage/loader.py
index 399e7fd..77fdecb 100644
--- a/reclass/storage/loader.py
+++ b/reclass/storage/loader.py
@@ -23,3 +23,9 @@
'"{1}"'.format(self._name, klassname))
return klass
+
+ def path_mangler(self, name='path_mangler'):
+ function = getattr(self._module, name, None)
+ if function is None:
+ raise AttributeError('Storage backend class {0} does not export "{1}"'.format(self._name, name))
+ return function
diff --git a/reclass/storage/yaml_fs/__init__.py b/reclass/storage/yaml_fs/__init__.py
index 19b2b8f..c9de29c 100644
--- a/reclass/storage/yaml_fs/__init__.py
+++ b/reclass/storage/yaml_fs/__init__.py
@@ -11,7 +11,8 @@
import yaml
from reclass.output.yaml_outputter import ExplicitDumper
from reclass.storage import NodeStorageBase
-from yamlfile import YamlFile
+from reclass.storage.common import NameMangler
+from reclass.storage.yamldata import YamlData
from directory import Directory
from reclass.datatypes import Entity
import reclass.errors
@@ -23,32 +24,40 @@
#print >>sys.stderr, msg
pass
+def path_mangler(inventory_base_uri, nodes_uri, classes_uri):
+
+ if inventory_base_uri is None:
+ # if inventory_base is not given, default to current directory
+ inventory_base_uri = os.getcwd()
+
+ nodes_uri = nodes_uri or 'nodes'
+ classes_uri = classes_uri or 'classes'
+
+ def _path_mangler_inner(path):
+ ret = os.path.join(inventory_base_uri, path)
+ ret = os.path.expanduser(ret)
+ return os.path.abspath(ret)
+
+ n, c = map(_path_mangler_inner, (nodes_uri, classes_uri))
+ if n == c:
+ raise errors.DuplicateUriError(n, c)
+ common = os.path.commonprefix((n, c))
+ if common == n or common == c:
+ raise errors.UriOverlapError(n, c)
+
+ return n, c
+
+
class ExternalNodeStorage(NodeStorageBase):
def __init__(self, nodes_uri, classes_uri, default_environment=None):
super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
- def name_mangler(relpath, name):
- # nodes are identified just by their basename, so
- # no mangling required
- return relpath, name
self._nodes_uri = nodes_uri
- self._nodes = self._enumerate_inventory(nodes_uri, name_mangler)
+ self._nodes = self._enumerate_inventory(nodes_uri, NameMangler.nodes)
- def name_mangler(relpath, name):
- if relpath == '.':
- # './' is converted to None
- return None, name
- parts = relpath.split(os.path.sep)
- if name != 'init':
- # "init" is the directory index, so only append the basename
- # to the path parts for all other filenames. This has the
- # effect that data in file "foo/init.yml" will be registered
- # as data for class "foo", not "foo.init"
- parts.append(name)
- return relpath, '.'.join(parts)
self._classes_uri = classes_uri
- self._classes = self._enumerate_inventory(classes_uri, name_mangler)
+ self._classes = self._enumerate_inventory(classes_uri, NameMangler.classes)
self._default_environment = default_environment
nodes_uri = property(lambda self: self._nodes_uri)
@@ -85,7 +94,7 @@
name = os.path.splitext(relpath)[0]
except KeyError, e:
raise reclass.errors.NodeNotFound(self.name, name, self.nodes_uri)
- entity = YamlFile(path).get_entity(name, self._default_environment)
+ entity = YamlData.from_file(path).get_entity(name, self._default_environment)
return entity
def get_class(self, name, nodename=None):
@@ -94,7 +103,7 @@
path = os.path.join(self.classes_uri, self._classes[name])
except KeyError, e:
raise reclass.errors.ClassNotFound(self.name, name, self.classes_uri)
- entity = YamlFile(path).get_entity(name)
+ entity = YamlData.from_file(path).get_entity(name)
return entity
def enumerate_nodes(self):
diff --git a/reclass/storage/yaml_fs/yamlfile.py b/reclass/storage/yamldata.py
similarity index 72%
rename from reclass/storage/yaml_fs/yamlfile.py
rename to reclass/storage/yamldata.py
index ad262cd..8d8363a 100644
--- a/reclass/storage/yaml_fs/yamlfile.py
+++ b/reclass/storage/yamldata.py
@@ -11,25 +11,37 @@
import os
from reclass.errors import NotFoundError
-class YamlFile(object):
+class YamlData(object):
- def __init__(self, path):
- ''' Initialise a yamlfile object '''
+ @classmethod
+ def from_file(cls, path):
+ ''' Initialise yaml data from a local file '''
if not os.path.isfile(path):
raise NotFoundError('No such file: %s' % path)
if not os.access(path, os.R_OK):
raise NotFoundError('Cannot open: %s' % path)
- self._path = path
- self._data = dict()
- self._read()
- path = property(lambda self: self._path)
-
- def _read(self):
- fp = file(self._path)
+ y = cls('yaml_fs://{0}'.format(path))
+ fp = file(path)
data = yaml.safe_load(fp)
if data is not None:
- self._data = data
+ y._data = data
fp.close()
+ return y
+
+ @classmethod
+ def from_string(cls, string, uri):
+ ''' Initialise yaml data from a string '''
+ y = cls(uri)
+ data = yaml.safe_load(string)
+ if data is not None:
+ y._data = data
+ return y
+
+ def __init__(self, uri):
+ self._uri = uri
+ self._data = dict()
+
+ uri = property(lambda self: self._uri)
def get_data(self):
return self._data
@@ -61,8 +73,7 @@
env = self._data.get('environment', default_environment)
return datatypes.Entity(classes, applications, parameters, exports,
- name=name, environment=env,
- uri='yaml_fs://{0}'.format(self._path))
+ name=name, environment=env, uri=self.uri)
def __repr__(self):
return '<{0} {1}, {2}>'.format(self.__class__.__name__, self._path,