yaml_git storage type
diff --git a/reclass/adapters/ansible.py b/reclass/adapters/ansible.py
index f9d08a6..6794a0d 100755
--- a/reclass/adapters/ansible.py
+++ b/reclass/adapters/ansible.py
@@ -57,7 +57,7 @@
storage = get_storage(options.storage_type, options.nodes_uri, options.classes_uri)
class_mappings = defaults.get('class_mappings')
- reclass = Core(storage, class_mappings)
+ reclass = Core(storage, class_mappings, default_environment=None)
if options.mode == MODE_NODEINFO:
data = reclass.nodeinfo(options.hostname)
diff --git a/reclass/adapters/salt.py b/reclass/adapters/salt.py
index 32319a2..702eca2 100755
--- a/reclass/adapters/salt.py
+++ b/reclass/adapters/salt.py
@@ -27,11 +27,11 @@
propagate_pillar_data_to_reclass=False):
nodes_uri, classes_uri = path_mangler(inventory_base_uri, nodes_uri, classes_uri)
- storage = get_storage(storage_type, nodes_uri, classes_uri, default_environment='base')
+ storage = get_storage(storage_type, nodes_uri, classes_uri)
input_data = None
if propagate_pillar_data_to_reclass:
input_data = pillar
- reclass = Core(storage, class_mappings, input_data=input_data)
+ reclass = Core(storage, class_mappings, input_data=input_data, default_environment='base')
data = reclass.nodeinfo(minion_id)
params = data.get('parameters', {})
@@ -48,8 +48,8 @@
classes_uri=OPT_CLASSES_URI, class_mappings=None):
nodes_uri, classes_uri = path_mangler(inventory_base_uri, nodes_uri, classes_uri)
- storage = get_storage(storage_type, nodes_uri, classes_uri, default_environment='base')
- reclass = Core(storage, class_mappings, input_data=None)
+ storage = get_storage(storage_type, nodes_uri, classes_uri)
+ reclass = Core(storage, class_mappings, input_data=None, default_environment='base')
# if the minion_id is not None, then return just the applications for the
# specific minion, otherwise return the entire top data (which we need for
diff --git a/reclass/cli.py b/reclass/cli.py
index 386d351..48ce31f 100644
--- a/reclass/cli.py
+++ b/reclass/cli.py
@@ -24,14 +24,14 @@
'output' : OPT_OUTPUT
}
defaults.update(find_and_read_configfile())
+
options = get_options(RECLASS_NAME, VERSION, DESCRIPTION,
defaults=defaults)
- storage = get_storage(options.storage_type, options.nodes_uri,
- options.classes_uri, default_environment='base')
+ storage = get_storage(options.storage_type, options.nodes_uri, options.classes_uri)
class_mappings = defaults.get('class_mappings')
- reclass = Core(storage, class_mappings)
+ reclass = Core(storage, class_mappings, default_environment='base')
if options.mode == MODE_NODEINFO:
data = reclass.nodeinfo(options.nodename)
diff --git a/reclass/core.py b/reclass/core.py
index b696e8f..197654b 100644
--- a/reclass/core.py
+++ b/reclass/core.py
@@ -9,9 +9,7 @@
import copy
import time
-#import types
import re
-#import sys
import fnmatch
import shlex
import string
@@ -23,10 +21,11 @@
class Core(object):
- def __init__(self, storage, class_mappings, input_data=None):
+ def __init__(self, storage, class_mappings, input_data=None, default_environment=None):
self._storage = storage
self._class_mappings = class_mappings
self._input_data = input_data
+ self._default_environment = default_environment
@staticmethod
def _get_timestamp():
@@ -86,17 +85,20 @@
p = Parameters(self._input_data)
return Entity(parameters=p, name='input data')
- def _recurse_entity(self, entity, merge_base=None, seen=None, nodename=None):
+ def _recurse_entity(self, entity, merge_base=None, seen=None, nodename=None, environment=None):
if seen is None:
seen = {}
+ if environment is None:
+ environment = self._default_environment
+
if merge_base is None:
merge_base = Entity(name='empty (@{0})'.format(nodename))
for klass in entity.classes.as_list():
if klass not in seen:
try:
- class_entity = self._storage.get_class(klass)
+ class_entity = self._storage.get_class(klass, environment)
except ClassNotFound, e:
e.set_nodename(nodename)
raise e
@@ -130,15 +132,17 @@
def _node_entity(self, nodename):
node_entity = self._storage.get_node(nodename)
+ if node_entity.environment == None:
+ node_entity.environment = self._default_environment
base_entity = Entity(name='base')
base_entity.merge(self._get_class_mappings_entity(node_entity.name))
base_entity.merge(self._get_input_data_entity())
base_entity.merge_parameters(self._get_automatic_parameters(nodename))
seen = {}
- merge_base = self._recurse_entity(base_entity, seen=seen,
- nodename=base_entity.name)
- return self._recurse_entity(node_entity, merge_base, seen=seen,
- nodename=node_entity.name)
+ merge_base = self._recurse_entity(base_entity, seen=seen, nodename=base_entity.name,
+ environment=node_entity.environment)
+ return self._recurse_entity(node_entity, merge_base, seen=seen, nodename=node_entity.name,
+ environment=node_entity.environment)
def _nodeinfo(self, nodename, inventory):
ret = self._node_entity(nodename)
diff --git a/reclass/datatypes/entity.py b/reclass/datatypes/entity.py
index 706dd23..cbcada6 100644
--- a/reclass/datatypes/entity.py
+++ b/reclass/datatypes/entity.py
@@ -29,7 +29,7 @@
self._set_exports(exports)
self._uri = uri or ''
self._name = name or ''
- self._environment = environment or ''
+ self._environment = environment
name = property(lambda s: s._name)
short_name = property(lambda s: s._short_name)
@@ -71,7 +71,8 @@
self._exports.merge(other._exports)
self._name = other.name
self._uri = other.uri
- self._environment = other.environment
+ if other.environment != None:
+ self._environment = other.environment
def merge_parameters(self, params):
self._parameters.merge(params)
diff --git a/reclass/storage/git_fs/__init__.py b/reclass/storage/git_fs/__init__.py
deleted file mode 100644
index 3c2ad07..0000000
--- a/reclass/storage/git_fs/__init__.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#
-# -*- coding: utf-8 -*-
-#
-# This file is part of reclass
-
-import collections
-import fnmatch
-import os
-import pygit2
-
-import reclass.errors
-from reclass.storage import NodeStorageBase
-from reclass.storage.common import NameMangler
-from reclass.storage.yamldata import YamlData
-
-FILE_EXTENSION = '.yml'
-STORAGE_NAME = 'git_fs'
-
-def path_mangler(inventory_base_uri, nodes_uri, classes_uri):
- if nodes_uri == classes_uri:
- raise errors.DuplicateUriError(nodes_uri, classes_uri)
- return nodes_uri, classes_uri
-
-
-def list_files_in_branch(repo, branch):
- def _files_in_tree(tree, path):
- files = []
- for entry in tree:
- if entry.filemode == pygit2.GIT_FILEMODE_TREE:
- subtree = repo.get(entry.id)
- if path == '':
- subpath = entry.name
- else:
- subpath = '/'.join([path, entry.name])
- files.extend(_files_in_tree(subtree, subpath))
- else:
- if path == '':
- relpath = entry.name
- else:
- relpath = '/'.join([path, entry.name])
- files.append(GitMD(entry.name, relpath, entry.id))
- return files
-
- tree = repo.revparse_single('master').tree
- return _files_in_tree(tree, '')
-
-
-GitMD = collections.namedtuple("GitMD", ["name", "path", "id"], verbose=False, rename=False)
-
-
-class ExternalNodeStorage(NodeStorageBase):
-
- def __init__(self, nodes_uri, classes_uri, default_environment=None):
- super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
-
- self._nodes_uri = nodes_uri
- self._nodes_repo = pygit2.Repository(self._nodes_uri)
- self._nodes = self._enumerate_nodes()
-
- self._classes_uri = classes_uri
- self._classes_repo = pygit2.Repository(self._classes_uri)
- self._classes_branches = self._classes_repo.listall_branches()
- self._classes = self._enumerate_classes()
- self._default_environment = default_environment
-
- nodes_uri = property(lambda self: self._nodes_uri)
- classes_uri = property(lambda self: self._classes_uri)
-
- def get_node(self, name):
- blob = self._nodes_repo.get(self._nodes[name].id)
- entity = YamlData.from_string(blob.data, 'git_fs://{0}:master/{1}'.format(self._nodes_uri, self._nodes[name].path)).get_entity(name, self._default_environment)
- return entity
-
- def get_class(self, name, nodename=None, branch='master'):
- blob = self._classes_repo.get(self._classes[branch][name].id)
- entity = YamlData.from_string(blob.data, 'git_fs://{0}:{1}/{2}'.format(self._classes_uri, branch, self._classes[branch][name].path)).get_entity(name, self._default_environment)
- return entity
-
- def enumerate_nodes(self):
- return self._nodes.keys()
-
- def _enumerate_nodes(self):
- ret = {}
- files = list_files_in_branch(self._nodes_repo, 'master')
- for file in files:
- if fnmatch.fnmatch(file.name, '*{0}'.format(FILE_EXTENSION)):
- name = os.path.splitext(file.name)[0]
- if name in ret:
- raise reclass.errors.DuplicateNodeNameError(self.name, name, ret[name], path)
- else:
- ret[name] = file
- return ret
-
- def _enumerate_classes(self):
- ret = {}
- for bname in self._classes_branches:
- branch = {}
- files = list_files_in_branch(self._classes_repo, bname)
- for file in files:
- if fnmatch.fnmatch(file.name, '*{0}'.format(FILE_EXTENSION)):
- name = os.path.splitext(file.name)[0]
- relpath = os.path.dirname(file.path)
- relpath, name = NameMangler.classes(relpath, name)
- if name in ret:
- raise reclass.errors.DuplicateNodeNameError(self.name, name, ret[name], path)
- else:
- branch[name] = file
- ret[bname] = branch
- return ret
diff --git a/reclass/storage/memcache_proxy.py b/reclass/storage/memcache_proxy.py
index 545c68f..a709d40 100644
--- a/reclass/storage/memcache_proxy.py
+++ b/reclass/storage/memcache_proxy.py
@@ -57,9 +57,9 @@
return MemcacheProxy._cache_proxy(name, self._nodes_cache,
self._real_storage.get_node)
- def get_class(self, name):
+ def get_class(self, name, environment):
if not self._cache_classes:
- return self._real_storage.get_class(name)
+ return self._real_storage.get_class(name, environment)
return MemcacheProxy._cache_proxy(name, self._classes_cache,
self._real_storage.get_class)
diff --git a/reclass/storage/yaml_fs/__init__.py b/reclass/storage/yaml_fs/__init__.py
index c9de29c..30bfed0 100644
--- a/reclass/storage/yaml_fs/__init__.py
+++ b/reclass/storage/yaml_fs/__init__.py
@@ -50,7 +50,7 @@
class ExternalNodeStorage(NodeStorageBase):
- def __init__(self, nodes_uri, classes_uri, default_environment=None):
+ def __init__(self, nodes_uri, classes_uri):
super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
self._nodes_uri = nodes_uri
@@ -58,7 +58,6 @@
self._classes_uri = classes_uri
self._classes = self._enumerate_inventory(classes_uri, NameMangler.classes)
- self._default_environment = default_environment
nodes_uri = property(lambda self: self._nodes_uri)
classes_uri = property(lambda self: self._classes_uri)
@@ -94,10 +93,10 @@
name = os.path.splitext(relpath)[0]
except KeyError, e:
raise reclass.errors.NodeNotFound(self.name, name, self.nodes_uri)
- entity = YamlData.from_file(path).get_entity(name, self._default_environment)
+ entity = YamlData.from_file(path).get_entity(name)
return entity
- def get_class(self, name, nodename=None):
+ def get_class(self, name, nodename=None, environment=None):
vvv('GET CLASS {0}'.format(name))
try:
path = os.path.join(self.classes_uri, self._classes[name])
diff --git a/reclass/storage/yaml_git/__init__.py b/reclass/storage/yaml_git/__init__.py
new file mode 100644
index 0000000..fdf2258
--- /dev/null
+++ b/reclass/storage/yaml_git/__init__.py
@@ -0,0 +1,162 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass
+
+import collections
+import fnmatch
+import os
+import pygit2
+
+import reclass.errors
+from reclass.storage import NodeStorageBase
+from reclass.storage.common import NameMangler
+from reclass.storage.yamldata import YamlData
+
+FILE_EXTENSION = '.yml'
+STORAGE_NAME = 'yaml_git'
+
+def path_mangler(inventory_base_uri, nodes_uri, classes_uri):
+ if nodes_uri == classes_uri:
+ raise errors.DuplicateUriError(nodes_uri, classes_uri)
+ return nodes_uri, classes_uri
+
+
+GitMD = collections.namedtuple('GitMD', ['name', 'path', 'id'], verbose=False, rename=False)
+
+
+class GitURI(object):
+
+ def __init__(self, dictionary):
+ self.repo = dictionary.get('repo', None)
+ self.branch = dictionary.get('branch', None)
+
+ def update(self, dictionary):
+ if 'repo' in dictionary: self.repo = dictionary['repo']
+ if 'branch' in dictionary: self.branch = dictionary['branch']
+
+ def __repr__(self):
+ return '<{0}: {1} {2}>'.format(self.__class__.__name__, self.repo, self.branch)
+
+
+class GitRepo(object):
+
+ def __init__(self, name):
+ self.name = name
+ self.repo = pygit2.Repository(name)
+ self.branches = self.repo.listall_branches()
+ self.files = self.files_in_repo()
+
+ def get(self, id):
+ return self.repo.get(id)
+
+ def files_in_tree(self, tree, path):
+ files = []
+ for entry in tree:
+ if entry.filemode == pygit2.GIT_FILEMODE_TREE:
+ subtree = self.repo.get(entry.id)
+ if path == '':
+ subpath = entry.name
+ else:
+ subpath = '/'.join([path, entry.name])
+ files.extend(self.files_in_tree(subtree, subpath))
+ else:
+ if path == '':
+ relpath = entry.name
+ else:
+ relpath = '/'.join([path, entry.name])
+ files.append(GitMD(entry.name, relpath, entry.id))
+ return files
+
+ def files_in_branch(self, branch):
+ tree = self.repo.revparse_single('master').tree
+ return self.files_in_tree(tree, '')
+
+ def files_in_repo(self):
+ ret = {}
+ for bname in self.branches:
+ branch = {}
+ files = self.files_in_branch(bname)
+ for file in files:
+ if fnmatch.fnmatch(file.name, '*{0}'.format(FILE_EXTENSION)):
+ name = os.path.splitext(file.name)[0]
+ relpath = os.path.dirname(file.path)
+ relpath, name = NameMangler.classes(relpath, name)
+ if name in ret:
+ raise reclass.errors.DuplicateNodeNameError(self.name + ' - ' + bname, name, ret[name], path)
+ else:
+ branch[name] = file
+ ret[bname] = branch
+ return ret
+
+ def nodes(self, branch):
+ ret = {}
+ for name, file in self.files[branch].iteritems():
+ if name in ret:
+ raise reclass.errors.DuplicateNodeNameError(self.name, name, files[name], path)
+ else:
+ ret[name] = file
+ return ret
+
+class ExternalNodeStorage(NodeStorageBase):
+
+ _repos = dict()
+
+ def __init__(self, nodes_uri, classes_uri):
+ super(ExternalNodeStorage, self).__init__(STORAGE_NAME)
+
+ self._nodes_uri = GitURI({ 'branch': 'master', 'repo': None })
+ self._nodes_uri.update(nodes_uri)
+ self._load_repo(self._nodes_uri.repo)
+ self._nodes = self._repos[self._nodes_uri.repo].nodes(self._nodes_uri.branch)
+
+ self._classes_default_uri = GitURI({ 'branch': '__env__', 'repo': None })
+ self._classes_default_uri.update(classes_uri)
+ self._load_repo(self._classes_default_uri.repo)
+
+ self._classes_uri = []
+ if 'env_overrides' in classes_uri:
+ for override in classes_uri['env_overrides']:
+ for env, options in override.iteritems():
+ uri = GitURI({ 'branch': env, 'repo': self._classes_default_uri.repo })
+ uri.update(options)
+ self._classes_uri.append((env, uri))
+ self._load_repo(uri.repo)
+
+ self._classes_uri.append(('*', self._classes_default_uri))
+
+ nodes_uri = property(lambda self: self._nodes_uri)
+ classes_uri = property(lambda self: self._classes_uri)
+
+ def get_node(self, name):
+ file = self._nodes[name]
+ blob = self._repos[self._nodes_uri.repo].get(file.id)
+ entity = YamlData.from_string(blob.data, 'git_fs://{0}#{1}/{2}'.format(self._nodes_uri.repo, self._nodes_uri.branch, file.path)).get_entity(name)
+ return entity
+
+ def get_class(self, name, nodename=None, environment=None):
+ uri = self._env_to_uri(environment)
+ file = self._repos[uri.repo].files[uri.branch][name]
+ blob = self._repos[uri.repo].get(file.id)
+ entity = YamlData.from_string(blob.data, 'git_fs://{0}#{1}/{2}'.format(uri.repo, uri.branch, file.path)).get_entity(name)
+ return entity
+
+ def enumerate_nodes(self):
+ return self._nodes.keys()
+
+ def _load_repo(self, url):
+ if url not in self._repos:
+ self._repos[url] = GitRepo(url)
+
+ def _env_to_uri(self, environment):
+ if environment is None:
+ ret = self._classes_default_uri
+ for env, uri in self._classes_uri:
+ if env == environment:
+ ret = uri
+ break
+ if ret.branch == '__env__':
+ ret.branch = environment
+ if ret.branch == None:
+ ret.branch = 'master'
+ return ret
diff --git a/reclass/storage/yamldata.py b/reclass/storage/yamldata.py
index 8d8363a..6894308 100644
--- a/reclass/storage/yamldata.py
+++ b/reclass/storage/yamldata.py
@@ -46,7 +46,7 @@
def get_data(self):
return self._data
- def get_entity(self, name=None, default_environment=None):
+ def get_entity(self, name=None):
classes = self._data.get('classes')
if classes is None:
classes = []
@@ -70,7 +70,7 @@
if name is None:
name = self._path
- env = self._data.get('environment', default_environment)
+ env = self._data.get('environment', None)
return datatypes.Entity(classes, applications, parameters, exports,
name=name, environment=env, uri=self.uri)