Import of working code base
Signed-off-by: martin f. krafft <madduck@madduck.net>
diff --git a/storage/__init__.py b/storage/__init__.py
new file mode 100644
index 0000000..0613bfe
--- /dev/null
+++ b/storage/__init__.py
@@ -0,0 +1,52 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+class NodeStorageBase(object):
+
+ def __init__(self, nodes_uri, classes_uri):
+ self._nodes_uri = nodes_uri
+ self._classes_uri = classes_uri
+
+ nodes_uri = property(lambda self: self._nodes_uri)
+ classes_uri = property(lambda self: self._classes_uri)
+
+ def _read_entity(self, node, base_uri, seen={}):
+ raise NotImplementedError, "Storage class not implement node info retrieval"
+
+ def nodeinfo(self, node):
+ entity, uri = self._read_nodeinfo(node, self.nodes_uri, {})
+ return {'RECLASS' : {'node': node, 'node_uri': uri},
+ 'classes': list(entity.classes),
+ 'applications': list(entity.applications),
+ 'parameters': dict(entity.parameters)
+ }
+
+ def _list_inventory(self):
+ raise NotImplementedError, "Storage class does not implement inventory listing"
+
+ def inventory(self):
+ entity, applications, classes = self._list_inventory()
+ ret = classes
+ ret.update([(k + '_hosts',v) for k,v in applications.iteritems()])
+ return ret
+
+class StorageBackendLoader(object):
+
+ def __init__(self, storage_type):
+ self._name = storage_type
+ try:
+ self._module = __import__(storage_type, globals(), locals(), storage_type)
+ except ImportError:
+ raise NotImplementedError
+
+ def load(self, attr='ExternalNodeStorage'):
+ klass = getattr(self._module, attr, None)
+ if klass is None:
+ raise AttributeError, \
+ 'Storage backend class {0} does not export "{1}"'.format(self._name, klass)
+ return klass
diff --git a/storage/yaml_fs/__init__.py b/storage/yaml_fs/__init__.py
new file mode 100644
index 0000000..0eff9ed
--- /dev/null
+++ b/storage/yaml_fs/__init__.py
@@ -0,0 +1,59 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+import os
+from storage import NodeStorageBase
+from yamlfile import YamlFile
+from directory import Directory
+
+FILE_EXTENSION = '.yml'
+
+class ExternalNodeStorage(NodeStorageBase):
+
+ def __init__(self, nodes_uri, classes_uri):
+ super(ExternalNodeStorage, self).__init__(nodes_uri, classes_uri)
+
+ def _read_nodeinfo(self, name, base_uri, seen):
+ path = os.path.join(base_uri, name + FILE_EXTENSION)
+ entity = YamlFile(path).entity
+ seen[name] = True
+ for klass in entity.classes:
+ if klass not in seen:
+ ret = self._read_nodeinfo(klass, self.classes_uri, seen)[0]
+ ret.merge(entity)
+ entity = ret
+ return entity, path
+
+ def _list_inventory(self):
+ d = Directory(self.nodes_uri)
+
+ entities = {}
+
+ def register_fn(dirpath, filenames):
+ for f in filter(lambda f: f.endswith(FILE_EXTENSION), filenames):
+ name = f[:-len(FILE_EXTENSION)]
+ nodeinfo = self.nodeinfo(name)
+ entities[name] = nodeinfo
+
+ d.walk(register_fn)
+
+ applications = {}
+ classes = {}
+ for f, nodeinfo in entities.iteritems():
+ for a in nodeinfo['applications']:
+ if a in applications:
+ applications[a].append(f)
+ else:
+ applications[a] = [f]
+ for c in nodeinfo['classes']:
+ if c in classes:
+ classes[c].append(f)
+ else:
+ classes[c] = [f]
+
+ return entities, applications, classes
diff --git a/storage/yaml_fs/directory.py b/storage/yaml_fs/directory.py
new file mode 100644
index 0000000..c085731
--- /dev/null
+++ b/storage/yaml_fs/directory.py
@@ -0,0 +1,53 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+import os
+import sys
+
+SKIPDIRS = ( '.git' , '.svn' , 'CVS', 'SCCS', '.hg', '_darcs' )
+FILE_EXTENSION = '.yml'
+
+def vvv(msg):
+ #print >>sys.stderr, msg
+ pass
+
+class Directory(object):
+
+ def __init__(self, path, fileclass=None):
+ ''' Initialise a directory object '''
+ self._path = path
+ self._fileclass = fileclass
+ self._files = {}
+
+ def _register_files(self, dirpath, filenames):
+ for f in filter(lambda f: f.endswith(FILE_EXTENSION), filenames):
+ vvv('REGISTER {0}'.format(f))
+ f = os.path.join(dirpath, f)
+ ptr = None if not self._fileclass else self._fileclass(f)
+ self._files[f] = ptr
+
+ files = property(lambda self: self._files)
+
+ def walk(self, register_fn=None):
+ def _error(error):
+ raise Exception('{0}: {1} ({2})'.format(error.filename, error.strerror, error.errno))
+ if not callable(register_fn): register_fn = self._register_files
+ for dirpath, dirnames, filenames in os.walk(self._path,
+ topdown=True,
+ onerror=_error,
+ followlinks=True):
+ vvv('RECURSE {0}, {1} files, {2} subdirectories'.format(
+ dirpath.replace(os.getcwd(), '.'), len(filenames), len(dirnames)))
+ for d in SKIPDIRS:
+ if d in dirnames:
+ vvv(' SKIP subdirectory {0}'.format(d))
+ dirnames.remove(d)
+ register_fn(dirpath, filenames)
+
+ def __repr__(self):
+ return '<{0} {1}>'.format(self.__class__.__name__, self._path)
diff --git a/storage/yaml_fs/tests/classes/basenode.yml b/storage/yaml_fs/tests/classes/basenode.yml
new file mode 100644
index 0000000..d9d997a
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/basenode.yml
@@ -0,0 +1,5 @@
+applications:
+- motd
+- firewall
+parameters:
+ realm: madduck.net
diff --git a/storage/yaml_fs/tests/classes/debiannode.yml b/storage/yaml_fs/tests/classes/debiannode.yml
new file mode 100644
index 0000000..43e35e4
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/debiannode.yml
@@ -0,0 +1,4 @@
+classes:
+- basenode
+applications:
+- apt
diff --git a/storage/yaml_fs/tests/classes/debiannode@sid.yml b/storage/yaml_fs/tests/classes/debiannode@sid.yml
new file mode 100644
index 0000000..806612a
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/debiannode@sid.yml
@@ -0,0 +1,4 @@
+classes:
+- debiannode
+parameters:
+ debian_codename: sid
diff --git a/storage/yaml_fs/tests/classes/debiannode@squeeze.yml b/storage/yaml_fs/tests/classes/debiannode@squeeze.yml
new file mode 100644
index 0000000..08075b5
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/debiannode@squeeze.yml
@@ -0,0 +1,4 @@
+classes:
+- debiannode
+parameters:
+ debian_codename: squeeze
diff --git a/storage/yaml_fs/tests/classes/debiannode@wheezy.yml b/storage/yaml_fs/tests/classes/debiannode@wheezy.yml
new file mode 100644
index 0000000..5ced15a
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/debiannode@wheezy.yml
@@ -0,0 +1,4 @@
+classes:
+- debiannode
+parameters:
+ debian_codename: wheezy
diff --git a/storage/yaml_fs/tests/classes/hosted@munich.yml b/storage/yaml_fs/tests/classes/hosted@munich.yml
new file mode 100644
index 0000000..1d14dad
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/hosted@munich.yml
@@ -0,0 +1,4 @@
+parameters:
+ location: munich
+ apt:
+ mirror_base: uni-erlangen
diff --git a/storage/yaml_fs/tests/classes/hosted@zurich.yml b/storage/yaml_fs/tests/classes/hosted@zurich.yml
new file mode 100644
index 0000000..050af27
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/hosted@zurich.yml
@@ -0,0 +1,4 @@
+parameters:
+ location: zurich
+ apt:
+ mirror_base: switch
diff --git a/storage/yaml_fs/tests/classes/mailserver.yml b/storage/yaml_fs/tests/classes/mailserver.yml
new file mode 100644
index 0000000..37d6673
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/mailserver.yml
@@ -0,0 +1,5 @@
+applications:
+- postfix
+parameters:
+ firewall:
+ openport: 25/tcp
diff --git a/storage/yaml_fs/tests/classes/subdir/subclass.yml b/storage/yaml_fs/tests/classes/subdir/subclass.yml
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/subdir/subclass.yml
diff --git a/storage/yaml_fs/tests/classes/webserver.yml b/storage/yaml_fs/tests/classes/webserver.yml
new file mode 100644
index 0000000..5db36ff
--- /dev/null
+++ b/storage/yaml_fs/tests/classes/webserver.yml
@@ -0,0 +1,5 @@
+applications:
+- lighttpd
+parameters:
+ firewall:
+ openport: 80/tcp
diff --git a/storage/yaml_fs/tests/nodes/blue.yml b/storage/yaml_fs/tests/nodes/blue.yml
new file mode 100644
index 0000000..d7d06ee
--- /dev/null
+++ b/storage/yaml_fs/tests/nodes/blue.yml
@@ -0,0 +1,9 @@
+classes:
+- debiannode@squeeze
+- hosted@munich
+- mailserver
+- webserver
+parameters:
+ motd:
+ greeting: This node is $nodename
+ colour: blue
diff --git a/storage/yaml_fs/tests/nodes/green.yml b/storage/yaml_fs/tests/nodes/green.yml
new file mode 100644
index 0000000..1f3d6d6
--- /dev/null
+++ b/storage/yaml_fs/tests/nodes/green.yml
@@ -0,0 +1,10 @@
+classes:
+- debiannode@wheezy
+- hosted@zurich
+- mailserver
+applications:
+- ~firewall
+parameters:
+ motd:
+ greeting: This node is $nodename
+ colour: green
diff --git a/storage/yaml_fs/tests/nodes/red.yml b/storage/yaml_fs/tests/nodes/red.yml
new file mode 100644
index 0000000..5fe618d
--- /dev/null
+++ b/storage/yaml_fs/tests/nodes/red.yml
@@ -0,0 +1,8 @@
+classes:
+- debiannode@sid
+- hosted@zurich
+- webserver
+parameters:
+ motd:
+ greeting: This node is $nodename
+ colour: red
diff --git a/storage/yaml_fs/tests/test_directory.py b/storage/yaml_fs/tests/test_directory.py
new file mode 100644
index 0000000..4c2604a
--- /dev/null
+++ b/storage/yaml_fs/tests/test_directory.py
@@ -0,0 +1,30 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+import storage.yaml_fs.directory as directory
+import os, sys
+
+TESTDIR = os.path.join(sys.path[0], 'classes')
+FILECOUNT = 10
+
+class TestDirectory:
+
+ def setUp(self):
+ self._dir = directory.Directory(TESTDIR)
+
+ def test_walk_registry(self):
+ def count_fn(d, f):
+ count_fn.c += len(f)
+ count_fn.c = 0
+ self._dir.walk(register_fn=count_fn)
+ assert count_fn.c == FILECOUNT
+
+ def test_walk(self):
+ self._dir.walk()
+ assert len(self._dir.files) == FILECOUNT
+
diff --git a/storage/yaml_fs/tests/test_yaml_fs.py b/storage/yaml_fs/tests/test_yaml_fs.py
new file mode 100644
index 0000000..d2cdf29
--- /dev/null
+++ b/storage/yaml_fs/tests/test_yaml_fs.py
@@ -0,0 +1,64 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+from storage.yaml_fs import ExternalNodeStorage
+
+import os
+
+PWD = os.path.dirname(__file__)
+HOSTS = ['red', 'blue', 'green']
+MEMBERSHIPS = {'apt_hosts': HOSTS,
+ 'motd_hosts': HOSTS,
+ 'firewall_hosts': HOSTS[:2],
+ 'lighttpd_hosts': HOSTS[:2],
+ 'postfix_hosts': HOSTS[1:],
+ 'basenode': HOSTS,
+ 'debiannode': HOSTS,
+ 'debiannode@sid': HOSTS[0:1],
+ 'debiannode@wheezy': HOSTS[2:3],
+ 'debiannode@squeeze': HOSTS[1:2],
+ 'hosted@munich': HOSTS[1:2],
+ 'hosted@zurich': [HOSTS[0], HOSTS[2]],
+ 'mailserver': HOSTS[1:],
+ 'webserver': HOSTS[:2]
+ }
+
+class TestYamlFs:
+
+ def setUp(self):
+ self._storage = ExternalNodeStorage(os.path.join(PWD, 'nodes'),
+ os.path.join(PWD, 'classes'))
+ self._inventory = self._storage.inventory()
+
+ def test_inventory_setup(self):
+ assert isinstance(self._inventory, dict)
+ assert len(self._inventory) == len(MEMBERSHIPS)
+ for i in MEMBERSHIPS.iterkeys():
+ assert i in self._inventory
+
+ def test_inventory_memberships(self):
+ for app, members in self._inventory.iteritems():
+ for i in MEMBERSHIPS[app]:
+ print i
+ assert i in members
+ for app, members in MEMBERSHIPS.iteritems():
+ for i in self._inventory[app]:
+ print i
+ assert i in members
+
+ def test_host_meta(self):
+ for n in HOSTS:
+ node = self._storage.nodeinfo(n)
+ assert 'RECLASS' in node
+
+ def test_host_entity(self):
+ for n in HOSTS:
+ node = self._storage.nodeinfo(n)
+ assert 'applications' in node
+ assert 'classes' in node
+ assert 'parameters' in node
diff --git a/storage/yaml_fs/tests/test_yamlfile.py b/storage/yaml_fs/tests/test_yamlfile.py
new file mode 100644
index 0000000..d8302f1
--- /dev/null
+++ b/storage/yaml_fs/tests/test_yamlfile.py
@@ -0,0 +1,28 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+import storage.yaml_fs.yamlfile as yamlfile
+import os, sys
+
+TESTFILE = os.path.join(sys.path[0], 'nodes', 'blue.yml')
+
+class TestYamlFile:
+
+ def setUp(self):
+ self._file = yamlfile.YamlFile(TESTFILE)
+
+ def test_data(self):
+ e = self._file.entity
+ c = e.classes
+ assert len(c) == 4
+ assert hasattr(c, 'merge')
+ p = e.parameters
+ assert len(p) == 2
+ assert 'motd' in p
+ assert 'colour' in p
+ assert hasattr(p, 'merge')
diff --git a/storage/yaml_fs/yamlfile.py b/storage/yaml_fs/yamlfile.py
new file mode 100644
index 0000000..0d1175a
--- /dev/null
+++ b/storage/yaml_fs/yamlfile.py
@@ -0,0 +1,37 @@
+#
+# -*- coding: utf-8 -*-
+#
+# This file is part of reclass (http://github.com/madduck/reclass)
+#
+# Copyright © 2007–13 martin f. krafft <madduck@madduck.net>
+# Released under the terms of the Artistic Licence 2.0
+#
+import datatypes
+import yaml
+
+class YamlFile(object):
+
+ def __init__(self, path):
+ ''' Initialise a yamlfile object '''
+ self._path = path
+ self._data = dict()
+ self._read()
+
+ def _read(self):
+ fp = file(self._path)
+ self._data = yaml.safe_load(fp)
+ self._name = self._data.get('name', self._path)
+ fp.close()
+
+ name = property(lambda self: self._name)
+
+ def _get_entity(self):
+ classes = datatypes.Classes(self._data.get('classes', []))
+ parameters = datatypes.Parameters(self._data.get('parameters', {}))
+ applications = datatypes.Applications(self._data.get('applications', []))
+ return datatypes.Entity(classes, parameters, applications)
+ entity = property(lambda self: self._get_entity())
+
+ def __repr__(self):
+ return '<{0} {1}, {2}>'.format(self.__class__.__name__, self._name,
+ self._data.keys())