Merge pull request #71 from salt-formulas/andrewp-yaml-git
usable yaml_git and mixed storage types
diff --git a/README-extensions.rst b/README-extensions.rst
index d321fa1..e67e441 100644
--- a/README-extensions.rst
+++ b/README-extensions.rst
@@ -577,3 +577,117 @@
...
If the subfolder path starts with the underscore character ``_``, then the subfolder path is NOT added to the node name.
+
+
+Git storage type
+----------------
+
+Reclass node and class yaml files can be read from a remote git repository with the yaml_git storage type. Use nodes_uri and
+classes_uri to define the git repos to use for nodes and classes. These can be the same repo.
+
+For salt masters using ssh connections the private and public keys must be readable by the salt daemon, which requires the
+private key NOT be password protected. For stand alone reclass using ssh connections if the privkey and pubkey options
+are not defined then any in memory key (from ssh-add) will be used.
+
+Salt master reclass config example:
+
+.. code-block:: yaml
+
+ storage_type:yaml_git
+ nodes_uri:
+ # branch to use
+ branch: master
+
+ # cache directory (default: ~/.reclass/git/cache)
+ cache_dir: /var/cache/reclass/git
+
+ # lock directory (default: ~/.reclass/git/lock)
+ lock_dir: /var/cache/reclass/lock
+
+ # private key for ssh connections (no default, but will used keys stored
+ # by ssh-add in memory if privkey and pubkey are not set)
+ privkey: /root/salt_rsa
+ # public key for ssh connections
+ pubkey: /root/salt_rsa.pub
+
+ repo: git+ssh://gitlab@remote.server:salt/nodes.git
+
+ classes_uri:
+ # branch to use or __env__ to use the branch matching the node
+ # environment name
+ branch: __env__
+
+ # cache directory (default: ~/.reclass/git/cache)
+ cache_dir: /var/cache/reclass/git
+
+ # lock directory (default: ~/.reclass/git/lock)
+ lock_dir: /var/cache/reclass/lock
+
+ # private key for ssh connections (no default, but will used keys stored
+ # by ssh-add in memory if privkey and pubkey are not set)
+ privkey: /root/salt_rsa
+ # public key for ssh connections
+ pubkey: /root/salt_rsa.pub
+
+ # branch/env overrides for specific branches
+ env_overrides:
+ # prod env uses master branch
+ - prod:
+ branch: master
+ # use master branch for nodes with no environment defined
+ - none:
+ branch: master
+
+ repo: git+ssh://gitlab@remote.server:salt/site.git
+
+ # root directory of the class hierarcy in git repo
+ # defaults to root directory of git repo if not given
+ root: classes
+
+
+Mixed storage type
+------------------
+
+Use a mixture of storage types.
+
+Salt master reclass config example, which by default uses yaml_git storage but overrides the location for
+classes for the pre-prod environment to use a directory on the local disc:
+
+.. code-block:: yaml
+
+ storage_type: mixed
+ nodes_uri:
+ # storage type to use
+ storage_type: yaml_git
+
+ # yaml_git storage options
+ branch: master
+ cache_dir: /var/cache/reclass/git
+ lock_dir: /var/cache/reclass/lock
+ privkey: /root/salt_rsa
+ pubkey: /root/salt_rsa.pub
+ repo: git+ssh://gitlab@remote.server:salt/nodes.git
+
+ classes_uri:
+ # storage type to use
+ storage_type: yaml_git
+
+ # yaml_git storage options
+ branch: __env__
+ cache_dir: /var/cache/reclass/git
+ lock_dir: /var/cache/reclass/lock
+ privkey: /root/salt_rsa
+ pubkey: /root/salt_rsa.pub
+ repo: git+ssh://gitlab@remote.server:salt/site.git
+ root: classes
+
+ env_overrides:
+ - prod:
+ branch: master
+ - none:
+ branch: master
+ - pre-prod:
+ # override storage type for this environment
+ storage_type: yaml_fs
+ # options for yaml_fs storage type
+ uri: /srv/salt/env/pre-prod/classes
diff --git a/reclass/core.py b/reclass/core.py
index 75eea54..6dac5c3 100644
--- a/reclass/core.py
+++ b/reclass/core.py
@@ -189,7 +189,7 @@
node.interpolate_single_export(q)
except InterpolationError as e:
e.nodename = nodename
- raise InvQueryError(q.contents(), e, context=p, uri=q.uri)
+ raise InvQueryError(q.contents, e, context=p, uri=q.uri)
inventory[nodename] = node.exports.as_dict()
return inventory
diff --git a/reclass/storage/yaml_git/__init__.py b/reclass/storage/yaml_git/__init__.py
index 45cb6c0..a28079b 100644
--- a/reclass/storage/yaml_git/__init__.py
+++ b/reclass/storage/yaml_git/__init__.py
@@ -9,8 +9,11 @@
import collections
import distutils.version
+import errno
+import fcntl
import fnmatch
import os
+import time
# Squelch warning on centos7 due to upgrading cffi
# see https://github.com/saltstack/salt/pull/39871
@@ -50,6 +53,7 @@
self.branch = None
self.root = None
self.cache_dir = None
+ self.lock_dir = None
self.pubkey = None
self.privkey = None
self.password = None
@@ -59,6 +63,7 @@
if 'repo' in dictionary: self.repo = dictionary['repo']
if 'branch' in dictionary: self.branch = dictionary['branch']
if 'cache_dir' in dictionary: self.cache_dir = dictionary['cache_dir']
+ if 'lock_dir' in dictionary: self.lock_dir = dictionary['lock_dir']
if 'pubkey' in dictionary: self.pubkey = dictionary['pubkey']
if 'privkey' in dictionary: self.privkey = dictionary['privkey']
if 'password' in dictionary: self.password = dictionary['password']
@@ -72,8 +77,31 @@
return '<{0}: {1} {2} {3}>'.format(self.__class__.__name__, self.repo, self.branch, self.root)
-class GitRepo(object):
+class LockFile():
+ def __init__(self, file):
+ self._file = file
+ def __enter__(self):
+ self._fd = open(self._file, 'w+')
+ start = time.time()
+ while True:
+ if (time.time() - start) > 120:
+ raise IOError('Timeout waiting to lock file: {0}'.format(self._file))
+ try:
+ fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ break
+ except IOError as e:
+ # raise on unrelated IOErrors
+ if e.errno != errno.EAGAIN:
+ raise
+ else:
+ time.sleep(0.1)
+
+ def __exit__(self, type, value, traceback):
+ self._fd.close()
+
+
+class GitRepo(object):
def __init__(self, uri, node_name_mangler, class_name_mangler):
if pygit2 is None:
raise errors.MissingModuleError('pygit2')
@@ -85,11 +113,18 @@
self.cache_dir = '{0}/{1}/{2}'.format(os.path.expanduser("~"), '.reclass/cache/git', self.name)
else:
self.cache_dir = '{0}/{1}'.format(uri.cache_dir, self.name)
-
+ if uri.lock_dir is None:
+ self.lock_file = '{0}/{1}/{2}'.format(os.path.expanduser("~"), '.reclass/cache/lock', self.name)
+ else:
+ self.lock_file = '{0}/{1}'.format(uri.lock_dir, self.name)
+ lock_dir = os.path.dirname(self.lock_file)
+ if not os.path.exists(lock_dir):
+ os.makedirs(lock_dir)
self._node_name_mangler = node_name_mangler
self._class_name_mangler = class_name_mangler
- self._init_repo(uri)
- self._fetch()
+ with LockFile(self.lock_file):
+ self._init_repo(uri)
+ self._fetch()
self.branches = self.repo.listall_branches()
self.files = self.files_in_repo()
@@ -99,10 +134,7 @@
else:
os.makedirs(self.cache_dir)
self.repo = pygit2.init_repository(self.cache_dir, bare=True)
-
- if not self.repo.remotes:
self.repo.create_remote('origin', self.url)
-
if 'ssh' in self.transport:
if '@' in self.url:
user, _, _ = self.url.partition('@')
@@ -130,7 +162,6 @@
if self.credentials is not None:
origin.credentials = self.credentials
fetch_results = origin.fetch(**fetch_kwargs)
-
remote_branches = self.repo.listall_branches(pygit2.GIT_BRANCH_REMOTE)
local_branches = self.repo.listall_branches()
for remote_branch_name in remote_branches:
@@ -208,8 +239,8 @@
ret[node_name] = file
return ret
-class ExternalNodeStorage(ExternalNodeStorageBase):
+class ExternalNodeStorage(ExternalNodeStorageBase):
def __init__(self, nodes_uri, classes_uri, compose_node_name):
super(ExternalNodeStorage, self).__init__(STORAGE_NAME, compose_node_name)
self._repos = dict()