Imported Upstream version 1.4.1
diff --git a/doc/.gitignore b/doc/.gitignore
new file mode 100644
index 0000000..796b96d
--- /dev/null
+++ b/doc/.gitignore
@@ -0,0 +1 @@
+/build
diff --git a/doc/Makefile b/doc/Makefile
new file mode 100644
index 0000000..591ae5c
--- /dev/null
+++ b/doc/Makefile
@@ -0,0 +1,153 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS =
+SPHINXBUILD = sphinx-build -N
+PAPER =
+BUILDDIR = build
+
+# Internal variables.
+PAPEROPT_a4 = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+help:
+ @echo "Please use \`make <target>' where <target> is one of"
+ @echo " html to make standalone HTML files"
+ @echo " dirhtml to make HTML files named index.html in directories"
+ @echo " singlehtml to make a single large HTML file"
+ @echo " pickle to make pickle files"
+ @echo " json to make JSON files"
+ @echo " htmlhelp to make HTML files and a HTML help project"
+ @echo " qthelp to make HTML files and a qthelp project"
+ @echo " devhelp to make HTML files and a Devhelp project"
+ @echo " epub to make an epub"
+ @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+ @echo " latexpdf to make LaTeX files and run them through pdflatex"
+ @echo " text to make text files"
+ @echo " man to make manual pages"
+ @echo " texinfo to make Texinfo files"
+ @echo " info to make Texinfo files and run them through makeinfo"
+ @echo " gettext to make PO message catalogs"
+ @echo " changes to make an overview of all changed/added/deprecated items"
+ @echo " linkcheck to check all external links for integrity"
+ @echo " doctest to run all doctests embedded in the documentation (if enabled)"
+
+clean:
+ -rm -rf $(BUILDDIR)/*
+
+html:
+ $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
+
+dirhtml:
+ $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+ @echo
+ @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+ $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+ @echo
+ @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+ $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+ @echo
+ @echo "Build finished; now you can process the pickle files."
+
+json:
+ $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+ @echo
+ @echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+ $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+ @echo
+ @echo "Build finished; now you can run HTML Help Workshop with the" \
+ ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+ $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+ @echo
+ @echo "Build finished; now you can run "qcollectiongenerator" with the" \
+ ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+ @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/reclass.qhcp"
+ @echo "To view the help file:"
+ @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/reclass.qhc"
+
+devhelp:
+ $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+ @echo
+ @echo "Build finished."
+ @echo "To view the help file:"
+ @echo "# mkdir -p $$HOME/.local/share/devhelp/reclass"
+ @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/reclass"
+ @echo "# devhelp"
+
+epub:
+ $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+ @echo
+ @echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo
+ @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+ @echo "Run \`make' in that directory to run these through (pdf)latex" \
+ "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+ $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+ @echo "Running LaTeX files through pdflatex..."
+ $(MAKE) -C $(BUILDDIR)/latex all-pdf
+ @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+ $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+ @echo
+ @echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+ $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+ @echo
+ @echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo
+ @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+ @echo "Run \`make' in that directory to run these through makeinfo" \
+ "(use \`make info' here to do that automatically)."
+
+info:
+ $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+ @echo "Running Texinfo files through makeinfo..."
+ make -C $(BUILDDIR)/texinfo info
+ @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+ $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+ @echo
+ @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+ $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+ @echo
+ @echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+ $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+ @echo
+ @echo "Link check complete; look for any errors in the above output " \
+ "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+ $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+ @echo "Testing of doctests in the sources finished, look at the " \
+ "results in $(BUILDDIR)/doctest/output.txt."
diff --git a/doc/source/ansible.rst b/doc/source/ansible.rst
new file mode 100644
index 0000000..b77a2c3
--- /dev/null
+++ b/doc/source/ansible.rst
@@ -0,0 +1,213 @@
+==========================
+Using reclass with Ansible
+==========================
+
+.. warning::
+
+ I was kicked out of the Ansible community, presumably for `asking the wrong
+ questions`_, and therefore I have no interest in developing this adapter
+ anymore. If you use it and have changes, I will take your patch.
+
+.. _asking the wrong questions: https://github.com/madduck/reclass/issues/6
+
+Quick start with Ansible
+------------------------
+The following steps should get you up and running quickly with |reclass| and
+`Ansible`_. Generally, we will be working in ``/etc/ansible``. However, if you
+are using a source-code checkout of Ansible, you might also want to work
+inside the ``./hacking`` directory instead.
+
+Or you can also just look into ``./examples/ansible`` of your |reclass|
+checkout, where the following steps have already been prepared.
+
+/…/reclass refers to the location of your |reclass| checkout.
+
+.. todo::
+
+ With |reclass| now in Debian, as well as installable from source, the
+ following should be checked for path consistency…
+
+#. Complete the installation steps described in the :doc:`installation section
+ <install>`.
+
+#. Symlink ``/usr/share/reclass/reclass-ansible`` (or wherever your distro put
+ that file), or ``/…/reclass/reclass/adapters/ansible.py`` (if running from
+ source) to ``/etc/ansible/hosts`` (or ``./hacking/hosts``).
+
+#. Copy the two directories ``nodes`` and ``classes`` from the example
+ subdirectory in the |reclass| checkout to ``/etc/ansible``
+
+ If you prefer to put those directories elsewhere, you can create
+ ``/etc/ansible/reclass-config.yml`` with contents such as::
+
+ storage_type: yaml_fs
+ inventory_base_uri: /srv/reclass
+
+ Note that ``yaml_fs`` is currently the only supported ``storage_type``, and
+ it's the default if you don't set it.
+
+#. Check out your inventory by invoking
+
+ ::
+
+ $ ./hosts --list
+
+ which should return 5 groups in JSON format, and each group has exactly
+ one member ``localhost``.
+
+4. See the node information for ``localhost``::
+
+ $ ./hosts --host localhost
+
+ This should print a set of keys and values, including a greeting,
+ a colour, and a sub-class called ``__reclas__``.
+
+5. Execute some ansible commands, e.g.::
+
+ $ ansible -i hosts \* --list-hosts
+ $ ansible -i hosts \* -m ping
+ $ ansible -i hosts \* -m debug -a 'msg="${greeting}"'
+ $ ansible -i hosts \* -m setup
+ $ ansible-playbook -i hosts test.yml
+
+6. You can also invoke |reclass| directly, which gives a slightly different
+ view onto the same data, i.e. before it has been adapted for Ansible::
+
+ $ /…/reclass/reclass.py --pretty-print --inventory
+ $ /…/reclass/reclass.py --pretty-print --nodeinfo localhost
+
+ Or, if |reclass| is properly installed, just use the |reclass| command.
+
+Integration with Ansible
+------------------------
+The integration between |reclass| and Ansible is performed through an adapter,
+and needs not be of our concern too much.
+
+However, Ansible has no concept of "nodes", "applications", "parameters", and
+"classes". Therefore it is necessary to explain how those correspond to
+Ansible. Crudely, the following mapping exists:
+
+================= ===============
+|reclass| concept Ansible concept
+================= ===============
+nodes hosts
+classes groups
+applications playbooks
+parameters host_vars
+================= ===============
+
+|reclass| does not provide any ``group_vars`` because of its node-centric
+perspective. While class definitions include parameters, those are inherited
+by the node definitions and hence become node_vars.
+
+|reclass| also does not provide playbooks, nor does it deal with any of the
+related Ansible concepts, i.e. ``vars_files``, vars, tasks, handlers, roles, etc..
+
+ Let it be said at this point that you'll probably want to stop using
+ ``host_vars``, ``group_vars`` and ``vars_files`` altogether, and if only
+ because you should no longer need them, but also because the variable
+ precedence rules of Ansible are full of surprises, at least to me.
+
+|reclass|' Ansible adapter massage the |reclass| output into Ansible-usable data,
+namely:
+
+- Every class in the ancestry of a node becomes a group to Ansible. This is
+ mainly useful to be able to target nodes during interactive use of
+ Ansible, e.g.::
+
+ $ ansible debiannode@wheezy -m command -a 'apt-get upgrade'
+ → upgrade all Debian nodes running wheezy
+
+ $ ansible ssh.server -m command -a 'invoke-rc.d ssh restart'
+ → restart all SSH server processes
+
+ $ ansible mailserver -m command -a 'tail -n1000 /var/log/mail.err'
+ → obtain the last 1,000 lines of all mailserver error log files
+
+ The attentive reader might stumble over the use of singular words, whereas
+ it might make more sense to address all ``mailserver*s*`` with this tool.
+ This is convention and up to you. I prefer to think of my node as
+ a (singular) mailserver when I add ``mailserver`` to its parent classes.
+
+- Every entry in the list of a host's applications might well correspond to
+ an Ansible playbook. Therefore, |reclass| creates a (Ansible-)group for
+ every application, and adds ``_hosts`` to the name. This postfix can be
+ configured with a CLI option (``--applications-postfix``) or in the
+ configuration file (``applications_postfix``).
+
+ For instance, the ssh.server class adds the ssh.server application to
+ a node's application list. Now the admin might create an Ansible playbook
+ like so::
+
+ - name: SSH server management
+ hosts: ssh.server_hosts ← SEE HERE
+ tasks:
+ - name: install SSH package
+ action: …
+ …
+
+ There's a bit of redundancy in this, but unfortunately Ansible playbooks
+ hardcode the nodes to which a playbook applies.
+
+ It's now trivial to apply this playbook across your infrastructure::
+
+ $ ansible-playbook ssh.server.yml
+
+ My suggested way to use Ansible site-wide is then to create a ``site.yml``
+ playbook that includes all the other playbooks (which shall hopefully be
+ based on Ansible roles), and then to invoke Ansible like this:
+
+ ansible-playbook site.yml
+
+ or, if you prefer only to reconfigure a subset of nodes, e.g. all
+ webservers::
+
+ $ ansible-playbook site.yml --limit webserver
+
+ Again, if the singular word ``webserver`` puts you off, change the
+ convention as you wish.
+
+ And if anyone comes up with a way to directly connect groups in the
+ inventory with roles, thereby making it unnecessary to write playbook
+ files (containing redundant information), please tell me!
+
+- Parameters corresponding to a node become ``host_vars`` for that host.
+
+Variable interpolation
+----------------------
+Ansible allows you to include `Jinja2`_-style variables in parameter values::
+
+ parameters:
+ motd:
+ greeting: Welcome to {{ ansible_fqdn }}!
+ closing: This system is part of {{ realm }}
+ dict_reference: {{ motd }}
+
+However, in resolving this, Ansible casts everything to a string, so in this
+example, ``dict_reference`` would be the string-representation of the
+dictionary under the ``motd`` key [#string_casts]_. To get at facts (such as
+``ansible_fqdn``), you still have to use this approach, but for pure parameter
+references, I strongly suggest to use |reclass| interpolation instead, as it
+supports deep references, does not clobber type information, and is more
+efficient anyway::
+
+ parameters:
+ motd:
+ greeting: Welcome to {{ ansible_fqdn }}!
+ closing: This system is part of ${realm}
+ dict_reference: ${motd}
+
+Now you just need to specify realm somewhere. The reference can reside in
+a parent class, while the variable is defined e.g. in the node definition.
+
+And as expected, ``dict_reference`` now points to a dictionary, not
+a string-representation thereof.
+
+.. [#string_casts] I pointed this out to Michael Dehaan, Ansible's chief
+ developer, but he denied this behaviour. When I tried to provide further
+ insights, I found myself banned from the mailing list, apparently because
+ I dared to point out flaws. If you care, you may look at
+ https://github.com/madduck/reclass/issues/6 for more information.
+
+.. include:: extrefs.inc
+.. include:: substs.inc
diff --git a/doc/source/changelog.rst b/doc/source/changelog.rst
new file mode 100644
index 0000000..d7aa7b2
--- /dev/null
+++ b/doc/source/changelog.rst
@@ -0,0 +1,43 @@
+=========
+ChangeLog
+=========
+
+========= ========== ========================================================
+Version Date Changes
+========= ========== ========================================================
+1.4.1 2014-10-28 * Revert debug logging, which wasn't fault-free and so
+ it needs more time to mature.
+1.4 2014-10-25 * Add rudimentary debug logging
+ * Prevent interpolate() from overwriting merged values
+ * Look for "init" instead of "index" when being fed
+ a directory.
+ * Fix error reporting on node name collision across
+ subdirectories.
+1.3 2014-03-01 * Salt: pillar data from previous pillars are now
+ available to reclass parameter interpolation
+ * yaml_fs: classes may be defined in subdirectories
+ (closes: #12, #19, #20)
+ * Migrate Salt adapter to new core API (closes: #18)
+ * Fix --nodeinfo invocation in docs (closes: #21)
+1.2.2 2013-12-27 * Recurse classes obtained from class mappings
+ (closes: #16)
+ * Fix class mapping regexp rendering in docs
+ (closes: #15)
+1.2.1 2013-12-26 * Fix Salt adapter wrt. class mappings
+ (closes: #14)
+1.2 2013-12-10 * Introduce class mappings (see :doc:`operations`)
+ (closes: #5)
+ * Fix parameter interpolation across merged lists
+ (closes: #13).
+ * Caching of classes for performance reasons, especially
+ during the inventory runs
+ * yaml_fs: nodes may be defined in subdirectories
+ (closes: #10).
+ * Classes and nodes URI must not overlap anymore
+ * Class names must not contain spaces
+1.1 2013-08-28 Salt adapter: fix interface to include minion_id, filter
+ output accordingly; fixes master_tops
+1.0.2 2013-08-27 Fix incorrect versioning in setuptools
+1.0.1 2013-08-27 Documentation updates, new homepage
+1.0 2013-08-26 Initial release
+========= ========== ========================================================
diff --git a/doc/source/concepts.rst b/doc/source/concepts.rst
new file mode 100644
index 0000000..76b5818
--- /dev/null
+++ b/doc/source/concepts.rst
@@ -0,0 +1,133 @@
+================
+reclass concepts
+================
+|reclass| assumes a node-centric perspective into your inventory. This is
+obvious when you query |reclass| for node-specific information, but it might not
+be clear when you ask |reclass| to provide you with a list of groups. In that
+case, |reclass| loops over all nodes it can find in its database, reads all
+information it can find about the nodes, and finally reorders the result to
+provide a list of groups with the nodes they contain.
+
+Since the term "groups" is somewhat ambiguous, it helps to start off with
+a short glossary of |reclass|-specific terminology:
+
+============ ==============================================================
+Concept Description
+============ ==============================================================
+node A node, usually a computer in your infrastructure
+class A category, tag, feature, or role that applies to a node
+ Classes may be nested, i.e. there can be a class hierarchy
+application A specific set of behaviour to apply
+parameter Node-specific variables, with inheritance throughout the class
+ hierarchy.
+============ ==============================================================
+
+A class consists of zero or more parent classes, zero or more applications,
+and any number of parameters.
+
+A class name must not contain spaces.
+
+A node is almost equivalent to a class, except that it usually does not (but
+can) specify applications.
+
+When |reclass| parses a node (or class) definition and encounters a parent
+class, it recurses to this parent class first before reading any data of the
+node (or class). When |reclass| returns from the recursive, depth first walk, it
+then merges all information of the current node (or class) into the
+information it obtained during the recursion.
+
+Furthermore, a node (or class) may define a list of classes it derives from,
+in which case classes defined further down the list will be able to override
+classes further up the list.
+
+Information in this context is essentially one of a list of applications or
+a list of parameters.
+
+The interaction between the depth-first walk and the delayed merging of data
+means that the node (and any class) may override any of the data defined by
+any of the parent classes (ancestors). This is in line with the assumption
+that more specific definitions ("this specific host") should have a higher
+precedence than more general definitions ("all webservers", which includes all
+webservers in Munich, which includes "this specific host", for example).
+
+Here's a quick example, showing how parameters accumulate and can get
+replaced.
+
+ All "unixnodes" (i.e. nodes who have the ``unixnode`` class in their
+ ancestry) have ``/etc/motd`` centrally-managed (through the ``motd``
+ application), and the `unixnode` class definition provides a generic
+ message-of-the-day to be put into this file.
+
+ All descendants of the class ``debiannode``, a descendant of ``unixnode``,
+ should include the Debian codename in this message, so the
+ message-of-the-day is overwritten in the ``debiannodes`` class.
+
+ The node ``quantum.example.org`` (a `debiannode`) will have a scheduled
+ downtime this weekend, so until Monday, an appropriate message-of-the-day is
+ added to the node definition.
+
+ When the ``motd`` application runs, it receives the appropriate
+ message-of-the-day (from ``quantum.example.org`` when run on that node) and
+ writes it into ``/etc/motd``.
+
+At this point it should be noted that parameters whose values are lists or
+key-value pairs don't get overwritten by children classes or node definitions,
+but the information gets merged (recursively) instead.
+
+Similarly to parameters, applications also accumulate during the recursive
+walk through the class ancestry. It is possible for a node or child class to
+*remove* an application added by a parent class, by prefixing the application
+with `~`.
+
+Finally, |reclass| happily lets you use multiple inheritance, and ensures that
+the resolution of parameters is still well-defined. Here's another example
+building upon the one about ``/etc/motd`` above:
+
+ ``quantum.example.org`` (which is back up and therefore its node definition
+ no longer contains a message-of-the-day) is at a site in Munich. Therefore,
+ it is a child of the class ``hosted@munich``. This class is independent of
+ the ``unixnode`` hierarchy, ``quantum.example.org`` derives from both.
+
+ In this example infrastructure, ``hosted@munich`` is more specific than
+ ``debiannode`` because there are plenty of Debian nodes at other sites (and
+ some non-Debian nodes in Munich). Therefore, ``quantum.example.org`` derives
+ from ``hosted@munich`` _after_ ``debiannodes``.
+
+ When an electricity outage is expected over the weekend in Munich, the admin
+ can change the message-of-the-day in the ``hosted@munich`` class, and it
+ will apply to all hosts in Munich.
+
+ However, not all hosts in Munich have ``/etc/motd``, because some of them
+ are of class ``windowsnode``. Since the ``windowsnode`` ancestry does not
+ specify the ``motd`` application, those hosts have access to the
+ message-of-the-day in the node variables, but the message won't get used…
+
+ … unless, of course, ``windowsnode`` specified a Windows-specific
+ application to bring such notices to the attention of the user.
+
+It's also trivial to ensure a certain order of class evaluation. Here's
+another example:
+
+ The ``ssh.server`` class defines the ``permit_root_login`` parameter to ``no``.
+
+ The ``backuppc.client`` class defines the parameter to ``without-password``,
+ because the BackupPC server might need to log in to the host as root.
+
+ Now, what happens if the admin accidentally provides the following two
+ classes?
+
+ - ``backuppc.client``
+ - ``ssh.server``
+
+ Theoretically, this would mean ``permit_root_login`` gets set to ``no``.
+
+ However, since all ``backuppc.client`` nodes need ``ssh.server`` (at least
+ in most setups), the class ``backuppc.client`` itself derives from
+ ``ssh.server``, ensuring that it gets parsed before ``backuppc.client``.
+
+ When |reclass| returns to the node and encounters the ``ssh.server`` class
+ defined there, it simply skips it, as it's already been processed.
+
+Now read about :doc:`operations`!
+
+.. include:: substs.inc
diff --git a/doc/source/conf.py b/doc/source/conf.py
new file mode 100644
index 0000000..422128e
--- /dev/null
+++ b/doc/source/conf.py
@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+#
+# reclass documentation build configuration file, created by
+# sphinx-quickstart on Mon Aug 26 12:56:14 2013.
+#
+# This file is execfile()d with the current directory set to its containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+import sys, os
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#sys.path.insert(0, os.path.abspath('.'))
+
+# -- General configuration -----------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be extensions
+# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.todo']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix of source filenames.
+source_suffix = '.rst'
+
+# The encoding of source files.
+#source_encoding = 'utf-8-sig'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'reclass'
+copyright = u'2013, martin f. krafft'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+import reclass.version
+# The short X.Y version.
+version = '.'.join(reclass.version.VERSION.split('.')[:2])
+# The full version, including alpha/beta/rc tags.
+release = reclass.version.VERSION
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#language = None
+
+# There are two options for replacing |today|: either, you set today to some
+# non-false value, then it is used:
+#today = ''
+# Else, today_fmt is used as the format for a strftime call.
+#today_fmt = '%B %d, %Y'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+exclude_patterns = []
+
+# The reST default role (used for this markup: `text`) to use for all documents.
+#default_role = None
+
+# If true, '()' will be appended to :func: etc. cross-reference text.
+#add_function_parentheses = True
+
+# If true, the current module name will be prepended to all description
+# unit titles (such as .. function::).
+#add_module_names = True
+
+# If true, sectionauthor and moduleauthor directives will be shown in the
+# output. They are ignored by default.
+#show_authors = False
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# A list of ignored prefixes for module index sorting.
+#modindex_common_prefix = []
+
+
+# -- Options for HTML output ---------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages. See the documentation for
+# a list of builtin themes.
+html_theme = 'default'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further. For a list of options available for each theme, see the
+# documentation.
+#html_theme_options = {}
+
+# Add any paths that contain custom themes here, relative to this directory.
+#html_theme_path = []
+
+# The name for this set of Sphinx documents. If None, it defaults to
+# "<project> v<release> documentation".
+#html_title = None
+
+# A shorter title for the navigation bar. Default is the same as html_title.
+html_short_title = 'reclass'
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+#html_logo = None
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
+# pixels large.
+#html_favicon = None
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+#html_static_path = ['_static']
+
+# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
+# using the given strftime format.
+#html_last_updated_fmt = '%b %d, %Y'
+
+# If true, SmartyPants will be used to convert quotes and dashes to
+# typographically correct entities.
+#html_use_smartypants = True
+
+# Custom sidebar templates, maps document names to template names.
+#html_sidebars = {}
+
+# Additional templates that should be rendered to pages, maps page names to
+# template names.
+#html_additional_pages = {}
+
+# If false, no module index is generated.
+#html_domain_indices = True
+
+# If false, no index is generated.
+#html_use_index = True
+
+# If true, the index is split into individual pages for each letter.
+#html_split_index = False
+
+# If true, links to the reST sources are added to the pages.
+html_show_sourcelink = False
+
+# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
+#html_show_sphinx = True
+
+# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
+#html_show_copyright = True
+
+# If true, an OpenSearch description file will be output, and all pages will
+# contain a <link> tag referring to it. The value of this option must be the
+# base URL from which the finished HTML is served.
+#html_use_opensearch = ''
+
+# This is the file name suffix for HTML files (e.g. ".xhtml").
+#html_file_suffix = None
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'reclassdoc'
+
+
+# -- Options for LaTeX output --------------------------------------------------
+
+latex_elements = {
+# The paper size ('letterpaper' or 'a4paper').
+'papersize': 'a4paper',
+
+# The font size ('10pt', '11pt' or '12pt').
+#'pointsize': '10pt',
+
+# Additional stuff for the LaTeX preamble.
+#'preamble': '',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title, author, documentclass [howto/manual]).
+latex_documents = [
+ ('index', 'reclass.tex', u'reclass Documentation',
+ u'martin f. krafft', 'manual'),
+]
+
+# The name of an image file (relative to this directory) to place at the top of
+# the title page.
+#latex_logo = None
+
+# For "manual" documents, if this is true, then toplevel headings are parts,
+# not chapters.
+#latex_use_parts = False
+
+# If true, show page references after internal links.
+#latex_show_pagerefs = False
+
+# If true, show URL addresses after external links.
+#latex_show_urls = False
+
+# Documents to append as an appendix to all manuals.
+#latex_appendices = []
+
+# If false, no module index is generated.
+#latex_domain_indices = True
+
+
+# -- Options for manual page output --------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+ ('manpage', 'reclass', u'command-line interface',
+ [u'martin f. krafft'], 1)
+]
+
+# If true, show URL addresses after external links.
+#man_show_urls = False
+
+
+# -- Options for Texinfo output ------------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+# dir menu entry, description, category)
+texinfo_documents = [
+ ('index', 'reclass', u'reclass Documentation',
+ u'martin f. krafft', 'reclass', 'One line description of project.',
+ 'Miscellaneous'),
+]
+
+# Documents to append as an appendix to all manuals.
+#texinfo_appendices = []
+
+# If false, no module index is generated.
+#texinfo_domain_indices = True
+
+# How to display URL addresses: 'footnote', 'no', or 'inline'.
+#texinfo_show_urls = 'footnote'
diff --git a/doc/source/configfile.rst b/doc/source/configfile.rst
new file mode 100644
index 0000000..497e6f7
--- /dev/null
+++ b/doc/source/configfile.rst
@@ -0,0 +1,33 @@
+==========================
+reclass configuration file
+==========================
+|reclass| can read some of its configuration from a file. The file is
+a YAML-file and simply defines key-value pairs.
+
+The configuration file can be used to set defaults for all the options that
+are otherwise configurable via the command-line interface, so please use the
+``--help`` output of |reclass| (or the :doc:`manual page <manpage>`) for
+reference. The command-line option ``--nodes-uri`` corresponds to the key
+``nodes_uri`` in the configuration file. For example::
+
+ storage_type: yaml_fs
+ pretty_print: True
+ output: json
+ inventory_base_uri: /etc/reclass
+ nodes_uri: ../nodes
+
+|reclass| first looks in the current directory for the file called
+``reclass-config.yml`` (see ``reclass/defaults.py``) and if no such file is
+found, it looks in ``$HOME``, then in ``/etc/reclass``, and then "next to" the
+``reclass`` script itself, i.e. if the script is symlinked to
+``/srv/provisioning/reclass``, then the the script will try to access
+``/srv/provisioning/reclass-config.yml``.
+
+Note that ``yaml_fs`` is currently the only supported ``storage_type``, and
+it's the default if you don't set it.
+
+Adapters may implement their own lookup logic, of course, so make sure to read
+their documentation (for :doc:`Salt <salt>`, for :doc:`Ansible <ansible>`, and
+for :doc:`Puppet <puppet>`).
+
+.. include:: substs.inc
diff --git a/doc/source/extrefs.inc b/doc/source/extrefs.inc
new file mode 100644
index 0000000..f0dddd4
--- /dev/null
+++ b/doc/source/extrefs.inc
@@ -0,0 +1,6 @@
+.. _Puppet: http://puppetlabs.com/puppet/puppet-open-source
+.. _Salt: http://saltstack.com/community
+.. _Ansible: http://www.ansibleworks.com
+.. _Hiera: http://projects.puppetlabs.com/projects/hiera
+.. _Artistic Licence 2.0: http://opensource.org/licenses/Artistic-2.0
+.. _Jinja2: http://jinja.pocoo.org
diff --git a/doc/source/hacking.rst b/doc/source/hacking.rst
new file mode 100644
index 0000000..89daaca
--- /dev/null
+++ b/doc/source/hacking.rst
@@ -0,0 +1,62 @@
+==================
+Hacking on reclass
+==================
+
+Installation
+------------
+If you just want to run |reclass| from source, e.g. because you are going to be
+making and testing changes, install it in "development mode"::
+
+ python setup.py develop
+
+Now the ``reclass`` script, as well as the adapters, will be available in
+``/usr/local/bin``, and you can also invoke them directly from the source
+tree.
+
+To uninstall::
+
+ python setup.py develop --uninstall
+
+Discussing reclass
+------------------
+If you want to talk about |reclass|, use the `mailing list`_ or to find me on
+IRC, in ``#reclass`` on ``irc.oftc.net``.
+
+.. _mailing list: http://lists.pantsfullofunix.net/listinfo/reclass
+
+Contributing to reclass
+-----------------------
+|reclass| is currently maintained `on Github
+<http://github.com/madduck/reclass>`_.
+
+Conttributions to |reclass| are very welcome. Since I prefer to keep a somewhat
+clean history, I will not just merge pull request.
+
+You can submit pull requests, of course, and I'll rebase them onto ``HEAD``
+before merging. Or send your patches using ``git-format-patch`` and
+``git-send-e-mail`` to `the mailing list
+<reclass@lists.pantsfullofunix.net>`_.
+
+I have added rudimentary unit tests, and it would be nice if you could submit
+your changes with appropriate changes to the tests. To run tests, invoke
+
+::
+
+ $ make tests
+
+in the top-level checkout directory. The tests are rather inconsistent, some
+using mock objects, and only the datatypes-related code is covered. If you are
+a testing expert, I could certainly use some help here to improve the
+consistency of the existing tests, as well as their coverage.
+
+Also, there is a Makefile giving access to PyLint and ``coverage.py`` (running
+tests). If you run that, you can see there is a lot of work to be done
+cleaning up the code. If this is the sort of stuff you want to do — by all
+means — be my guest! ;)
+
+There are a number of items on the :doc:`to-do list <todo>`, so if you are
+bored…
+
+If you have larger ideas, I'll be looking forward to discuss them with you.
+
+.. include:: substs.inc
diff --git a/doc/source/index.rst b/doc/source/index.rst
new file mode 100644
index 0000000..b077586
--- /dev/null
+++ b/doc/source/index.rst
@@ -0,0 +1,81 @@
+================================================
+reclass — Recursive external node classification
+================================================
+.. include:: intro.inc
+
+Releases and source code
+------------------------
+The latest released |reclass| version is |release|. Please have a look at the
+:doc:`change log <changelog>` for information about recent changes.
+
+For now, |reclass| is hosted `on Github`_, and you may clone it with the
+following command::
+
+ git clone https://github.com/madduck/reclass.git
+
+Please see the :doc:`install instructions <install>` for information about
+distribution packages and tarballs.
+
+.. _on Github: https://github.com/madduck/reclass
+
+Community
+---------
+There is a `mailing list`_, where you can bring up anything related to
+|reclass|.
+
+.. _mailing list: http://lists.pantsfullofunix.net/listinfo/reclass
+
+For real-time communication, please join the ``#reclass`` IRC channel on
+``irc.oftc.net``.
+
+If you're using `Salt`_, you can also ask your |reclass|-and-Salt-related
+questions on the mailing list, ideally specifying "reclass" in the subject of
+your message.
+
+Licence
+-------
+|reclass| is © 2007–2014 by martin f. krafft and released under the terms of
+the `Artistic Licence 2.0`_.
+
+Contents
+--------
+These documents aim to get you started with |reclass|:
+
+.. toctree::
+ :maxdepth: 2
+
+ install
+ concepts
+ operations
+ usage
+ refs
+ manpage
+ configfile
+ salt
+ ansible
+ puppet
+ hacking
+ todo
+ changelog
+
+About the name
+--------------
+"reclass" stands for **r**\ ecursive **e**\ xternal node **class**\ ifier,
+which is somewhat of a misnomer. I chose the name very early on, based on the
+recursive nature of the data merging. However, to the user, a better paradigm
+would be "hierarchical", as s/he does not and should not care too much about
+the implementation internals. By the time that I realised this, unfortunately,
+`Hiera`_ (Puppet-specific) had already occupied this prefix. Oh well. Once you
+start using |reclass|, you'll think recursively as well as hierarchically at
+the same time. It's really quite simple.
+
+..
+ Indices and tables
+ ==================
+
+ * :ref:`genindex`
+ * :ref:`modindex`
+ * :ref:`search`
+
+.. include:: extrefs.inc
+.. include:: substs.inc
diff --git a/doc/source/install.rst b/doc/source/install.rst
new file mode 100644
index 0000000..8affd9d
--- /dev/null
+++ b/doc/source/install.rst
@@ -0,0 +1,100 @@
+============
+Installation
+============
+
+For Debian users (including Ubuntu)
+-----------------------------------
+|reclass| has been `packaged for Debian`_. To use it, just install it with
+APT::
+
+ $ apt-get install reclass [reclass-doc]
+
+.. _packaged for Debian: http://packages.debian.org/search?keywords=reclass
+
+For ArchLinux users
+-------------------
+|reclass| is `available for ArchLinux`_, thanks to Niels Abspoel.
+Dowload the tarball_ from ``aur`` or ``yaourt``::
+
+ $ yaourt -S reclass
+
+or::
+
+ $ tar xvzf reclass-git.tar.gz
+ $ cd reclass-git; makepkg
+ $ sudo pacman -U reclass-git-<git-commit-hash>.tar.gz
+
+.. _available for ArchLinux: https://aur.archlinux.org/packages/reclass-git/
+.. _tarball: https://aur.archlinux.org/packages/re/reclass-git/reclass-git.tar.gz
+
+Other distributions
+-------------------
+Developers of other distributions are cordially invited to package |reclass|
+themselves and `write to the mailing list
+<mailto:reclass@pantsfullofunix.net>`_ to have details included here. Or send
+a patch!
+
+From source
+-----------
+|reclass| is currently maintained `on Github
+<http://github.com/madduck/reclass>`_, so to obtain the source, run::
+
+ $ git clone https://github.com/madduck/reclass.git
+
+or::
+
+ $ git clone ssh://git@github.com:madduck/reclass.git
+
+If you want a tarball, please `obtain it from the Debian archive`_.
+
+.. _obtain it from the Debian archive: http://http.debian.net/debian/pool/main/r/reclass/
+
+Before you can use |reclass|, you need to install it into a place where Python
+can find it. The following step should install the package to ``/usr/local``::
+
+ $ python setup.py install
+
+If you want to install to a different location, use --prefix like so::
+
+ $ python setup.py install --prefix=/opt/local
+
+.. todo::
+
+ These will install the ``reclass-salt`` and ``reclass-ansible`` adapters to
+ ``$prefix/bin``, but they should go to ``$prefix/share/reclass``. How can
+ setup.py be told to do so? It would be better for consistency if this was
+ done "upstream", rather than fixed by the distros.
+
+Just make sure that the destination is in the Python module search path, which
+you can check like this::
+
+ $ python -c 'import sys; print sys.path'
+
+More options can be found in the output of
+
+::
+
+ $ python setup.py install --help
+ $ python setup.py --help
+ $ python setup.py --help-commands
+ $ python setup.py --help [cmd]
+
+If you just want to run |reclass| from source, e.g. because you are going to be
+making and testing changes, install it in "development mode"::
+
+ $ python setup.py develop
+
+To uninstall (the rm call is necessary due to `a bug in setuptools`_)::
+
+ $ python setup.py develop --uninstall
+ $ rm /usr/local/bin/reclass*
+
+`Uninstallation currently isn't possible`_ for packages installed to
+/usr/local as per the above method, unfortunately. The following should do::
+
+ $ rm -r /usr/local/lib/python*/dist-packages/reclass* /usr/local/bin/reclass*
+
+.. _a bug in setuptools: http://bugs.debian.org/714960
+.. _Uninstallation currently isn't possible: http://bugs.python.org/issue4673
+
+.. include:: substs.inc
diff --git a/doc/source/intro.inc b/doc/source/intro.inc
new file mode 100644
index 0000000..975bac0
--- /dev/null
+++ b/doc/source/intro.inc
@@ -0,0 +1,25 @@
+|reclass| is an "external node classifier" (ENC) as can be used with
+automation tools, such as `Puppet`_, `Salt`_, and `Ansible`_. It is also
+a stand-alone tool for merging data sources recursively.
+
+The purpose of an ENC is to allow a system administrator to maintain an
+inventory of nodes to be managed, completely separately from the configuration
+of the automation tool. Usually, the external node classifier completely
+replaces the tool-specific inventory (such as ``site.pp`` for Puppet,
+``ext_pillar``/``master_tops`` for Salt, or ``/etc/ansible/hosts``).
+
+With respect to the configuration management tool, the ENC then fulfills two
+jobs:
+
+- it provides information about groups of nodes and group memberships
+- it gives access to node-specific information, such as variables
+
+|reclass| allows you to define your nodes through class inheritance, while
+always able to override details further up the tree (i.e. in more specific
+nodes). Think of classes as feature sets, as commonalities between nodes, or
+as tags. Add to that the ability to nest classes (multiple inheritance is
+allowed, well-defined, and encouraged), and you can assemble your
+infrastructure from smaller bits, eliminating duplication and exposing all
+important parameters to a single location, logically organised. And if that
+isn't enough, |reclass| lets you reference other parameters in the very
+hierarchy you are currently assembling.
diff --git a/doc/source/manpage.rst b/doc/source/manpage.rst
new file mode 100644
index 0000000..129a4ab
--- /dev/null
+++ b/doc/source/manpage.rst
@@ -0,0 +1,55 @@
+===============
+reclass manpage
+===============
+
+Synopsis
+--------
+| |reclass| --help
+| |reclass| *[options]* --inventory
+| |reclass| *[options]* --nodeinfo=NODENAME
+
+Description
+-----------
+.. include:: intro.inc
+
+|reclass| will be used indirectly through adapters most of the time. However,
+there exists a command-line interface that allows querying the database. This
+manual page describes this interface.
+
+Options
+-------
+Please see the output of ``reclass --help`` for the default values of these
+options:
+
+Database options
+''''''''''''''''
+-s, --storage-type The type of storage backend to use
+-b, --inventory-base-uri The base URI to prepend to nodes and classes
+-u, --nodes-uri The URI to the nodes storage
+-c, --classes-uri The URI to the classes storage
+
+Output options
+''''''''''''''
+-o, --output The output format to use (yaml or json)
+-y, --pretty-print Try to make the output prettier
+
+Modes
+'''''
+-i, --inventory Output the entire inventory
+-n, --nodeinfo Output information for a specific node
+
+Information
+'''''''''''
+-h, --help Help output
+--version Display version number
+
+See also
+--------
+Please visit http://reclass.pantsfullofunix.net/ for more information about
+|reclass|.
+
+The documentation is also available from the ``./doc`` subtree in the source
+checkout, or from ``/usr/share/doc/reclass-doc``.
+
+.. include:: substs.inc
+.. include:: extrefs.inc
diff --git a/doc/source/operations.rst b/doc/source/operations.rst
new file mode 100644
index 0000000..f744148
--- /dev/null
+++ b/doc/source/operations.rst
@@ -0,0 +1,151 @@
+==================
+reclass operations
+==================
+
+YAML FS storage
+---------------
+While |reclass| has been built to support different storage backends through
+plugins, currently only the ``yaml_fs`` storage backend exists. This is a very
+simple, yet powerful, YAML-based backend, using flat files on the filesystem
+(as suggested by the ``_fs`` postfix).
+
+``yaml_fs`` works with two directories, one for node definitions, and another
+for class definitions. The two directories must not be the same, nor can one
+be a parent of the other.
+
+Files in those directories are YAML-files, specifying key-value pairs. The
+following three keys are read by |reclass|:
+
+============ ================================================================
+Key Description
+============ ================================================================
+classes a list of parent classes
+appliations a list of applications to append to the applications defined by
+ ancestors. If an application name starts with ``~``, it would
+ remove this application from the list, if it had already been
+ added — but it does not prevent a future addition.
+ E.g. ``~firewalled``
+parameters key-value pairs to set defaults in class definitions, override
+ existing data, or provide node-specific information in node
+ specifications.
+ \
+ By convention, parameters corresponding to an application
+ should be provided as subkey-value pairs, keyed by the name of
+ the application, e.g.::
+
+ applications:
+ - ssh.server
+ parameters:
+ ssh.server:
+ permit_root_login: no
+environment only relevant for nodes, this allows to specify an "environment"
+ into which the node definition is supposed to be place.
+============ ================================================================
+
+Classes files may reside in subdirectories, which act as namespaces. For
+instance, a class ``ssh.server`` will result in the class definition to be
+read from ``ssh/server.yml``. Specifying just ``ssh`` will cause the class
+data to be read from ``ssh/init.yml`` or ``ssh.yml``. Note, however, that only
+one of those two may be present.
+
+Nodes may also be defined in subdirectories. However, node names (filename)
+must be unique across all subdirectories, and |reclass| will exit with an
+error if a node is defined multiple times. Subdirectories therefore really
+only exist for the administrator's local data structuring. They may be used in
+mappings (see below) to tag additional classes onto nodes.
+
+Data merging
+------------
+|reclass| has two modes of operation: node information retrieval and inventory
+listing. The second is really just a loop of the first across all defined
+nodes, and needs not be further described.
+
+When retrieving information about a node, |reclass| first obtains the node
+definition from the storage backend. Then, it iterates the list of classes
+defined for the node and recursively asks the storage backend for each class
+definition (unless already cached).
+
+Next, |reclass| recursively descends each class, looking at the classes it
+defines, and so on, until a leaf node is reached, i.e. a class that references
+no other classes.
+
+Now, the merging starts. At every step, the list of applications and the set
+of parameters at each level is merged into what has been accumulated so far.
+
+Merging of parameters is done "deeply", meaning that lists and dictionaries
+are extended (recursively), rather than replaced. However, a scalar value
+*does* overwrite a dictionary or list value. While the scalar could be
+appended to an existing list, there is no sane default assumption in the
+context of a dictionary, so this behaviour seems the most logical. Plus, it
+allows for a dictionary to be erased by overwriting it with the null value.
+
+After all classes (and the classes they reference) have been visited,
+|reclass| finally merges the applications list and parameters defined for the
+node into what has been accumulated during the processing of the classes, and
+returns the final result.
+
+Wildcard/Regexp mappings
+------------------------
+Using the :doc:`configuration file <configfile>`, it is also possible to
+provide a list mappings between node names and classes. For instance::
+
+ class_mappings:
+ - \* default
+ - /^www\d+/ webserver
+ - \*.ch hosted@switzerland another_class_to_show_that_it_can_take_lists
+
+This will assign the ``default`` class to all nodes (make sure to escape
+a leading asterisk (\*) to keep YAML happy), ``webserver`` to all nodes named
+``www1`` or ``www999``, and ``hosted-in-switzerland`` to all nodes whose names
+end with ``.ch`` (again, note the escaped leading asterisk). Multiple classes
+can be assigned to each mapping by providing a space-separated list (class
+names cannot contain spaces anyway).
+
+.. warning::
+
+ The class mappings do not really belong in the configuration file, as they
+ are data, not configuration inmformation. Therefore, they are likely going
+ to move elsewhere, but I have not quite figured out to where. Most likely,
+ there will be an additional file, specified in the configuration file, which
+ then lists the mappings.
+
+Note that mappings are not designed to replace node definitions. Mappings can
+be used to pre-populate the classes of existing nodes, but you still need to
+define all nodes (and if only to allow them to be enumerated for the
+inventory).
+
+The mapped classes can also contain backreferences when regular expressions
+are used, although they need to be escaped, e.g.::
+
+ class_mappings:
+ - /\.(\S+)$/ tld-\\1
+
+Furthermore, since the outer slashes ('/') are used to "quote" the regular
+expression, *any* slashes within the regular expression must be escaped. For
+instance, the following class mapping assigns a ``subdir-X`` class to all
+nodes that are defined in a subdirectory (using yaml_fs)::
+
+ class_mappings:
+ - /^([^\/]+)\// subdir-\\1
+
+Parameter interpolation
+------------------------
+Parameters may reference each other, including deep references, e.g.::
+
+ parameters:
+ location: Munich, Germany
+ motd:
+ header: This node sits in ${location}
+ for_demonstration: ${motd:header}
+ dict_reference: ${motd}
+
+After merging and interpolation, which happens automatically inside the
+storage modules, the ``for_demonstration`` parameter will have a value of
+"This node sits in Munich, Germany".
+
+Types are preserved if the value contains nothing but a reference. Hence, the
+value of ``dict_reference`` will actually be a dictionary.
+
+You should now be ready to :doc:`use reclass <usage>`!
+
+.. include:: substs.inc
diff --git a/doc/source/puppet.rst b/doc/source/puppet.rst
new file mode 100644
index 0000000..15dc0ce
--- /dev/null
+++ b/doc/source/puppet.rst
@@ -0,0 +1,16 @@
+=========================
+Using reclass with Puppet
+=========================
+
+.. todo::
+
+ The adapter between |reclass| and `Puppet`_ has not actually been written,
+ since I rage-quit using Puppet before the rewrite of |reclass|.
+
+ It should be trivial to do, and if you need it or are interested in working
+ on it, and you require assistance, please get in touch with me `on the
+ mailing list <mailto:reclass@pantsfullofunix.net>`_. Else just send the
+ patch!
+
+.. include:: extrefs.inc
+.. include:: substs.inc
diff --git a/doc/source/refs.rst b/doc/source/refs.rst
new file mode 100644
index 0000000..dc21b78
--- /dev/null
+++ b/doc/source/refs.rst
@@ -0,0 +1,28 @@
+===================
+External references
+===================
+
+* I `presented reclass`__ at `LCA 2014`_, which as been recorded:
+
+ * (Slides forthcoming)
+ * `Video recording`__
+
+__ http://linux.conf.au/schedule/30203/view_talk?day=wednesday
+__ http://mirror.linux.org.au/pub/linux.conf.au/2014/Wednesday/59-Hierarchical_infrastructure_description_for_your_system_management_needs_-_Martin_Krafft.mp4
+
+.. _LCA 2014: https://lca2014.linux.org.au
+
+* I gave `a talk about reclass`__ at `DebConf13`_, which has been recorded:
+
+ * `Slides`__
+ * Video recording: `high quality (ogv)`__ | `high quality (webm)`__ | `low(er) quality (ogv)`__
+
+__ http://penta.debconf.org/dc13_schedule/events/1048.en.html
+__ http://annex.debconf.org/debconf-share/debconf13/slides/reclass.pdf
+__ http://meetings-archive.debian.net/pub/debian-meetings/2013/debconf13/high/1048_Recursive_node_classification_for_system_automation.ogv
+__ http://meetings-archive.debian.net/pub/debian-meetings/2013/debconf13/webm-high/1048_Recursive_node_classification_for_system_automation.webm
+__ http://meetings-archive.debian.net/pub/debian-meetings/2013/debconf13/low/1048_Recursive_node_classification_for_system_automation.ogv
+
+.. _DebConf13: http://debconf13.debconf.org
+
+.. include:: substs.inc
diff --git a/doc/source/salt.rst b/doc/source/salt.rst
new file mode 100644
index 0000000..2743c35
--- /dev/null
+++ b/doc/source/salt.rst
@@ -0,0 +1,215 @@
+=======================
+Using reclass with Salt
+=======================
+
+.. warning::
+
+ You need Salt 0.17 to use `reclass`, as older versions do not include the
+ `reclass` adapter. You could use the ``cmd_yaml`` adapters, but at least for
+ ``ext_pillar``, they are currently not useable, as they `do not export the
+ minion ID to the command they run`_.
+
+.. _do not export the minion ID to the command they run:
+ https://github.com/saltstack/salt/issues/2276
+
+Quick start
+-----------
+The following steps should get you up and running quickly with |reclass| and
+`Salt`_. You will need to decide for yourself where to put your |reclass|
+inventory. This can be your first ``base`` ``file_root`` (the default), or it
+could be ``/etc/reclass``, or ``/srv/salt``. The following shall assume the
+latter.
+
+Or you can also just look into ``./examples/salt`` of your |reclass|
+checkout (``/usr/share/doc/examples/salt`` on Debian-systems), where the
+following steps have already been prepared.
+
+/…/reclass refers to the location of your |reclass| checkout.
+
+.. todo::
+
+ With |reclass| now in Debian, as well as installable from source, the
+ following should be checked for path consistency…
+
+#. Complete the installation steps described in the :doc:`installation section
+ <install>`.
+
+ Alternatively, you can also tell Salt via the master config file where to
+ look for |reclass|, but then you won't be able to interact with
+ |reclass| through the command line.
+
+#. Copy the two directories ``nodes`` and ``classes`` from the example
+ subdirectory in the |reclass| checkout to e.g. ``/srv/salt``.
+
+ It's handy to symlink |reclass|' Salt adapter itself to that directory::
+
+ $ ln -s /usr/share/reclass/reclass-salt /srv/salt/states/reclass
+
+ As you can now just inspect the data right there from the command line::
+
+ $ ./reclass --top
+
+ If you don't want to do this, you can also let |reclass| know where to
+ look for the inventory with the following contents in
+ ``$HOME/reclass-config.yml``::
+
+ storage_type: yaml_fs
+ base_inventory_uri: /srv/reclass
+
+ Or you can reuse the first entry of ``file_roots`` under ``base`` in the Salt
+ master config.
+
+ Note that ``yaml_fs`` is currently the only supported ``storage_type``, and
+ it's the default if you don't set it.
+
+#. Check out your inventory by invoking
+
+ ::
+
+ $ reclass-salt --top
+
+ which should return all the information about all defined nodes, which is
+ only ``localhost`` in the example. This is essentially the same information
+ that you would keep in your ``top.sls`` file.
+
+ If you symlinked the script to your inventory base directory, use
+
+ ::
+
+ $ ./reclass --top
+
+#. See the pillar information for ``localhost``::
+
+ $ reclass-salt --pillar localhost
+
+#. Now add |reclass| to ``/etc/salt/master``, like so::
+
+ reclass: &reclass
+ inventory_base_uri: /srv/salt
+ reclass_source_path: ~/code/reclass
+
+ master_tops:
+ […]
+ reclass: *reclass
+
+ ext_pillar:
+ - reclass: *reclass
+
+ .. warning::
+
+ When using ``ext_pillar`` and/or ``master_tops``, you should make sure
+ that your ``file_roots`` paths do not contain a ``top.sls`` file. Even
+ though they ought to be able to coexist, there are a few sharp edges
+ around at the moment, so beware!
+
+ If you did not install |reclass| (but you are running it from source),
+ you can either specify the source path like above, or you can add it to
+ ``PYTHONPATH`` before invoking the Salt master, to ensure that Python can
+ find it::
+
+ PYTHONPATH=/…/reclass /etc/init.d/salt-master restart
+
+#. Provided that you have set up ``localhost`` as a Salt minion, the following
+ commands should now return the same data as above, but processed through
+ salt::
+
+ $ salt localhost pillar.items # shows just the parameters
+ $ salt localhost state.show_top # shows only the states (applications)
+
+ Alternatively, if you don't have the Salt minion running yet::
+
+ $ salt-call pillar.items # shows just the parameters
+ $ salt-call state.show_top # shows only the states (applications)
+
+#. You can also invoke |reclass| directly, which gives a slightly different
+ view onto the same data, i.e. before it has been adapted for Salt::
+
+ $ reclass --inventory
+ $ reclass --nodeinfo localhost
+
+Configuration file and master configuration
+-------------------------------------------
+Even though the Salt adapter of |reclass| looks for and reads the
+:doc:`configuration file <configfile>`, a better means to pass information to
+the adapter is via Salt's master configuration file, as shown above. Not all
+configuration options can be passed this way (e.g. ``output`` is hardcoded to
+YAML, which Salt uses), but it *is* possible to specify :doc:`class mappings
+<operations>` next to all the storage-specific options.
+
+.. warning::
+
+ The Salt CLI adapter does *not* read Salt's master configuration, so if you
+ are calling ``reclass-salt`` from the command-line (the CLI exists for
+ debugging purposes, mainly), be aware that it will be run in a different
+ environment than when Salt queries reclass directly.
+
+Integration with Salt
+---------------------
+|reclass| hooks into Salt at two different points: ``master_tops`` and
+``ext_pillar``. For both, Salt provides plugins. These plugins need to know
+where to find |reclass|, so if |reclass| is not properly installed (but
+you are running it from source), make sure to export ``PYTHONPATH``
+accordingly before you start your Salt master, or specify the path in the
+master configuration file, as show above.
+
+Salt has no concept of "nodes", "applications", "parameters", and "classes".
+Therefore it is necessary to explain how those correspond to Salt. Crudely,
+the following mapping exists:
+
+================= ================
+|reclass| concept Salt terminology
+================= ================
+nodes hosts
+classes (none) [#nodegroups]_
+applications states
+parameters pillar
+environment environment
+================= ================
+
+.. [#nodegroups] See `Salt issue #5787`_ for steps into the direction of letting
+ |reclass| provide nodegroup information.
+
+.. _Salt issue #5787: https://github.com/saltstack/salt/issues/5787
+
+Whatever applications you define for a node will become states applicable to
+a host. If those applications are added via ancestor classes, then that's
+fine, but currently, Salt does not do anything with the classes ancestry.
+
+Similarly, all parameters that are collected and merged eventually end up in
+the pillar data of a specific node.
+
+The pillar data of a node include all the information about classes and
+applications, so you could theoretically use them to target your Salt calls at
+groups of nodes defined in the |reclass| inventory, e.g.
+
+::
+
+ salt -I __reclass__:classes:salt_minion test.ping
+
+Unfortunately, this does not work yet, please stay tuned, and let me know
+if you figure out a way. `Salt issue #5787`_ is also of relevance.
+
+Optionally, data from pillars that run before the |reclass| ``ext_pillar``
+(i.e. Salt's builtin ``pillar_roots``, as well as other ``ext_pillar`` modules
+listed before the ``reclass_adapter``) can be made available to |reclass|.
+Please use this with caution as referencing data from Salt in the inventory
+will make it harder or impossible to run |reclass| in other environments. This
+feature is therefore turned off by default and must be explicitly enabled in
+the Salt master configuration file, like this::
+
+ ext_pillar:
+ - reclass:
+ […]
+ propagate_pillar_data_to_reclass: True
+
+Unfortunately, to use this, currently you cannot use YAML references (i.e.
+``*reclass``) as shown above, as the ``master_tops`` subsystem does not accept
+this configuration parameter, and there seems to be no way to extend an alias.
+Specifically, the following is not possible — let me know if it is!::
+
+ ext_pillar:
+ - reclass: *reclass # WARNING: this does not work!
+ propagate_pillar_data_to_reclass: True
+
+.. include:: substs.inc
+.. include:: extrefs.inc
diff --git a/doc/source/substs.inc b/doc/source/substs.inc
new file mode 100644
index 0000000..0948883
--- /dev/null
+++ b/doc/source/substs.inc
@@ -0,0 +1 @@
+.. |reclass| replace:: **reclass**
diff --git a/doc/source/todo.rst b/doc/source/todo.rst
new file mode 100644
index 0000000..9a48d4b
--- /dev/null
+++ b/doc/source/todo.rst
@@ -0,0 +1,150 @@
+==================
+reclass to-do list
+==================
+
+Common set of classes
+---------------------
+A lot of the classes I have set up during the various stages of development of
+|reclass| are generic. It would probably be sensible to make them available as
+part of |reclass|, to give people a common baseline to work from, and to
+ensure a certain level of consistency between users.
+
+This could also provide a more realistic example to users on how to use
+|reclass|.
+
+Testing framework
+-----------------
+There is rudimentary testing in place, but it's inconsistent. I got
+side-tracked into discussions about the philosphy of mocking objects. This
+could all be fixed and unified.
+
+Also, storage, outputters, CLI and adapters have absolutely no tests yet…
+
+The testing framework should also incorporate the example classes mentioned
+above.
+
+Configurable file extension
+---------------------------
+Right now, ``.yml`` is hard-coded. This could be exported to the
+configuration file, or even given as a list, so that ``.yml`` and ``.yaml``
+can both be used.
+
+Actually, I don't think this is such a good idea. If we create too many
+options right now, it'll be harder to unify later. Please also see `issue #17
+<https://github.com/madduck/reclass/issues/17`_ for a discussion about this.
+
+Verbosity, debugging
+--------------------
+Verbose output and debug logging would be a very useful addition to help
+people understand what's going on, where data are being changed/merged, and to
+help solve problems.
+
+Data from CMS for interpolation
+-------------------------------
+Depending on the CMS in question, it would be nice if |reclass| had access to
+the host-specific data (facts, grains, etc.) and could use those in parameter
+interpolation. I can imagine this working for Salt, where the ``grains``
+dictionary (and results from previous external node classifiers) is made
+available to the external node classifiers, but I am not convinced this will
+be possible in Ansible and Puppet.
+
+On the other hand, providing CMS-specific data to reclass will make people
+depend on it, meaning Salt cannot be used with multiple tools anymore.
+
+The way to deal with that would be to map grains, facts, whatever the CMS
+calls them, to a shared naming scheme/taxonomy, but that's a painful task,
+I think. It would mean, however, that even templates could be shared between
+CMSs if they only use the data provided by reclass (i.e. grains/facts become
+pillar data).
+
+Membership information
+----------------------
+It would be nice if |reclass| could provide e.g. the Nagios master node with
+a list of clients that define it as their master. That would short-circuit
+Puppet's ``storeconfigs`` and Salt's ``mine``.
+
+The way I envision this currently is to provide something I call "inventory
+queries". For instance, the Nagios master node's reclass data would contain
+the following (``$[…]`` would denote interpolation to create a list, a bit
+like list comprehension)::
+
+ parameters:
+ nagios:
+ hosts: $[nagios:master == SELF.nodename]
+
+This would cause |reclass| to iterate the inventory and generate a list of all
+the nodes that define a parameter ``nagios:master`` whose value equals to the
+name of the current node.
+
+This could be greatly simplified. For instance, we could simply limit
+comparisons against the name of the current node and just specify
+
+::
+
+ $[nagios:master]
+
+which would be expanded to a list of node names whose pillar data includes
+``${nagios:master}`` that matches the current node's name.
+
+Or it could be made arbitrarily complex and flexible, e.g. any of the
+following::
+
+ $[nagios:master == SELF.nodename] # replace with nodename
+ $[nagios:master == SELF.nodename | nodename] # name replacement value
+ $[nagios:master == SELF.nodename | nagios:nodeid] # replace with pillar data
+ $[x:nagios:nodeid foreach x | x:nagios:master == SELF.nodename]
+ …
+
+I'd rather not code this up from scratch, so I am looking for ideas for reuse…
+
+Configuration file lookup improvements
+--------------------------------------
+Right now, the adapters and the CLI look for the :doc:`configuration file
+<configfile>` in a fixed set of locations. On of those derives from
+``OPT_INVENTORY_BASE_URI``, the default inventory base URI (``/etc/reclass``).
+This should probably be updated in case the user changes the URI.
+
+Furthermore, ``$CWD`` and ``~`` might not make a lot of sense in all
+use-cases.
+
+However, this might be better addressed by the following point:
+
+Adapter class hierarchy
+-----------------------
+At the moment, adapters are just imperative code. It might make more sense to
+wrap them in classes, which customise things like command-line and config file
+parsing.
+
+One nice way would be to generalise configuration file reading, integrate it
+with command-line parsing, and then allow the consumers (the adapters) to
+configure them, for instance, in the Salt adapter::
+
+ config_proxy = ConfigProxy()
+ config_proxy.set_configfile_search_path(['/etc/reclass', '/etc/salt'])
+ config_proxy.lock_config_option('output', 'yaml')
+
+The last call would effectively remove the ``--output`` config option from the
+CLI, and yield an error (or warning) if the option was encountered while
+parsing the configuration file.
+
+Furthermore, the class instances could become long-lived and keep a reference
+to a storage proxy, e.g. to prevent having to reload storage on every request.
+
+Node lists
+----------
+Class mappings are still experimental, and one of the reasons I am not too
+happy with them right now is that one would still need to provide node files
+for all nodes for ``inventory`` invocations. This is because class mappings
+can assign classes based on patterns or regular expressions, but it is not
+possible to turn a pattern or regular expression into a list of valid nodes.
+
+`Issue #9 <https://github.com/madduck/reclass/issues/9>`_ contains a lengthy
+discussion on this. At the moment, I am unsure what the best way forward is.
+
+Inventory filters
+-----------------
+As described in `issue #11 <https://github.com/madduck/reclass/issues/11>`_:
+provide a means to limit the enumeration of the inventory, according to node
+name patterns, or using classes white-/blacklists.
+
+.. include:: substs.inc
diff --git a/doc/source/usage.rst b/doc/source/usage.rst
new file mode 100644
index 0000000..7a6c1d8
--- /dev/null
+++ b/doc/source/usage.rst
@@ -0,0 +1,43 @@
+=============
+Using reclass
+=============
+.. todo::
+
+ With |reclass| now in Debian, as well as installable from source, the
+ following should be checked for path consistency…
+
+For information on how to use |reclass| directly, call ``reclass --help``
+and study the output, or have a look at its :doc:`manual page <manpage>`.
+
+The three options, ``--inventory-base-uri``, ``--nodes-uri``, and
+``--classes-uri`` together specify the location of the inventory. If the base
+URI is specified, then it is prepended to the other two URIs, unless they are
+absolute URIs. If these two URIs are not specified, they default to ``nodes``
+and ``classes``. Therefore, if your inventory is in ``/etc/reclass/nodes`` and
+``/etc/reclass/classes``, all you need to specify is the base URI as
+``/etc/reclass`` — which is actually the default (specified in
+``reclass/defaults.py``).
+
+If you've installed |reclass| from source as per the :doc:`installation
+instructions <install>`, try to run it from the source directory like this::
+
+ $ reclass -b examples/ --inventory
+ $ reclass -b examples/ --nodeinfo localhost
+
+This will make it use the data from ``examples/nodes`` and
+``examples/classes``, and you can surely make your own way from here.
+
+On Debian-systems, use the following::
+
+ $ reclass -b /usr/share/doc/reclass/examples/ --inventory
+ $ reclass -b /usr/share/doc/reclass/examples/ --nodeinfo localhost
+
+More commonly, however, use of |reclass| will happen indirectly, and through
+so-called adapters. The job of an adapter is to translate between different
+invocation paradigms, provide a sane set of default options, and massage the
+data from |reclass| into the format expected by the automation tool in use.
+Please have a look at the respective README files for these adapters, i.e.
+for :doc:`Salt <salt>`, for :doc:`Ansible <ansible>`, and for :doc:`Puppet
+<puppet>`.
+
+.. include:: substs.inc