Added support modules for schema operations
* Added support for backup
EME-13
Change-Id: I11cabb77bb66f3e9f1e35c9a00c9c196ded0ceb3
diff --git a/_modules/modelschema.py b/_modules/modelschema.py
new file mode 100644
index 0000000..3a67bc9
--- /dev/null
+++ b/_modules/modelschema.py
@@ -0,0 +1,224 @@
+
+from __future__ import absolute_import
+
+import glob
+import json
+import logging
+import os.path
+import yaml
+
+# Import third party libs
+try:
+ from jsonschema import validate as _validate
+ from jsonschema.validators import validator_for as _validator_for
+ from jsonschema.exceptions import SchemaError, ValidationError
+ HAS_JSONSCHEMA = True
+except ImportError:
+ HAS_JSONSCHEMA = False
+
+__virtualname__ = 'modelschema'
+
+LOG = logging.getLogger(__name__)
+
+
+def __virtual__():
+ """
+ Only load if jsonschema library exist.
+ """
+ if not HAS_JSONSCHEMA:
+ return (
+ False,
+ 'Can not load module jsonschema: jsonschema library not found')
+ return __virtualname__
+
+
+def _get_base_dir():
+ return __salt__['config.get']('pilllar_schema_path',
+ '/usr/share/salt-formulas/env')
+
+
+def _dict_deep_merge(a, b, path=None):
+ """
+ Merges dict(b) into dict(a)
+ """
+ if path is None:
+ path = []
+ for key in b:
+ if key in a:
+ if isinstance(a[key], dict) and isinstance(b[key], dict):
+ _dict_deep_merge(a[key], b[key], path + [str(key)])
+ elif a[key] == b[key]:
+ pass # same leaf value
+ else:
+ raise Exception(
+ 'Conflict at {}'.format('.'.join(path + [str(key)])))
+ else:
+ a[key] = b[key]
+ return a
+
+
+def schema_list():
+ """
+ Returns list of all defined schema files.
+
+ CLI Examples:
+
+ .. code-block:: bash
+
+ salt-call modelutils.schema_list
+
+
+ """
+ output = {}
+ schemas = glob.glob('{}/*/*/schemas/*.yaml'.format(_get_base_dir()))
+ for schema in schemas:
+ if os.path.exists(schema):
+ role_name = schema.split('/')[-1].replace('.yaml', '')
+ service_name = schema.split('/')[-3]
+ print role_name, service_name
+ name = '{}-{}'.format(service_name, role_name)
+ output[name] = {
+ 'service': service_name,
+ 'role': role_name,
+ 'path': schema,
+ 'valid': schema_validate(service_name, role_name)[name]
+ }
+ return output
+
+
+def schema_get(service, role):
+ """
+ Returns pillar schema for given service and role. If no service and role
+ is specified, method will return all known schemas.
+
+ CLI Examples:
+
+ .. code-block:: bash
+
+ salt-call modelutils.schema_get ntp server
+
+ """
+ schema_path = 'salt://{}/schemas/{}.yaml'.format(service, role)
+ schema = __salt__['cp.get_file_str'](schema_path)
+ if schema:
+ try:
+ data = yaml.safe_load(schema)
+ except yaml.YAMLError as exc:
+ raise Exception("Failed to parse schema:{}\n"
+ "{}".format(schema_path, exc))
+ else:
+ raise Exception("Schema not found:{}".format(schema_path))
+ return {'{}-{}'.format(service, role): data}
+
+
+def schema_validate(service, role):
+ """
+ Validates pillar schema itself of given service and role.
+
+ CLI Examples:
+
+ .. code-block:: bash
+
+ salt-call modelutils.schema_validate ntp server
+
+ """
+
+ schema = schema_get(service, role)['{}-{}'.format(service, role)]
+ cls = _validator_for(schema)
+ LOG.debug("Validating schema..")
+ try:
+ cls.check_schema(schema)
+ LOG.debug("Schema is valid")
+ data = 'Schema is valid'
+ except SchemaError as exc:
+ LOG.error("SchemaError:{}".format(exc))
+ data = repr(exc)
+ return {'{}-{}'.format(service, role): data}
+
+
+def model_validate(service=None, role=None):
+ """
+ Validates pillar metadata by schema for given service and role. If
+ no service and role is specified, method will validate all defined
+ services.
+
+ CLI Example:
+ .. code-block:: bash
+ salt-run modelschema.model_validate keystone server
+
+ """
+ schema = schema_get(service, role)['{}-{}'.format(service, role)]
+ model = __salt__['pillar.get']('{}:{}'.format(service, role))
+ try:
+ _validate(model, schema)
+ data = 'Model is valid'
+ except SchemaError as exc:
+ LOG.error("SchemaError:{}".format(exc))
+ data = repr(exc)
+ except ValidationError as exc:
+ LOG.error("ValidationError:{}\nInstance:{}\n"
+ "SchemaPath:{}".format(exc.message, exc.instance,
+ exc.schema_path))
+ raise Exception("ValidationError")
+ return {'{}-{}'.format(service, role): data}
+
+
+def data_validate(model, schema):
+ """
+ Validates model by given schema.
+
+ CLI Example:
+ .. code-block:: bash
+ salt-run modelschema.data_validate {'a': 'b'} {'a': 'b'}
+ """
+ try:
+ _validate(model, schema)
+ data = 'Model is valid'
+ except SchemaError as exc:
+ LOG.error("SchemaError:{}".format(exc))
+ data = str(exc)
+ except ValidationError as exc:
+ LOG.error("ValidationError:{}\nInstance:{}\n"
+ "SchemaPath:{}".format(exc.message, exc.instance,
+ exc.schema_path))
+ raise Exception("ValidationError")
+ return data
+
+
+def schema_from_tests(service):
+ """
+ Generate pillar schema skeleton for given service. Method iterates throught
+ test pillars and generates schema scaffold structure in JSON format that
+ can be passed to service like http://jsonschema.net/ to get the basic
+ schema for the individual roles of the service.
+
+ CLI Examples:
+
+ .. code-block:: bash
+
+ salt-call modelutils.schema_from_tests keystone
+ """
+ pillars = glob.glob(
+ '{}/{}/tests/pillar/*.sls'.format(_get_base_dir(), service))
+ raw_data = {}
+ for pillar in pillars:
+ if os.path.exists(pillar):
+ with open(pillar, 'r') as stream:
+ try:
+ data = yaml.load(stream)
+ except yaml.YAMLError as exc:
+ data = {}
+ LOG.error('{}: {}'.format(pillar, repr(exc)))
+ try:
+ _dict_deep_merge(raw_data, data)
+ except Exception as exc:
+ LOG.error('{}: {}'.format(pillar, repr(exc)))
+ if service not in raw_data.keys():
+ raise Exception(
+ "Could not find applicable data "
+ "for:{}\n at:{}".format(service, _get_base_dir()))
+ data = raw_data[service]
+ output = {}
+ for role_name, role in data.items():
+ output[role_name] = json.dumps(role)
+ return output
diff --git a/_runners/modelschema.py b/_runners/modelschema.py
new file mode 100644
index 0000000..1a82179
--- /dev/null
+++ b/_runners/modelschema.py
@@ -0,0 +1,72 @@
+
+# Import python libs
+import logging
+import glob
+import os
+import yaml
+
+# Import salt modules
+import salt.client
+
+# Import third party libs
+from jsonschema import validate
+from jsonschema.validators import validator_for
+from jsonschema.exceptions import SchemaError
+
+
+__virtualname__ = 'modelschema'
+
+LOG = logging.getLogger(__name__)
+
+BASE_DIR = '/usr/share/salt-formulas/env/_formulas'
+
+
+def _get_schemas():
+ '''
+ Method will return all known schemas.
+ '''
+ output = {}
+ schemas = glob.glob('{}/*/schemas/*.yaml'.format(BASE_DIR))
+ for schema in schemas:
+ if os.path.exists(schema):
+ filename = schema.split('/')[-1].replace('.yaml', '')
+ service_name, role_name = filename.split('-')
+ if service_name not in output:
+ output[service_name] = {}
+ with open(schema, 'r') as stream:
+ try:
+ data = yaml.load(stream)
+ except yaml.YAMLError as exc:
+ data = None
+ LOG.error(exc)
+ output[service_name][role_name] = data
+ return output
+
+
+def validate_node_model(target, service, role):
+ '''
+ Validates pillar by schema for given given minion, service and role.
+ If no service and role is specified, method will validate all
+ defined services on minion.
+
+ CLI Example:
+ .. code-block:: bash
+ salt-run modelschema.validate_node_model
+ '''
+ client = salt.client.LocalClient(__opts__['conf_file'])
+ schema = _get_schemas()[service][role]
+ result = {}
+
+ validator = validator_for(schema)
+ try:
+ validator.check_schema(schema)
+ except SchemaError as exception:
+ LOG.error(exception)
+ return result
+
+ minions = client.cmd(target, 'pillar.data', timeout=1)
+ for minion, pillar in minions.items():
+ model = pillar[service][role]
+ validation_result = validator(schema).validate(model)
+ result[minion] = validation_result
+ return result
diff --git a/salt/meta/backupninja.yml b/salt/meta/backupninja.yml
index 37979e7..ba2eea5 100644
--- a/salt/meta/backupninja.yml
+++ b/salt/meta/backupninja.yml
@@ -3,8 +3,7 @@
backup:
salt:
fs_includes:
- {%- if master.pillar.engine == 'reclass' or (master.pillar.engine == 'composite' and master.pillar.reclass is defined) %}
- - {{ master.pillar.get('reclass', {}).get('inventory_base_uri', '/srv/salt/reclass') }}
- {%- endif %}
+ - /srv/salt
+ - /etc/salt
fs_excludes: []
{%- endif %}
diff --git a/salt/schemas/minion.yaml b/salt/schemas/minion.yaml
new file mode 100644
index 0000000..3a20f77
--- /dev/null
+++ b/salt/schemas/minion.yaml
@@ -0,0 +1,26 @@
+%YAML 1.1
+---
+title: Salt minion role
+description: |
+ Salt service in client role.
+type: object
+additionalProperties: false
+
+properties:
+ enabled:
+ description: |
+ Enables the Salt minion role.
+ type: boolean
+ masters:
+ description: |
+ List of Salt masters to connect to.
+ type: array
+ items:
+ $ref: "#/definitions/master"
+
+definitions:
+ master:
+ description: |
+ Hostname or IP address of the masters server
+ type: string
+ format: hostname-ip