_modules/modelschema.py

from __future__ import absolute_import

import glob
import json
import logging
import os.path
import yaml

# Import third party libs
try:
    from jsonschema import validate as _validate
    from jsonschema.validators import validator_for as _validator_for
    from jsonschema.exceptions import SchemaError, ValidationError
    HAS_JSONSCHEMA = True
except ImportError:
    HAS_JSONSCHEMA = False

__virtualname__ = 'modelschema'

LOG = logging.getLogger(__name__)


def __virtual__():
    """
    Only load if jsonschema library exist.
    """
    if not HAS_JSONSCHEMA:
        return (
            False,
            'Can not load module jsonschema: jsonschema library not found')
    return __virtualname__


def _get_base_dir():
    return __salt__['config.get']('pilllar_schema_path',
                                  '/usr/share/salt-formulas/env')


def _dict_deep_merge(a, b, path=None):
    """
    Merges dict(b) into dict(a)
    """
    if path is None:
        path = []
    for key in b:
        if key in a:
            if isinstance(a[key], dict) and isinstance(b[key], dict):
                _dict_deep_merge(a[key], b[key], path + [str(key)])
            elif a[key] == b[key]:
                pass  # same leaf value
            else:
                raise Exception(
                    'Conflict at {}'.format('.'.join(path + [str(key)])))
        else:
            a[key] = b[key]
    return a


def schema_list():
    """
    Returns list of all defined schema files.

    CLI Examples:

    .. code-block:: bash

        salt-call modelutils.schema_list


    """
    output = {}
    schemas = glob.glob('{}/*/schemas/*.yaml'.format(_get_base_dir()))
    for schema in schemas:
        if os.path.exists(schema):
            role_name = schema.split('/')[-1].replace('.yaml', '')
            service_name = schema.split('/')[-3]
            print role_name, service_name
            name = '{}-{}'.format(service_name, role_name)
            output[name] = {
                'service': service_name,
                'role': role_name,
                'path': schema,
                'valid': schema_validate(service_name, role_name)[name]
            }
    return output


def schema_get(service, role):
    """
    Returns pillar schema for given service and role. If no service and role
    is specified, method will return all known schemas.

    CLI Examples:

    .. code-block:: bash

        salt-call modelutils.schema_get ntp server

    """
    schema_path = 'salt://{}/schemas/{}.yaml'.format(service, role)
    schema = __salt__['cp.get_file_str'](schema_path)
    if schema:
        try:
            data = yaml.safe_load(schema)
        except yaml.YAMLError as exc:
            raise Exception("Failed to parse schema:{}\n"
                            "{}".format(schema_path, exc))
    else:
        raise Exception("Schema not found:{}".format(schema_path))
    return {'{}-{}'.format(service, role): data}


def schema_validate(service, role):
    """
    Validates pillar schema itself of given service and role.

    CLI Examples:

    .. code-block:: bash

        salt-call modelutils.schema_validate ntp server

    """

    schema = schema_get(service, role)['{}-{}'.format(service, role)]
    cls = _validator_for(schema)
    LOG.debug("Validating schema..")
    try:
        cls.check_schema(schema)
        LOG.debug("Schema is valid")
        data = 'Schema is valid'
    except SchemaError as exc:
        LOG.error("SchemaError:{}".format(exc))
        raise Exception("SchemaError")
    return {'{}-{}'.format(service, role): data}


def model_validate(service=None, role=None):
    """
    Validates pillar metadata by schema for given service and role. If
    no service and role is specified, method will validate all defined
    services.

    CLI Example:
    .. code-block:: bash
        salt-run modelschema.model_validate keystone server

    """
    schema = schema_get(service, role)['{}-{}'.format(service, role)]
    model = __salt__['pillar.get']('{}:{}'.format(service, role))
    try:
        _validate(model, schema)
        data = 'Model is valid'
    except SchemaError as exc:
        LOG.error("SchemaError:{}".format(exc))
        raise Exception("SchemaError")
    except ValidationError as exc:
        LOG.error("ValidationError:{}\nInstance:{}\n"
                  "Schema title:{}\n"
                  "SchemaPath:{}".format(exc.message,
                                         exc.instance,
                                         exc.schema.get(
                                             "title",
                                             "Schema title not set!"),
                                         exc.schema_path))
        raise Exception("ValidationError")
    return {'{}-{}'.format(service, role): data}


def data_validate(model, schema):
    """
    Validates model by given schema.

    CLI Example:
    .. code-block:: bash
        salt-run modelschema.data_validate {'a': 'b'} {'a': 'b'}
    """
    try:
        _validate(model, schema)
        data = 'Model is valid'
    except SchemaError as exc:
        LOG.error("SchemaError:{}".format(exc))
        raise Exception("SchemaError")
    except ValidationError as exc:
        LOG.error("ValidationError:{}\nInstance:{}\n"
                  "Schema title:{}\n"
                  "SchemaPath:{}".format(exc.message,
                                         exc.instance,
                                         exc.schema.get(
                                             "title",
                                             "Schema title not set!"),
                                         exc.schema_path))
        raise Exception("ValidationError")
    return data


def schema_from_tests(service):
    """
    Generate pillar schema skeleton for given service. Method iterates throught
    test pillars and generates schema scaffold structure in JSON format that
    can be passed to service like http://jsonschema.net/ to get the basic
    schema for the individual roles of the service.

    CLI Examples:

    .. code-block:: bash

        salt-call modelutils.schema_from_tests keystone
    """
    pillars = glob.glob(
        '{}/{}/tests/pillar/*.sls'.format(_get_base_dir(), service))
    raw_data = {}
    for pillar in pillars:
        if os.path.exists(pillar):
            with open(pillar, 'r') as stream:
                try:
                    data = yaml.load(stream)
                except yaml.YAMLError as exc:
                    data = {}
                    LOG.error('{}: {}'.format(pillar, repr(exc)))
            try:
                _dict_deep_merge(raw_data, data)
            except Exception as exc:
                LOG.error('{}: {}'.format(pillar, repr(exc)))
    if service not in raw_data.keys():
        LOG.error("Could not find applicable  data "
                  "for:{}\n at:{}".format(service, _get_base_dir()))
        raise Exception("DataError")

    data = raw_data[service]
    output = {}
    for role_name, role in data.items():
        output[role_name] = json.dumps(role)
    return output