blob: e9880c578e5cdc8831ad1e91b24adfb1c60d07c6 [file] [log] [blame]
# -*- coding: utf-8 -*-
'''
Module for handling reclass metadata models.
'''
from __future__ import absolute_import
import io
import json
import logging
import os
import socket
import sys
import six
import yaml
import re
import urlparse
from string import Template
try:
from reclass import get_storage, output
from reclass.adapters.salt import ext_pillar
from reclass.core import Core
from reclass.config import find_and_read_configfile
from reclass.errors import ReclassException
HAS_RECLASS = True
except ImportError:
HAS_RECLASS = False
LOG = logging.getLogger(__name__)
def __virtual__():
'''
Only load this module if reclass
is installed on this minion.
'''
if not HAS_RECLASS:
return False, "'reclass' python library is unavailable"
return 'reclass'
def _deps(ret_classes=True, ret_errors=False):
'''
Returns classes if ret_classes=True, else returns soft_params if ret_classes=False
'''
defaults = find_and_read_configfile()
path = defaults.get('inventory_base_uri')
classes = {}
soft_params = {}
errors = []
# find classes
for root, dirs, files in os.walk(path):
# skip hidden files and folders in reclass dir
files = [f for f in files if not f[0] == '.']
dirs[:] = [d for d in dirs if not d[0] == '.']
# translate found init.yml to valid class name
if 'init.yml' in files:
class_file = root + '/' + 'init.yml'
class_name = class_file.replace(path, '')[:-9].replace('/', '.')
classes[class_name] = {'file': class_file}
for f in files:
if f.endswith('.yml') and f != 'init.yml':
class_file = root + '/' + f
class_name = class_file.replace(path, '')[:-4].replace('/', '.')
classes[class_name] = {'file': class_file}
# read classes
for class_name, params in classes.items():
LOG.debug("Processing:{}".format(params['file']))
with open(params['file'], 'r') as f:
# read raw data
raw = f.read()
pr = re.findall('\${_param:(.*?)}', raw)
if pr:
params['params_required'] = list(set(pr))
# load yaml
try:
data = yaml.load(raw)
except yaml.scanner.ScannerError as e:
errors.append(params['file'] + ' ' + str(e))
pass
if type(data) == dict:
if data.get('classes'):
params['includes'] = data.get('classes', [])
if data.get('parameters') and data['parameters'].get('_param'):
params['params_created'] = data['parameters']['_param']
if not(data.get('classes') or data.get('parameters')):
errors.append(params['file'] + ' ' + 'file missing classes and parameters')
else:
errors.append(params['file'] + ' ' + 'is not valid yaml')
if ret_classes:
return classes
elif ret_errors:
return errors
# find parameters and its usage
for class_name, params in classes.items():
for pn, pv in params.get('params_created', {}).items():
# create param if missing
if pn not in soft_params:
soft_params[pn] = {'created_at': {}, 'required_at': []}
# add created_at
if class_name not in soft_params[pn]['created_at']:
soft_params[pn]['created_at'][class_name] = pv
for pn in params.get('params_required', []):
# create param if missing
if pn not in soft_params:
soft_params[pn] = {'created_at': {}, 'required_at': []}
# add created_at
soft_params[pn]['required_at'].append(class_name)
return soft_params
def _get_nodes_dir():
defaults = find_and_read_configfile()
return defaults.get('nodes_uri') or \
os.path.join(defaults.get('inventory_base_uri'), 'nodes')
def _get_classes_dir():
defaults = find_and_read_configfile()
return os.path.join(defaults.get('inventory_base_uri'), 'classes')
def _get_cluster_dir():
classes_dir = _get_classes_dir()
return os.path.join(classes_dir, 'cluster')
def _get_node_meta(name, cluster="default", environment="prd", classes=None, parameters=None):
host_name = name.split('.')[0]
domain_name = '.'.join(name.split('.')[1:])
if classes == None:
meta_classes = []
else:
if isinstance(classes, six.string_types):
meta_classes = json.loads(classes)
else:
meta_classes = classes
if parameters == None:
meta_parameters = {}
else:
if isinstance(parameters, six.string_types):
meta_parameters = json.loads(parameters)
else:
# generate dict from OrderedDict
meta_parameters = {k: v for (k, v) in parameters.items()}
node_meta = {
'classes': meta_classes,
'parameters': {
'_param': meta_parameters,
'linux': {
'system': {
'name': host_name,
'domain': domain_name,
'cluster': cluster,
'environment': environment,
}
}
}
}
return node_meta
def soft_meta_list():
'''
Returns all defined soft metadata parameters.
CLI Examples:
.. code-block:: bash
salt '*' reclass.soft_meta_list
'''
return _deps(ret_classes=False)
def class_list():
'''
Returns list of all classes defined within reclass inventory.
CLI Examples:
.. code-block:: bash
salt '*' reclass.class_list
'''
return _deps(ret_classes=True)
def soft_meta_get(name):
'''
Returns single soft metadata parameter.
:param name: expects the following format: apt_mk_version
CLI Examples:
.. code-block:: bash
salt '*' reclass.soft_meta_get openstack_version
'''
soft_params = _deps(ret_classes=False)
if name in soft_params:
return {name: soft_params.get(name)}
else:
return {'Error': 'No param {0} found'.format(name)}
def class_get(name):
'''
Returns detailes information about class file in reclass inventory.
:param name: expects the following format classes.system.linux.repo
CLI Examples:
.. code-block:: bash
salt '*' reclass.class_get classes.system.linux.repo
'''
classes = _deps(ret_classes=True)
tmp_name = '.' + name
if tmp_name in classes:
return {name: classes.get(tmp_name)}
else:
return {'Error': 'No class {0} found'.format(name)}
def node_create(name, path=None, cluster="default", environment="prd", classes=None, parameters=None, **kwargs):
'''
Create a reclass node
:param name: new node FQDN
:param path: custom path in nodes for new node
:param classes: classes given to the new node
:param parameters: parameters given to the new node
:param environment: node's environment
:param cluster: node's cluster
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_create server.domain.com classes=[system.neco1, system.neco2]
salt '*' reclass.node_create namespace/test enabled=False
'''
ret = {}
node = node_get(name=name)
if node and not "Error" in node:
LOG.debug("node {0} exists".format(name))
ret[name] = node
return ret
host_name = name.split('.')[0]
domain_name = '.'.join(name.split('.')[1:])
node_meta = _get_node_meta(name, cluster, environment, classes, parameters)
LOG.debug(node_meta)
if path == None:
file_path = os.path.join(_get_nodes_dir(), name + '.yml')
else:
file_path = os.path.join(_get_nodes_dir(), path, name + '.yml')
with open(file_path, 'w') as node_file:
node_file.write(yaml.safe_dump(node_meta, default_flow_style=False))
return node_get(name)
def node_delete(name, **kwargs):
'''
Delete a reclass node
:params node: Node name
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_delete demo01.domain.com
salt '*' reclass.node_delete name=demo01.domain.com
'''
node = node_get(name=name)
if 'Error' in node:
return {'Error': 'Unable to retreive node'}
if node[name]['path'] == '':
file_path = os.path.join(_get_nodes_dir(), name + '.yml')
else:
file_path = os.path.join(_get_nodes_dir(), node[name]['path'], name + '.yml')
os.remove(file_path)
ret = 'Node {0} deleted'.format(name)
return ret
def node_get(name, path=None, **kwargs):
'''
Return a specific node
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_get host01.domain.com
salt '*' reclass.node_get name=host02.domain.com
'''
ret = {}
nodes = node_list(**kwargs)
if not name in nodes:
return {'Error': 'Error in retrieving node'}
ret[name] = nodes[name]
return ret
def node_list(**connection_args):
'''
Return a list of available nodes
CLI Example:
.. code-block:: bash
salt '*' reclass.node_list
'''
ret = {}
for root, sub_folders, files in os.walk(_get_nodes_dir()):
# skip hidden files and folders in reclass dir
files = [f for f in files if not f[0] == '.']
sub_folders[:] = [d for d in sub_folders if not d[0] == '.']
for fl in files:
file_path = os.path.join(root, fl)
with open(file_path, 'r') as file_handle:
file_read = yaml.load(file_handle.read())
file_data = file_read or {}
classes = file_data.get('classes', [])
parameters = file_data.get('parameters', {}).get('_param', {})
name = fl.replace('.yml', '')
host_name = name.split('.')[0]
domain_name = '.'.join(name.split('.')[1:])
path = root.replace(_get_nodes_dir()+'/', '')
ret[name] = {
'name': host_name,
'domain': domain_name,
'cluster': 'default',
'environment': 'prd',
'path': path,
'classes': classes,
'parameters': parameters
}
return ret
def _is_valid_ipv4_address(address):
try:
socket.inet_pton(socket.AF_INET, address)
except AttributeError:
try:
socket.inet_aton(address)
except socket.error:
return False
return address.count('.') == 3
except socket.error:
return False
return True
def _is_valid_ipv6_address(address):
try:
socket.inet_pton(socket.AF_INET6, address)
except socket.error:
return False
return True
def _get_grains(*args, **kwargs):
res = __salt__['saltutil.cmd'](tgt='*',
fun='grains.item',
arg=args,
**{'timeout': 10})
return res or {}
def _guess_host_from_target(network_grains, host, domain=' '):
'''
Guess minion ID from given host and domain arguments. Host argument can contain
hostname, FQDN, IPv4 or IPv6 addresses.
'''
key = None
value = None
if _is_valid_ipv4_address(host):
key = 'ipv4'
value = host
elif _is_valid_ipv6_address(host):
key = 'ipv6'
value = host
elif host.endswith(domain):
key = 'fqdn'
value = host
else:
key = 'fqdn'
value = '%s.%s' % (host, domain)
target = None
if network_grains and isinstance(network_grains, dict) and key and value:
for minion, grains in network_grains.items():
if grains.get('retcode', 1) == 0 and value in grains.get('ret', {}).get(key, ''):
target = minion
return target or host
def _interpolate_graph_data(graph_data, **kwargs):
new_nodes = []
network_grains = _get_grains('ipv4', 'ipv6', 'fqdn')
for node in graph_data:
if not node.get('relations', []):
node['relations'] = []
for relation in node.get('relations', []):
if not relation.get('status', None):
relation['status'] = 'unknown'
if relation.get('host_from_target', None):
host = _guess_host_from_target(network_grains, relation.pop('host_from_target'))
relation['host'] = host
if relation.get('host_external', None):
parsed_host_external = [urlparse.urlparse(item).netloc
for item
in relation.get('host_external', '').split(' ')
if urlparse.urlparse(item).netloc]
service = parsed_host_external[0] if parsed_host_external else ''
host = relation.get('service', '')
relation['host'] = host
del relation['host_external']
relation['service'] = service
host_list = [n.get('host', '') for n in graph_data + new_nodes]
service_list = [n.get('service', '') for n in graph_data + new_nodes if host in n.get('host', '')]
if host not in host_list or (host in host_list and service not in service_list):
new_node = {
'host': host,
'service': service,
'type': relation.get('type', ''),
'relations': []
}
new_nodes.append(new_node)
graph_data = graph_data + new_nodes
return graph_data
def _grain_graph_data(*args, **kwargs):
ret = _get_grains('salt:graph')
graph_data = []
for minion_ret in ret.values():
if minion_ret.get('retcode', 1) == 0:
graph_datum = minion_ret.get('ret', {}).get('salt:graph', [])
graph_data = graph_data + graph_datum
graph_nodes = _interpolate_graph_data(graph_data)
graph = {}
for node in graph_nodes:
if node.get('host') not in graph:
graph[node.get('host')] = {}
graph[node.pop('host')][node.pop('service')] = node
return {'graph': graph}
def _pillar_graph_data(*args, **kwargs):
graph = {}
nodes = inventory()
for node, node_data in nodes.items():
for role in node_data.get('roles', []):
if node not in graph:
graph[node] = {}
graph[node][role] = {'relations': []}
return {'graph': graph}
def graph_data(*args, **kwargs):
'''
Returns graph data for visualization app
CLI Examples:
.. code-block:: bash
salt-call reclass.graph_data
'''
pillar_data = _pillar_graph_data().get('graph')
grain_data = _grain_graph_data().get('graph')
for host, services in pillar_data.items():
for service, service_data in services.items():
grain_service = grain_data.get(host, {}).get(service, {})
service_data.update(grain_service)
graph = []
for host, services in pillar_data.items():
for service, service_data in services.items():
additional_data = {
'host': host,
'service': service,
'status': 'unknown'
}
service_data.update(additional_data)
graph.append(service_data)
for host, services in grain_data.items():
for service, service_data in services.items():
additional_data = {
'host': host,
'service': service,
'status': 'success'
}
service_data.update(additional_data)
host_list = [g.get('host', '') for g in graph]
service_list = [g.get('service', '') for g in graph if g.get('host') == host]
if host not in host_list or (host in host_list and service not in service_list):
graph.append(service_data)
return {'graph': graph}
def node_update(name, classes=None, parameters=None, **connection_args):
'''
Update a node metadata information, classes and parameters.
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_update name=nodename classes="[clas1, class2]" parameters="{param: value, another_param: another_value}"
'''
node = node_get(name=name)
if node.has_key('Error'):
return {'Error': 'Error in retrieving node'}
for name, values in node.items():
param = values.get('parameters', {})
path = values.get('path')
cluster = values.get('cluster')
environment = values.get('environment')
write_class = values.get('classes', [])
if parameters:
param.update(parameters)
if classes:
for classe in classes:
if not classe in write_class:
write_class.append(classe)
node_meta = _get_node_meta(name, cluster, environment, write_class, param)
LOG.debug(node_meta)
if path == None:
file_path = os.path.join(_get_nodes_dir(), name + '.yml')
else:
file_path = os.path.join(_get_nodes_dir(), path, name + '.yml')
with open(file_path, 'w') as node_file:
node_file.write(yaml.safe_dump(node_meta, default_flow_style=False))
return node_get(name)
def _get_node_classes(node_data, class_mapping_fragment):
classes = []
for value_tmpl_string in class_mapping_fragment.get('value_template', []):
value_tmpl = Template(value_tmpl_string.replace('<<', '${').replace('>>', '}'))
rendered_value = value_tmpl.safe_substitute(node_data)
classes.append(rendered_value)
for value in class_mapping_fragment.get('value', []):
classes.append(value)
return classes
def _get_params(node_data, class_mapping_fragment):
params = {}
for param_name, param in class_mapping_fragment.items():
value = param.get('value', None)
value_tmpl_string = param.get('value_template', None)
if value:
params.update({param_name: value})
elif value_tmpl_string:
value_tmpl = Template(value_tmpl_string.replace('<<', '${').replace('>>', '}'))
rendered_value = value_tmpl.safe_substitute(node_data)
params.update({param_name: rendered_value})
return params
def _validate_condition(node_data, expressions):
# allow string expression definition for single expression conditions
if isinstance(expressions, six.string_types):
expressions = [expressions]
result = []
for expression_tmpl_string in expressions:
expression_tmpl = Template(expression_tmpl_string.replace('<<', '${').replace('>>', '}'))
expression = expression_tmpl.safe_substitute(node_data)
if expression and expression == 'all':
result.append(True)
elif expression:
val_a = expression.split('__')[0]
val_b = expression.split('__')[2]
condition = expression.split('__')[1]
if condition == 'startswith':
result.append(val_a.startswith(val_b))
elif condition == 'equals':
result.append(val_a == val_b)
return all(result)
def node_classify(node_name, node_data={}, class_mapping={}, **kwargs):
'''
CLassify node by given class_mapping dictionary
:param node_name: node FQDN
:param node_data: dictionary of known informations about the node
:param class_mapping: dictionary of classes and parameters, with conditions
'''
# clean node_data
node_data = {k: v for (k, v) in node_data.items() if not k.startswith('__')}
classes = []
node_params = {}
cluster_params = {}
ret = {'node_create': '', 'cluster_param': {}}
for type_name, node_type in class_mapping.items():
valid = _validate_condition(node_data, node_type.get('expression', ''))
if valid:
gen_classes = _get_node_classes(node_data, node_type.get('node_class', {}))
classes = classes + gen_classes
gen_node_params = _get_params(node_data, node_type.get('node_param', {}))
node_params.update(gen_node_params)
gen_cluster_params = _get_params(node_data, node_type.get('cluster_param', {}))
cluster_params.update(gen_cluster_params)
if classes:
create_kwargs = {'name': node_name, 'path': '_generated', 'classes': classes, 'parameters': node_params}
ret['node_create'] = node_create(**create_kwargs)
for name, value in cluster_params.items():
ret['cluster_param'][name] = cluster_meta_set(name, value)
return ret
def validate_yaml():
'''
Returns list of all reclass YAML files that contain syntax
errors.
CLI Examples:
.. code-block:: bash
salt-call reclass.validate_yaml
'''
errors = _deps(ret_classes=False, ret_errors=True)
if errors:
ret = {'Errors': errors}
return ret
def validate_pillar(node_name=None, **kwargs):
'''
Validates whether the pillar of given node is in correct state.
If node is not specified it validates pillars of all known nodes.
Returns error message for every node with currupted metadata.
:param node_name: target minion ID
CLI Examples:
.. code-block:: bash
salt-call reclass.validate_pillar
salt-call reclass.validate_pillar minion-id
'''
if node_name is None:
ret={}
nodes = node_list(**kwargs)
for node_name, node in nodes.items():
ret.update(validate_pillar(node_name))
return ret
else:
defaults = find_and_read_configfile()
meta = ''
error = None
try:
pillar = ext_pillar(node_name, {}, defaults['storage_type'], defaults['inventory_base_uri'])
except (ReclassException, Exception) as e:
msg = "Validation failed in %s on %s" % (repr(e), node_name)
LOG.error(msg)
meta = {'Error': msg}
s = str(type(e))
if 'yaml.scanner.ScannerError' in s:
error = re.sub(r"\r?\n?$", "", repr(str(e)), 1)
else:
error = e.message
if 'Error' in meta:
ret = {node_name: error}
else:
ret = {node_name: ''}
return ret
def node_pillar(node_name, **kwargs):
'''
Returns pillar metadata for given node from reclass inventory.
:param node_name: target minion ID
CLI Examples:
.. code-block:: bash
salt-call reclass.node_pillar minion_id
'''
defaults = find_and_read_configfile()
pillar = ext_pillar(node_name, {}, defaults['storage_type'], defaults['inventory_base_uri'])
output = {node_name: pillar}
return output
def inventory(**connection_args):
'''
Get all nodes in inventory and their associated services/roles.
CLI Examples:
.. code-block:: bash
salt '*' reclass.inventory
'''
defaults = find_and_read_configfile()
storage = get_storage(defaults['storage_type'], _get_nodes_dir(), _get_classes_dir())
reclass = Core(storage, None)
nodes = reclass.inventory()["nodes"]
output = {}
for node in nodes:
service_classification = []
role_classification = []
for service in nodes[node]['parameters']:
if service not in ['_param', 'private_keys', 'public_keys', 'known_hosts']:
service_classification.append(service)
for role in nodes[node]['parameters'][service]:
if role not in ['_support', '_orchestrate', 'common']:
role_classification.append('%s.%s' % (service, role))
output[node] = {
'roles': role_classification,
'services': service_classification,
}
return output
def cluster_meta_list(file_name="overrides.yml", cluster="", **kwargs):
'''
List all cluster level soft metadata overrides.
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_list
'''
path = os.path.join(_get_cluster_dir(), cluster, file_name)
try:
with io.open(path, 'r') as file_handle:
meta_yaml = yaml.safe_load(file_handle.read())
meta = meta_yaml or {}
except Exception as e:
msg = "Unable to load cluster metadata YAML %s: %s" % (path, repr(e))
LOG.debug(msg)
meta = {'Error': msg}
return meta
def cluster_meta_delete(name, file_name="overrides.yml", cluster="", **kwargs):
'''
Delete cluster level soft metadata override entry.
:param name: name of the override entry (dictionary key)
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_delete foo
'''
ret = {}
path = os.path.join(_get_cluster_dir(), cluster, file_name)
meta = __salt__['reclass.cluster_meta_list'](path, **kwargs)
if 'Error' not in meta:
metadata = meta.get('parameters', {}).get('_param', {})
if name not in metadata:
return ret
del metadata[name]
try:
with io.open(path, 'w') as file_handle:
file_handle.write(unicode(yaml.safe_dump(meta, default_flow_style=False)))
except Exception as e:
msg = "Unable to save cluster metadata YAML: %s" % repr(e)
LOG.error(msg)
return {'Error': msg}
ret = 'Cluster metadata entry {0} deleted'.format(name)
return ret
def cluster_meta_set(name, value, file_name="overrides.yml", cluster="", **kwargs):
'''
Create cluster level metadata override entry.
:param name: name of the override entry (dictionary key)
:param value: value of the override entry (dictionary value)
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_set foo bar
'''
path = os.path.join(_get_cluster_dir(), cluster, file_name)
meta = __salt__['reclass.cluster_meta_list'](path, **kwargs)
if 'Error' not in meta:
if not meta:
meta = {'parameters': {'_param': {}}}
metadata = meta.get('parameters', {}).get('_param', {})
if name in metadata and metadata[name] == value:
return {name: 'Cluster metadata entry %s already exists and is in correct state' % name}
metadata.update({name: value})
try:
with io.open(path, 'w') as file_handle:
file_handle.write(unicode(yaml.safe_dump(meta, default_flow_style=False)))
except Exception as e:
msg = "Unable to save cluster metadata YAML %s: %s" % (path, repr(e))
LOG.error(msg)
return {'Error': msg}
return cluster_meta_get(name, path, **kwargs)
return meta
def cluster_meta_get(name, file_name="overrides.yml", cluster="", **kwargs):
'''
Get single cluster level override entry
:param name: name of the override entry (dictionary key)
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_get foo
'''
ret = {}
path = os.path.join(_get_cluster_dir(), cluster, file_name)
meta = __salt__['reclass.cluster_meta_list'](path, **kwargs)
metadata = meta.get('parameters', {}).get('_param', {})
if 'Error' in meta:
ret['Error'] = meta['Error']
elif name in metadata:
ret[name] = metadata.get(name)
return ret