blob: e5055aa1ce30d8e9ef78ff352b76f82ce761ee56 [file] [log] [blame]
# -*- coding: utf-8 -*-
'''
Module for handling reclass metadata models.
'''
from __future__ import absolute_import
import io
import json
import logging
import os
import socket
import sys
import six
import yaml
import re
from urlparse import urlparse
from reclass import get_storage, output
from reclass.adapters.salt import ext_pillar
from reclass.core import Core
from reclass.config import find_and_read_configfile
from string import Template
from reclass.errors import ReclassException
LOG = logging.getLogger(__name__)
def __virtual__():
'''
Only load this module if reclass
is installed on this minion.
'''
return 'reclass'
def _deps(ret_classes=True, ret_errors=False):
'''
Returns classes if ret_classes=True, else returns soft_params if ret_classes=False
'''
defaults = find_and_read_configfile()
path = defaults.get('inventory_base_uri')
classes = {}
soft_params = {}
errors = []
# find classes
for root, dirs, files in os.walk(path):
if 'init.yml' in files:
class_file = root + '/' + 'init.yml'
class_name = class_file.replace(path, '')[:-9].replace('/', '.')
classes[class_name] = {'file': class_file}
for f in files:
if f.endswith('.yml') and f != 'init.yml':
class_file = root + '/' + f
class_name = class_file.replace(path, '')[:-4].replace('/', '.')
classes[class_name] = {'file': class_file}
# read classes
for class_name, params in classes.items():
with open(params['file'], 'r') as f:
# read raw data
raw = f.read()
pr = re.findall('\${_param:(.*?)}', raw)
if pr:
params['params_required'] = list(set(pr))
# load yaml
try:
data = yaml.load(raw)
except yaml.scanner.ScannerError as e:
errors.append(params['file'] + ' ' + str(e))
pass
if type(data) == dict:
if data.get('classes'):
params['includes'] = data.get('classes', [])
if data.get('parameters') and data['parameters'].get('_param'):
params['params_created'] = data['parameters']['_param']
if not(data.get('classes') or data.get('parameters')):
errors.append(params['file'] + ' ' + 'file missing classes and parameters')
else:
errors.append(params['file'] + ' ' + 'is not valid yaml')
if ret_classes:
return classes
elif ret_errors:
return errors
# find parameters and its usage
for class_name, params in classes.items():
for pn, pv in params.get('params_created', {}).items():
# create param if missing
if pn not in soft_params:
soft_params[pn] = {'created_at': {}, 'required_at': []}
# add created_at
if class_name not in soft_params[pn]['created_at']:
soft_params[pn]['created_at'][class_name] = pv
for pn in params.get('params_required', []):
# create param if missing
if pn not in soft_params:
soft_params[pn] = {'created_at': {}, 'required_at': []}
# add created_at
soft_params[pn]['required_at'].append(class_name)
return soft_params
def _get_nodes_dir():
defaults = find_and_read_configfile()
return defaults.get('nodes_uri') or \
os.path.join(defaults.get('inventory_base_uri'), 'nodes')
def _get_classes_dir():
defaults = find_and_read_configfile()
return os.path.join(defaults.get('inventory_base_uri'), 'classes')
def _get_cluster_dir():
classes_dir = _get_classes_dir()
return os.path.join(classes_dir, 'cluster')
def _get_node_meta(name, cluster="default", environment="prd", classes=None, parameters=None):
host_name = name.split('.')[0]
domain_name = '.'.join(name.split('.')[1:])
if classes == None:
meta_classes = []
else:
if isinstance(classes, six.string_types):
meta_classes = json.loads(classes)
else:
meta_classes = classes
if parameters == None:
meta_parameters = {}
else:
if isinstance(parameters, six.string_types):
meta_parameters = json.loads(parameters)
else:
# generate dict from OrderedDict
meta_parameters = {k: v for (k, v) in parameters.items()}
node_meta = {
'classes': meta_classes,
'parameters': {
'_param': meta_parameters,
'linux': {
'system': {
'name': host_name,
'domain': domain_name,
'cluster': cluster,
'environment': environment,
}
}
}
}
return node_meta
def validate_yaml_syntax():
'''
Returns list of yaml files with syntax errors
CLI Examples:
.. code-block:: bash
salt '*' reclass.validate_yaml_syntax
'''
errors = _deps(ret_classes=False, ret_errors=True)
if errors:
ret = {'Errors': errors}
return ret
def soft_meta_list():
'''
Returns params list
CLI Examples:
.. code-block:: bash
salt '*' reclass.soft_meta_list
'''
return _deps(ret_classes=False)
def class_list():
'''
Returns classes list
CLI Examples:
.. code-block:: bash
salt '*' reclass.class_list
'''
return _deps(ret_classes=True)
def soft_meta_get(name):
'''
:param name: expects the following format: apt_mk_version
Returns detail of the params
CLI Examples:
.. code-block:: bash
salt '*' reclass.soft_meta_get apt_mk_version
'''
soft_params = _deps(ret_classes=False)
if name in soft_params:
return {name: soft_params.get(name)}
else:
return {'Error': 'No param {0} found'.format(name)}
def class_get(name):
'''
:param name: expects the following format classes.system.linux.repo
Returns detail data of the class
CLI Examples:
.. code-block:: bash
salt '*' reclass.class_get classes.system.linux.repo
'''
classes = _deps(ret_classes=True)
tmp_name = '.' + name
if tmp_name in classes:
return {name: classes.get(tmp_name)}
else:
return {'Error': 'No class {0} found'.format(name)}
def node_create(name, path=None, cluster="default", environment="prd", classes=None, parameters=None, **kwargs):
'''
Create a reclass node
:param name: new node FQDN
:param path: custom path in nodes for new node
:param classes: classes given to the new node
:param parameters: parameters given to the new node
:param environment: node's environment
:param cluster: node's cluster
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_create server.domain.com classes=[system.neco1, system.neco2]
salt '*' reclass.node_create namespace/test enabled=False
'''
ret = {}
node = node_get(name=name)
if node and not "Error" in node:
LOG.debug("node {0} exists".format(name))
ret[name] = node
return ret
host_name = name.split('.')[0]
domain_name = '.'.join(name.split('.')[1:])
node_meta = _get_node_meta(name, cluster, environment, classes, parameters)
LOG.debug(node_meta)
if path == None:
file_path = os.path.join(_get_nodes_dir(), name + '.yml')
else:
file_path = os.path.join(_get_nodes_dir(), path, name + '.yml')
with open(file_path, 'w') as node_file:
node_file.write(yaml.safe_dump(node_meta, default_flow_style=False))
return node_get(name)
def node_delete(name, **kwargs):
'''
Delete a reclass node
:params node: Node name
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_delete demo01.domain.com
salt '*' reclass.node_delete name=demo01.domain.com
'''
node = node_get(name=name)
if 'Error' in node:
return {'Error': 'Unable to retreive node'}
if node[name]['path'] == '':
file_path = os.path.join(_get_nodes_dir(), name + '.yml')
else:
file_path = os.path.join(_get_nodes_dir(), node[name]['path'], name + '.yml')
os.remove(file_path)
ret = 'Node {0} deleted'.format(name)
return ret
def node_get(name, path=None, **kwargs):
'''
Return a specific node
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_get host01.domain.com
salt '*' reclass.node_get name=host02.domain.com
'''
ret = {}
nodes = node_list(**kwargs)
if not name in nodes:
return {'Error': 'Error in retrieving node'}
ret[name] = nodes[name]
return ret
def node_list(**connection_args):
'''
Return a list of available nodes
CLI Example:
.. code-block:: bash
salt '*' reclass.node_list
'''
ret = {}
for root, sub_folders, files in os.walk(_get_nodes_dir()):
for fl in files:
file_path = os.path.join(root, fl)
with open(file_path, 'r') as file_handle:
file_read = yaml.load(file_handle.read())
file_data = file_read or {}
classes = file_data.get('classes', [])
parameters = file_data.get('parameters', {}).get('_param', [])
name = fl.replace('.yml', '')
host_name = name.split('.')[0]
domain_name = '.'.join(name.split('.')[1:])
path = root.replace(_get_nodes_dir()+'/', '')
ret[name] = {
'name': host_name,
'domain': domain_name,
'cluster': 'default',
'environment': 'prd',
'path': path,
'classes': classes,
'parameters': parameters
}
return ret
def _is_valid_ipv4_address(address):
try:
socket.inet_pton(socket.AF_INET, address)
except AttributeError:
try:
socket.inet_aton(address)
except socket.error:
return False
return address.count('.') == 3
except socket.error:
return False
return True
def _is_valid_ipv6_address(address):
try:
socket.inet_pton(socket.AF_INET6, address)
except socket.error:
return False
return True
def _get_grains(*args, **kwargs):
res = __salt__['saltutil.cmd'](tgt='*',
fun='grains.item',
arg=args,
**{'timeout': 10})
return res or {}
def _guess_host_from_target(network_grains, host, domain=' '):
'''
Guess minion ID from given host and domain arguments. Host argument can contain
hostname, FQDN, IPv4 or IPv6 addresses.
'''
key = None
value = None
if _is_valid_ipv4_address(host):
key = 'ipv4'
value = host
elif _is_valid_ipv6_address(host):
key = 'ipv6'
value = host
elif host.endswith(domain):
key = 'fqdn'
value = host
else:
key = 'fqdn'
value = '%s.%s' % (host, domain)
target = None
if network_grains and isinstance(network_grains, dict) and key and value:
for minion, grains in network_grains.items():
if grains.get('retcode', 1) == 0 and value in grains.get('ret', {}).get(key, ''):
target = minion
return target or host
def _interpolate_graph_data(graph_data, **kwargs):
new_nodes = []
network_grains = _get_grains('ipv4', 'ipv6', 'fqdn')
for node in graph_data:
if not node.get('relations', []):
node['relations'] = []
for relation in node.get('relations', []):
if not relation.get('status', None):
relation['status'] = 'unknown'
if relation.get('host_from_target', None):
host = _guess_host_from_target(network_grains, relation.pop('host_from_target'))
relation['host'] = host
if relation.get('host_external', None):
parsed_host_external = [urlparse(item).netloc
for item
in relation.get('host_external', '').split(' ')
if urlparse(item).netloc]
service = parsed_host_external[0] if parsed_host_external else ''
host = relation.get('service', '')
relation['host'] = host
del relation['host_external']
relation['service'] = service
host_list = [n.get('host', '') for n in graph_data + new_nodes]
service_list = [n.get('service', '') for n in graph_data + new_nodes if host in n.get('host', '')]
if host not in host_list or (host in host_list and service not in service_list):
new_node = {
'host': host,
'service': service,
'type': relation.get('type', ''),
'relations': []
}
new_nodes.append(new_node)
graph_data = graph_data + new_nodes
return graph_data
def _grain_graph_data(*args, **kwargs):
ret = _get_grains('salt:graph')
graph_data = []
for minion_ret in ret.values():
if minion_ret.get('retcode', 1) == 0:
graph_datum = minion_ret.get('ret', {}).get('salt:graph', [])
graph_data = graph_data + graph_datum
graph_nodes = _interpolate_graph_data(graph_data)
graph = {}
for node in graph_nodes:
if node.get('host') not in graph:
graph[node.get('host')] = {}
graph[node.pop('host')][node.pop('service')] = node
return {'graph': graph}
def _pillar_graph_data(*args, **kwargs):
graph = {}
nodes = inventory()
for node, node_data in nodes.items():
for role in node_data.get('roles', []):
if node not in graph:
graph[node] = {}
graph[node][role] = {'relations': []}
return {'graph': graph}
def graph_data(*args, **kwargs):
'''
Returns graph data for visualization app
CLI Examples:
.. code-block:: bash
salt-call reclass.graph_data
'''
pillar_data = _pillar_graph_data().get('graph')
grain_data = _grain_graph_data().get('graph')
for host, services in pillar_data.items():
for service, service_data in services.items():
grain_service = grain_data.get(host, {}).get(service, {})
service_data.update(grain_service)
graph = []
for host, services in pillar_data.items():
for service, service_data in services.items():
additional_data = {
'host': host,
'service': service,
'status': 'unknown'
}
service_data.update(additional_data)
graph.append(service_data)
for host, services in grain_data.items():
for service, service_data in services.items():
additional_data = {
'host': host,
'service': service,
'status': 'success'
}
service_data.update(additional_data)
host_list = [g.get('host', '') for g in graph]
service_list = [g.get('service', '') for g in graph if g.get('host') == host]
if host not in host_list or (host in host_list and service not in service_list):
graph.append(service_data)
return {'graph': graph}
def node_update(name, classes=None, parameters=None, **connection_args):
'''
Update a node metadata information, classes and parameters.
CLI Examples:
.. code-block:: bash
salt '*' reclass.node_update name=nodename classes="[clas1, class2]"
'''
node = node_get(name=name)
if not node.has_key('Error'):
node = node[name.split("/")[1]]
else:
return {'Error': 'Error in retrieving node'}
def _get_node_classes(node_data, class_mapping_fragment):
classes = []
for value_tmpl_string in class_mapping_fragment.get('value_template', []):
value_tmpl = Template(value_tmpl_string.replace('<<', '${').replace('>>', '}'))
rendered_value = value_tmpl.safe_substitute(node_data)
classes.append(rendered_value)
for value in class_mapping_fragment.get('value', []):
classes.append(value)
return classes
def _get_params(node_data, class_mapping_fragment):
params = {}
for param_name, param in class_mapping_fragment.items():
value = param.get('value', None)
value_tmpl_string = param.get('value_template', None)
if value:
params.update({param_name: value})
elif value_tmpl_string:
value_tmpl = Template(value_tmpl_string.replace('<<', '${').replace('>>', '}'))
rendered_value = value_tmpl.safe_substitute(node_data)
params.update({param_name: rendered_value})
return params
def _validate_condition(node_data, expressions):
# allow string expression definition for single expression conditions
if isinstance(expressions, six.string_types):
expressions = [expressions]
result = []
for expression_tmpl_string in expressions:
expression_tmpl = Template(expression_tmpl_string.replace('<<', '${').replace('>>', '}'))
expression = expression_tmpl.safe_substitute(node_data)
if expression and expression == 'all':
result.append(True)
elif expression:
val_a = expression.split('__')[0]
val_b = expression.split('__')[2]
condition = expression.split('__')[1]
if condition == 'startswith':
result.append(val_a.startswith(val_b))
elif condition == 'equals':
result.append(val_a == val_b)
return all(result)
def node_classify(node_name, node_data={}, class_mapping={}, **kwargs):
'''
CLassify node by given class_mapping dictionary
:param node_name: node FQDN
:param node_data: dictionary of known informations about the node
:param class_mapping: dictionary of classes and parameters, with conditions
'''
# clean node_data
node_data = {k: v for (k, v) in node_data.items() if not k.startswith('__')}
classes = []
node_params = {}
cluster_params = {}
ret = {'node_create': '', 'cluster_param': {}}
for type_name, node_type in class_mapping.items():
valid = _validate_condition(node_data, node_type.get('expression', ''))
if valid:
gen_classes = _get_node_classes(node_data, node_type.get('node_class', {}))
classes = classes + gen_classes
gen_node_params = _get_params(node_data, node_type.get('node_param', {}))
node_params.update(gen_node_params)
gen_cluster_params = _get_params(node_data, node_type.get('cluster_param', {}))
cluster_params.update(gen_cluster_params)
if classes:
create_kwargs = {'name': node_name, 'path': '_generated', 'classes': classes, 'parameters': node_params}
ret['node_create'] = node_create(**create_kwargs)
for name, value in cluster_params.items():
ret['cluster_param'][name] = cluster_meta_set(name, value)
return ret
def validate_node_params(node_name, **kwargs):
'''
Validates if pillar of a node is in correct state.
Returns error message only if error occurred.
:param node_name: target minion ID
CLI Examples:
.. code-block:: bash
salt-call reclass.validate_node_params minion_id
'''
defaults = find_and_read_configfile()
meta = ''
error = None
try:
pillar = ext_pillar(node_name, {}, defaults['storage_type'], defaults['inventory_base_uri'])
except (ReclassException, Exception) as e:
msg = "Validation failed in %s on %s" % (repr(e), node_name)
LOG.error(msg)
meta = {'Error': msg}
s = str(type(e))
if 'yaml.scanner.ScannerError' in s:
error = re.sub(r"\r?\n?$", "", repr(str(e)), 1)
else:
error = e.message
if 'Error' in meta:
ret = {node_name: error}
else:
ret = {node_name: ''}
return ret
def validate_nodes_params(**connection_args):
'''
Validates if pillar all known nodes is in correct state.
Returns error message for every node where problem occurred.
CLI Examples:
.. code-block:: bash
salt-call reclass.validate_nodes_params
'''
ret={}
nodes = node_list(**connection_args)
for node_name, node in nodes.items():
ret.update(validate_node_params(node_name))
return ret
def node_pillar(node_name, **kwargs):
'''
Returns pillar data for given minion from reclass inventory.
:param node_name: target minion ID
CLI Examples:
.. code-block:: bash
salt-call reclass.node_pillar minion_id
'''
defaults = find_and_read_configfile()
pillar = ext_pillar(node_name, {}, defaults['storage_type'], defaults['inventory_base_uri'])
output = {node_name: pillar}
return output
def inventory(**connection_args):
'''
Get all nodes in inventory and their associated services/roles classification.
CLI Examples:
.. code-block:: bash
salt '*' reclass.inventory
'''
defaults = find_and_read_configfile()
storage = get_storage(defaults['storage_type'], _get_nodes_dir(), _get_classes_dir())
reclass = Core(storage, None)
nodes = reclass.inventory()["nodes"]
output = {}
for node in nodes:
service_classification = []
role_classification = []
for service in nodes[node]['parameters']:
if service not in ['_param', 'private_keys', 'public_keys', 'known_hosts']:
service_classification.append(service)
for role in nodes[node]['parameters'][service]:
if role not in ['_support', '_orchestrate', 'common']:
role_classification.append('%s.%s' % (service, role))
output[node] = {
'roles': role_classification,
'services': service_classification,
}
return output
def cluster_meta_list(file_name="overrides.yml", cluster="", **kwargs):
'''
List all cluster level overrides
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_list
'''
path = os.path.join(_get_cluster_dir(), cluster, file_name)
try:
with io.open(path, 'r') as file_handle:
meta_yaml = yaml.safe_load(file_handle.read())
meta = meta_yaml or {}
except Exception as e:
msg = "Unable to load cluster metadata YAML %s: %s" % (path, repr(e))
LOG.debug(msg)
meta = {'Error': msg}
return meta
def cluster_meta_delete(name, file_name="overrides.yml", cluster="", **kwargs):
'''
Delete cluster level override entry
:param name: name of the override entry (dictionary key)
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_delete foo
'''
ret = {}
path = os.path.join(_get_cluster_dir(), cluster, file_name)
meta = __salt__['reclass.cluster_meta_list'](path, **kwargs)
if 'Error' not in meta:
metadata = meta.get('parameters', {}).get('_param', {})
if name not in metadata:
return ret
del metadata[name]
try:
with io.open(path, 'w') as file_handle:
file_handle.write(unicode(yaml.dump(meta, default_flow_style=False)))
except Exception as e:
msg = "Unable to save cluster metadata YAML: %s" % repr(e)
LOG.error(msg)
return {'Error': msg}
ret = 'Cluster metadata entry {0} deleted'.format(name)
return ret
def cluster_meta_set(name, value, file_name="overrides.yml", cluster="", **kwargs):
'''
Create cluster level override entry
:param name: name of the override entry (dictionary key)
:param value: value of the override entry (dictionary value)
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_set foo bar
'''
path = os.path.join(_get_cluster_dir(), cluster, file_name)
meta = __salt__['reclass.cluster_meta_list'](path, **kwargs)
if 'Error' not in meta:
if not meta:
meta = {'parameters': {'_param': {}}}
metadata = meta.get('parameters', {}).get('_param', {})
if name in metadata and metadata[name] == value:
return {name: 'Cluster metadata entry %s already exists and is in correct state' % name}
metadata.update({name: value})
try:
with io.open(path, 'w') as file_handle:
file_handle.write(unicode(yaml.dump(meta, default_flow_style=False)))
except Exception as e:
msg = "Unable to save cluster metadata YAML %s: %s" % (path, repr(e))
LOG.error(msg)
return {'Error': msg}
return cluster_meta_get(name, path, **kwargs)
return meta
def cluster_meta_get(name, file_name="overrides.yml", cluster="", **kwargs):
'''
Get single cluster level override entry
:param name: name of the override entry (dictionary key)
:param file_name: name of the override file, defaults to: overrides.yml
CLI Examples:
.. code-block:: bash
salt-call reclass.cluster_meta_get foo
'''
ret = {}
path = os.path.join(_get_cluster_dir(), cluster, file_name)
meta = __salt__['reclass.cluster_meta_list'](path, **kwargs)
metadata = meta.get('parameters', {}).get('_param', {})
if 'Error' in meta:
ret['Error'] = meta['Error']
elif name in metadata:
ret[name] = metadata.get(name)
return ret