blob: e279b702964f4fbdd523a6c971d8cfe9ed049066 [file] [log] [blame]
# -*- coding: utf-8 -*-
'''
Module for handling maas calls.
:optdepends: pyapi-maas Python adapter
:configuration: This module is not usable until the following are specified
either in a pillar or in the minion's config file::
maas.url: 'https://maas.domain.com/'
maas.token: fdsfdsdsdsfa:fsdfae3fassd:fdsfdsfsafasdfsa
'''
from __future__ import absolute_import
import collections
import copy
import hashlib
import io
import json
import logging
import os.path
import time
import urllib2
LOG = logging.getLogger(__name__)
# Import third party libs
HAS_MASS = False
try:
from maas_client import MAASClient, MAASDispatcher, MAASOAuth
HAS_MASS = True
except ImportError:
LOG.debug('Missing python-oauth module. Skipping')
def __virtual__():
'''
Only load this module if maas-client
is installed on this minion.
'''
if HAS_MASS:
return 'maas'
return False
APIKEY_FILE = '/var/lib/maas/.maas_credentials'
STATUS_NAME_DICT = dict([
(0, 'New'), (1, 'Commissioning'), (2, 'Failed commissioning'),
(3, 'Missing'), (4, 'Ready'), (5, 'Reserved'), (10, 'Allocated'),
(9, 'Deploying'), (6, 'Deployed'), (7, 'Retired'), (8, 'Broken'),
(11, 'Failed deployment'), (12, 'Releasing'),
(13, 'Releasing failed'), (14, 'Disk erasing'),
(15, 'Failed disk erasing')])
def _format_data(data):
class Lazy:
def __str__(self):
return ' '.join(['{0}={1}'.format(k, v)
for k, v in data.iteritems()])
return Lazy()
def _create_maas_client():
global APIKEY_FILE
try:
api_token = file(APIKEY_FILE).read().splitlines()[-1].strip()\
.split(':')
except:
LOG.exception('token')
auth = MAASOAuth(*api_token)
api_url = 'http://localhost:5240/MAAS'
dispatcher = MAASDispatcher()
return MAASClient(auth, dispatcher, api_url)
class MaasObject(object):
def __init__(self):
self._maas = _create_maas_client()
self._extra_data_urls = {}
self._extra_data = {}
self._update = False
self._element_key = 'name'
self._update_key = 'id'
def send(self, data):
LOG.info('%s %s', self.__class__.__name__.lower(), _format_data(data))
if self._update:
return self._maas.put(
self._update_url.format(data[self._update_key]), **data).read()
if isinstance(self._create_url, tuple):
return self._maas.post(self._create_url[0].format(**data),
*self._create_url[1:], **data).read()
return self._maas.post(self._create_url.format(**data),
None, **data).read()
def process(self, objects_name=None):
ret = {
'success': [],
'errors': {},
'updated': [],
}
try:
config = __salt__['config.get']('maas')
for part in self._config_path.split('.'):
config = config.get(part, {})
extra = {}
for name, url_call in self._extra_data_urls.iteritems():
key = 'id'
key_name = 'name'
if isinstance(url_call, tuple):
if len(url_call) == 2:
url_call, key = url_call[:]
else:
url_call, key, key_name = url_call[:]
json_res = json.loads(self._maas.get(url_call).read())
if key:
extra[name] = {v[key_name]: v[key] for v in json_res}
else:
extra[name] = {v[key_name]: v for v in json_res}
if self._all_elements_url:
all_elements = {}
elements = self._maas.get(self._all_elements_url).read()
res_json = json.loads(elements)
for element in res_json:
if isinstance(element, (str, unicode)):
all_elements[element] = {}
else:
all_elements[element[self._element_key]] = element
else:
all_elements = {}
def process_single(name, config_data):
self._update = False
try:
data = self.fill_data(name, config_data, **extra)
if data is None:
ret['updated'].append(name)
return
if name in all_elements:
self._update = True
data = self.update(data, all_elements[name])
self.send(data)
ret['updated'].append(name)
else:
self.send(data)
ret['success'].append(name)
except urllib2.HTTPError as e:
etxt = e.read()
LOG.error('Failed for object %s reason %s', name, etxt)
ret['errors'][name] = str(etxt)
except Exception as e:
LOG.error('Failed for object %s reason %s', name, e)
ret['errors'][name] = str(e)
if objects_name is not None:
if ',' in objects_name:
objects_name = objects_name.split(',')
else:
objects_name = [objects_name]
for object_name in objects_name:
process_single(object_name, config[object_name])
else:
for name, config_data in config.iteritems():
process_single(name, config_data)
except Exception as e:
LOG.exception('Error Global')
raise
if ret['errors']:
raise Exception(ret)
return ret
class Fabric(MaasObject):
def __init__(self):
super(Fabric, self).__init__()
self._all_elements_url = u'api/2.0/fabrics/'
self._create_url = u'api/2.0/fabrics/'
self._update_url = u'api/2.0/fabrics/{0}/'
self._config_path = 'region.fabrics'
def fill_data(self, name, fabric):
data = {
'name': name,
'description': fabric.get('description', ''),
}
if 'class_type' in fabric:
data['class_type'] = fabric.get('class_type'),
return data
def update(self, new, old):
new['id'] = str(old['id'])
return new
class Subnet(MaasObject):
def __init__(self):
super(Subnet, self).__init__()
self._all_elements_url = u'api/2.0/subnets/'
self._create_url = u'api/2.0/subnets/'
self._update_url = u'api/2.0/subnets/{0}/'
self._config_path = 'region.subnets'
self._extra_data_urls = {'fabrics': u'api/2.0/fabrics/'}
def fill_data(self, name, subnet, fabrics):
data = {
'name': name,
'fabric': str(fabrics[subnet.get('fabric', '')]),
'cidr': subnet.get('cidr'),
'gateway_ip': subnet['gateway_ip'],
}
self._iprange = subnet['iprange']
return data
def update(self, new, old):
new['id'] = str(old['id'])
return new
def send(self, data):
response = super(Subnet, self).send(data)
res_json = json.loads(response)
self._process_iprange(res_json['id'])
return response
def _process_iprange(self, subnet_id):
ipranges = json.loads(self._maas.get(u'api/2.0/ipranges/').read())
LOG.warn('all %s ipranges %s', subnet_id, ipranges)
update = False
old_data = None
for iprange in ipranges:
if iprange['subnet']['id'] == subnet_id:
update = True
old_data = iprange
break
data = {
'start_ip': self._iprange.get('start'),
'end_ip': self._iprange.get('end'),
'subnet': str(subnet_id),
'type': self._iprange.get('type', 'dynamic')
}
LOG.warn('INFO: %s\n OLD: %s', data, old_data)
LOG.info('iprange %s', _format_data(data))
if update:
LOG.warn('UPDATING %s %s', data, old_data)
self._maas.put(u'api/2.0/ipranges/{0}/'.format(old_data['id']),
**data)
else:
self._maas.post(u'api/2.0/ipranges/', None, **data)
class DHCPSnippet(MaasObject):
def __init__(self):
super(DHCPSnippet, self).__init__()
self._all_elements_url = u'api/2.0/dhcp-snippets/'
self._create_url = u'api/2.0/dhcp-snippets/'
self._update_url = u'api/2.0/dhcp-snippets/{0}/'
self._config_path = 'region.dhcp_snippets'
self._extra_data_urls = {'subnets': u'api/2.0/subnets/'}
def fill_data(self, name, snippet, subnets):
data = {
'name': name,
'value': snippet['value'],
'description': snippet['description'],
'enabled': str(snippet['enabled'] and 1 or 0),
'subnet': str(subnets[snippet['subnet']]),
}
return data
def update(self, new, old):
new['id'] = str(old['id'])
return new
class PacketRepository(MaasObject):
def __init__(self):
super(PacketRepository, self).__init__()
self._all_elements_url = u'api/2.0/package-repositories/'
self._create_url = u'api/2.0/package-repositories/'
self._update_url = u'api/2.0/package-repositories/{0}/'
self._config_path = 'region.package_repositories'
def fill_data(self, name, package_repository):
data = {
'name': name,
'url': package_repository['url'],
'distributions': package_repository['distributions'],
'components': package_repository['components'],
'arches': package_repository['arches'],
'key': package_repository['key'],
'enabled': str(package_repository['enabled'] and 1 or 0),
}
if 'disabled_pockets' in package_repository:
data['disabled_pockets'] = package_repository['disable_pockets'],
return data
def update(self, new, old):
new['id'] = str(old['id'])
return new
class Device(MaasObject):
def __init__(self):
super(Device, self).__init__()
self._all_elements_url = u'api/2.0/devices/'
self._create_url = u'api/2.0/devices/'
self._update_url = u'api/2.0/devices/{0}/'
self._config_path = 'region.devices'
self._element_key = 'hostname'
self._update_key = 'system_id'
def fill_data(self, name, device_data):
data = {
'mac_addresses': device_data['mac'],
'hostname': name,
}
self._interface = device_data['interface']
return data
def update(self, new, old):
old_macs = set(v['mac_address'].lower() for v in old['interface_set'])
if new['mac_addresses'].lower() not in old_macs:
self._update = False
LOG.info('Mac changed deleting old device %s', old['system_id'])
self._maas.delete(u'api/2.0/devices/{0}/'.format(old['system_id']))
else:
new[self._update_key] = str(old[self._update_key])
return new
def send(self, data):
response = super(Device, self).send(data)
resp_json = json.loads(response)
system_id = resp_json['system_id']
iface_id = resp_json['interface_set'][0]['id']
self._link_interface(system_id, iface_id)
return response
def _link_interface(self, system_id, interface_id):
data = {
'mode': self._interface.get('mode', 'STATIC'),
'subnet': self._interface['subnet'],
'ip_address': self._interface['ip_address'],
}
if 'default_gateway' in self._interface:
data['default_gateway'] = self._interface.get('default_gateway')
if self._update:
data['force'] = '1'
LOG.info('interfaces link_subnet %s %s %s', system_id, interface_id,
_format_data(data))
self._maas.post(u'/api/2.0/nodes/{0}/interfaces/{1}/'
.format(system_id, interface_id), 'link_subnet',
**data)
class Machine(MaasObject):
def __init__(self):
super(Machine, self).__init__()
self._all_elements_url = u'api/2.0/machines/'
self._create_url = u'api/2.0/machines/'
self._update_url = u'api/2.0/machines/{0}/'
self._config_path = 'region.machines'
self._element_key = 'hostname'
self._update_key = 'system_id'
def fill_data(self, name, machine_data):
power_data = machine_data['power_parameters']
data = {
'hostname': name,
'architecture': machine_data.get('architecture', 'amd64/generic'),
'mac_addresses': machine_data['interface']['mac'],
'power_type': machine_data.get('power_type', 'ipmi'),
'power_parameters_power_address': power_data['power_address'],
}
if 'power_driver' in power_data:
data['power_parameters_power_driver'] = power_data['power_driver']
if 'power_user' in power_data:
data['power_parameters_power_user'] = power_data['power_user']
if 'power_password' in power_data:
data['power_parameters_power_pass'] = \
power_data['power_password']
if 'power_id' in power_data:
data['power_parameters_power_id'] = power_data['power_id']
return data
def update(self, new, old):
old_macs = set(v['mac_address'].lower() for v in old['interface_set'])
if new['mac_addresses'].lower() not in old_macs:
self._update = False
LOG.info('Mac changed deleting old machine %s', old['system_id'])
self._maas.delete(u'api/2.0/machines/{0}/'
.format(old['system_id']))
else:
new[self._update_key] = str(old[self._update_key])
return new
class AssignMachinesIP(MaasObject):
READY = 4
DEPLOYED = 6
def __init__(self):
super(AssignMachinesIP, self).__init__()
self._all_elements_url = None
self._create_url = \
(u'/api/2.0/nodes/{system_id}/interfaces/{interface_id}/',
'link_subnet')
self._config_path = 'region.machines'
self._element_key = 'hostname'
self._update_key = 'system_id'
self._extra_data_urls = {'machines': (u'api/2.0/machines/',
None, 'hostname')}
def fill_data(self, name, data, machines):
interface = data['interface']
machine = machines[name]
if machine['status'] == self.DEPLOYED:
return
if machine['status'] != self.READY:
raise Exception('Machine:{} not in READY state'.format(name))
if 'ip' not in interface:
return
data = {
'mode': 'STATIC',
'subnet': str(interface.get('subnet')),
'ip_address': str(interface.get('ip')),
}
if 'default_gateway' in interface:
data['default_gateway'] = interface.get('gateway')
data['force'] = '1'
data['system_id'] = str(machine['system_id'])
data['interface_id'] = str(machine['interface_set'][0]['id'])
return data
class DeployMachines(MaasObject):
# FIXME
READY = 4
DEPLOYED = 6
def __init__(self):
super(DeployMachines, self).__init__()
self._all_elements_url = None
self._create_url = (u'api/2.0/machines/{system_id}/', 'deploy')
self._config_path = 'region.machines'
self._element_key = 'hostname'
self._extra_data_urls = {'machines': (u'api/2.0/machines/',
None, 'hostname')}
def fill_data(self, name, machine_data, machines):
machine = machines[name]
if machine['status'] == self.DEPLOYED:
return
if machine['status'] != self.READY:
raise Exception('Not in ready state')
data = {
'system_id': machine['system_id'],
}
if 'distro_series' in machine_data:
data['distro_series'] = machine_data['distro_series']
if 'hwe_kernel' in machine_data:
data['hwe_kernel'] = machine_data['hwe_kernel']
return data
def send(self, data):
LOG.info('%s %s', self.__class__.__name__.lower(), _format_data(data))
self._maas.post(u'api/2.0/machines/', 'allocate', system_id=data['system_id']).read()
return self._maas.post(self._create_url[0].format(**data),
*self._create_url[1:], **data).read()
class BootResource(MaasObject):
def __init__(self):
super(BootResource, self).__init__()
self._all_elements_url = u'api/2.0/boot-resources/'
self._create_url = u'api/2.0/boot-resources/'
self._update_url = u'api/2.0/boot-resources/{0}/'
self._config_path = 'region.boot_resources'
def fill_data(self, name, boot_data):
sha256 = hashlib.sha256()
sha256.update(file(boot_data['content']).read())
data = {
'name': name,
'title': boot_data['title'],
'architecture': boot_data['architecture'],
'filetype': boot_data['filetype'],
'size': str(os.path.getsize(boot_data['content'])),
'sha256': sha256.hexdigest(),
'content': io.open(boot_data['content']),
}
return data
def update(self, new, old):
self._update = False
return new
class CommissioningScripts(MaasObject):
def __init__(self):
super(CommissioningScripts, self).__init__()
self._all_elements_url = u'api/2.0/commissioning-scripts/'
self._create_url = u'api/2.0/commissioning-scripts/'
self._config_path = 'region.commissioning_scripts'
self._update_url = u'api/2.0/commissioning-scripts/{0}'
self._update_key = 'name'
def fill_data(self, name, file_path):
data = {
'name': name,
'content': io.open(file_path),
}
return data
def update(self, new, old):
return new
class MaasConfig(MaasObject):
def __init__(self):
super(MaasConfig, self).__init__()
self._all_elements_url = None
self._create_url = (u'api/2.0/maas/', u'set_config')
self._config_path = 'region.maas_config'
def fill_data(self, name, value):
data = {
'name': name,
'value': str(value),
}
return data
def update(self, new, old):
self._update = False
return new
class SSHPrefs(MaasObject):
def __init__(self):
super(SSHPrefs, self).__init__()
self._all_elements_url = None
self._create_url = u'api/2.0/account/prefs/sshkeys/'
self._config_path = 'region.sshprefs'
self._element_key = 'hostname'
self._update_key = 'system_id'
def fill_data(self, value):
data = {
'key': value,
}
return data
def process(self):
config = __salt__['config.get']('maas')
for part in self._config_path.split('.'):
config = config.get(part, {})
extra = {}
for name, url_call in self._extra_data_urls.iteritems():
key = 'id'
if isinstance(url_call, tuple):
url_call, key = url_call[:]
json_res = json.loads(self._maas.get(url_call).read())
extra[name] = {v['name']: v[key] for v in json_res}
if self._all_elements_url:
all_elements = {}
elements = self._maas.get(self._all_elements_url).read()
res_json = json.loads(elements)
for element in res_json:
if isinstance(element, (str, unicode)):
all_elements[element] = {}
else:
all_elements[element[self._element_key]] = element
else:
all_elements = {}
ret = {
'success': [],
'errors': {},
'updated': [],
}
for config_data in config:
name = config_data[:10]
try:
data = self.fill_data(config_data, **extra)
self.send(data)
ret['success'].append(name)
except urllib2.HTTPError as e:
etxt = e.read()
LOG.exception('Failed for object %s reason %s', name, etxt)
ret['errors'][name] = str(etxt)
except Exception as e:
LOG.exception('Failed for object %s reason %s', name, e)
ret['errors'][name] = str(e)
if ret['errors']:
raise Exception(ret)
return ret
class Domain(MaasObject):
def __init__(self):
super(Domain, self).__init__()
self._all_elements_url = u'/api/2.0/domains/'
self._create_url = u'/api/2.0/domains/'
self._config_path = 'region.domain'
self._update_url = u'/api/2.0/domains/{0}/'
def fill_data(self, value):
data = {
'name': value,
}
self._update = True
return data
def update(self, new, old):
new['id'] = str(old['id'])
new['authoritative'] = str(old['authoritative'])
return new
def process(self):
ret = {
'success': [],
'errors': {},
'updated': [],
}
config = __salt__['config.get']('maas')
for part in self._config_path.split('.'):
config = config.get(part, {})
extra = {}
for name, url_call in self._extra_data_urls.iteritems():
key = 'id'
if isinstance(url_call, tuple):
url_call, key = url_call[:]
json_res = json.loads(self._maas.get(url_call).read())
extra[name] = {v['name']: v[key] for v in json_res}
if self._all_elements_url:
all_elements = {}
elements = self._maas.get(self._all_elements_url).read()
res_json = json.loads(elements)
for element in res_json:
if isinstance(element, (str, unicode)):
all_elements[element] = {}
else:
all_elements[element[self._element_key]] = element
else:
all_elements = {}
try:
data = self.fill_data(config, **extra)
data = self.update(data, all_elements.values()[0])
self.send(data)
ret['success'].append('domain')
except urllib2.HTTPError as e:
etxt = e.read()
LOG.exception('Failed for object %s reason %s', 'domain', etxt)
ret['errors']['domain'] = str(etxt)
except Exception as e:
LOG.exception('Failed for object %s reason %s', 'domain', e)
ret['errors']['domain'] = str(e)
if ret['errors']:
raise Exception(ret)
return ret
class MachinesStatus(MaasObject):
@classmethod
def execute(cls, objects_name=None):
cls._maas = _create_maas_client()
result = cls._maas.get(u'api/2.0/machines/')
json_result = json.loads(result.read())
res = []
summary = collections.Counter()
if objects_name:
if ',' in objects_name:
objects_name = set(objects_name.split(','))
else:
objects_name = set([objects_name])
for machine in json_result:
if objects_name and machine['hostname'] not in objects_name:
continue
status = STATUS_NAME_DICT[machine['status']]
summary[status] += 1
res.append(
{'hostname': machine['hostname'],
'system_id': machine['system_id'],
'status': status})
return {'machines': res, 'summary': summary}
@classmethod
def wait_for_machine_status(cls, **kwargs):
"""
A function that wait for any requested status, for any set of maas
machines.
If no kwargs has been passed - will try to wait ALL
defined in salt::maas::region::machines
See readme file for more examples.
CLI Example:
.. code-block:: bash
salt-call state.apply maas.machines.wait_for_deployed
:param kwargs:
timeout: in s; Global timeout for wait
poll_time: in s;Sleep time, between retry
req_status: string; Polling status
machines: list; machine names
ignore_machines: list; machine names
:ret: True
Exception - if something fail/timeout reached
"""
timeout = kwargs.get("timeout", 60 * 120)
poll_time = kwargs.get("poll_time", 30)
req_status = kwargs.get("req_status", "Ready")
to_discover = kwargs.get("machines", None)
ignore_machines = kwargs.get("ignore_machines", None)
if not to_discover:
try:
to_discover = __salt__['config.get']('maas')['region'][
'machines'].keys()
except KeyError:
LOG.warning("No defined machines!")
return True
total = copy.deepcopy(to_discover) or []
if ignore_machines and total:
total = [x for x in to_discover if x not in ignore_machines]
started_at = time.time()
while len(total) <= len(to_discover):
for m in to_discover:
for discovered in MachinesStatus.execute()['machines']:
if m == discovered['hostname'] and \
discovered['status'].lower() == req_status.lower():
if m in total:
total.remove(m)
if len(total) <= 0:
LOG.debug(
"Machines:{} are:{}".format(to_discover, req_status))
return True
if (timeout - (time.time() - started_at)) <= 0:
raise Exception(
'Machines:{}not in {} state'.format(total, req_status))
LOG.info(
"Waiting status:{} "
"for machines:{}"
"\nsleep for:{}s "
"Timeout:{}s".format(req_status, total, poll_time, timeout))
time.sleep(poll_time)
def process_fabrics():
return Fabric().process()
def process_subnets():
return Subnet().process()
def process_dhcp_snippets():
return DHCPSnippet().process()
def process_package_repositories():
return PacketRepository().process()
def process_devices(*args):
return Device().process(*args)
def process_machines(*args):
return Machine().process(*args)
def process_assign_machines_ip(*args):
return AssignMachinesIP().process(*args)
def machines_status(*args):
return MachinesStatus.execute(*args)
def deploy_machines(*args):
return DeployMachines().process(*args)
def process_boot_resources():
return BootResource().process()
def process_maas_config():
return MaasConfig().process()
def process_commissioning_scripts():
return CommissioningScripts().process()
def process_domain():
return Domain().process()
def process_sshprefs():
return SSHPrefs().process()
def wait_for_machine_status(**kwargs):
return MachinesStatus.wait_for_machine_status(**kwargs)