Merge pull request #15 from damjanek/master

Adding some capabilities
diff --git a/README.rst b/README.rst
index 407a2d9..021b8c4 100644
--- a/README.rst
+++ b/README.rst
@@ -70,7 +70,6 @@
                - trusty
           components:
               - main
-              - extra
           arches: amd64
           key: "-----BEGIN PGP PUBLIC KEY BLOCK-----
                Version: GnuPG v2
@@ -107,12 +106,14 @@
       machines:
         machine1:
           interfaces:
-            - one1: "11:22:33:44:55:66"
+            mac: "11:22:33:44:55:66"
           power_parameters:
             power_type: ipmi
             power_address: '192.168.10.10'
             power_user: bmc_user
             power_password: bmc_password
+          distro_series: xenial
+          hwe_kernel: hwe-16.04
       devices:
         machine1-ipmi:
           interface:
diff --git a/_modules/maas.py b/_modules/maas.py
index 4b127d8..d6ca2a9 100644
--- a/_modules/maas.py
+++ b/_modules/maas.py
@@ -15,6 +15,7 @@
 
 import io
 import logging
+import collections
 import os.path
 import subprocess
 import urllib2
@@ -32,6 +33,7 @@
 except ImportError:
     LOG.exception('why??')
 
+
 def __virtual__():
     '''
     Only load this module if maas-client
@@ -41,21 +43,23 @@
         return 'maas'
     return False
 
+
 APIKEY_FILE = '/var/lib/maas/.maas_credentials'
 
+
 def _format_data(data):
     class Lazy:
         def __str__(self):
             return ' '.join(['{0}={1}'.format(k, v)
-                             for k, v in data.iteritems()])
-
+                            for k, v in data.iteritems()])
     return Lazy()
 
 
 def _create_maas_client():
     global APIKEY_FILE
     try:
-        api_token = file(APIKEY_FILE).read().splitlines()[-1].strip().split(':')
+        api_token = file(APIKEY_FILE).read().splitlines()[-1].strip()\
+            .split(':')
     except:
         LOG.exception('token')
     auth = MAASOAuth(*api_token)
@@ -63,6 +67,7 @@
     dispatcher = MAASDispatcher()
     return MAASClient(auth, dispatcher, api_url)
 
+
 class MaasObject(object):
     def __init__(self):
         self._maas = _create_maas_client()
@@ -75,57 +80,85 @@
     def send(self, data):
         LOG.info('%s %s', self.__class__.__name__.lower(), _format_data(data))
         if self._update:
-            return self._maas.put(self._update_url.format(data[self._update_key]), **data).read()
+            return self._maas.put(
+                self._update_url.format(data[self._update_key]), **data).read()
         if isinstance(self._create_url, tuple):
-            return self._maas.post(*self._create_url, **data).read()
-        return self._maas.post(self._create_url, None, **data).read()
+            return self._maas.post(self._create_url[0].format(**data),
+                                   *self._create_url[1:], **data).read()
+        return self._maas.post(self._create_url.format(**data),
+                                None, **data).read()
 
-    def process(self):
-        config = __salt__['config.get']('maas')
-        for part in self._config_path.split('.'):
-            config = config.get(part, {})
-        extra = {}
-        for name, url_call in self._extra_data_urls.iteritems():
-            key = 'id'
-            if isinstance(url_call, tuple):
-                url_call, key = url_call[:]
-            extra[name] = {v['name']: v[key] for v in
-                            json.loads(self._maas.get(url_call).read())}
-        if self._all_elements_url:
-            all_elements = {}
-            elements = self._maas.get(self._all_elements_url).read()
-            res_json = json.loads(elements)
-            for element in res_json:
-                if isinstance(element, (str, unicode)):
-                    all_elements[element] = {}
-                else:
-                    all_elements[element[self._element_key]] = element
-        else:
-            all_elements = {}
+    def process(self, objects_name=None):
         ret = {
             'success': [],
             'errors': {},
             'updated': [],
         }
-        for name, config_data in config.iteritems():
-            self._update = False
-            try:
-                data = self.fill_data(name, config_data, **extra)
-                if name in all_elements:
-                    self._update = True
-                    data = self.update(data, all_elements[name])
-                    self.send(data)
-                    ret['updated'].append(name)
+        try:
+            config = __salt__['config.get']('maas')
+            for part in self._config_path.split('.'):
+                config = config.get(part, {})
+            extra = {}
+            for name, url_call in self._extra_data_urls.iteritems():
+                key = 'id'
+                key_name = 'name'
+                if isinstance(url_call, tuple):
+                    if len(url_call) == 2:
+                        url_call, key = url_call[:]
+                    else:
+                        url_call, key, key_name = url_call[:]
+                json_res = json.loads(self._maas.get(url_call).read())
+                if key:
+                    extra[name] = {v[key_name]: v[key] for v in json_res}
                 else:
-                    self.send(data)
-                    ret['success'].append(name)
-            except urllib2.HTTPError as e:
-                etxt = e.read()
-                LOG.exception('Failed for object %s reason %s', name, etxt)
-                ret['errors'][name] = str(etxt)
-            except Exception as e:
-                LOG.exception('Failed for object %s reason %s', name, e)
-                ret['errors'][name] = str(e)
+                    extra[name] = {v[key_name]: v for v in json_res}
+            if self._all_elements_url:
+                all_elements = {}
+                elements = self._maas.get(self._all_elements_url).read()
+                res_json = json.loads(elements)
+                for element in res_json:
+                    if isinstance(element, (str, unicode)):
+                        all_elements[element] = {}
+                    else:
+                        all_elements[element[self._element_key]] = element
+            else:
+                all_elements = {}
+
+            def process_single(name, config_data):
+                self._update = False
+                try:
+                    data = self.fill_data(name, config_data, **extra)
+                    if data is None:
+                        ret['updated'].append(name)
+                        return
+                    if name in all_elements:
+                        self._update = True
+                        data = self.update(data, all_elements[name])
+                        self.send(data)
+                        ret['updated'].append(name)
+                    else:
+                        self.send(data)
+                        ret['success'].append(name)
+                except urllib2.HTTPError as e:
+                    etxt = e.read()
+                    LOG.error('Failed for object %s reason %s', name, etxt)
+                    ret['errors'][name] = str(etxt)
+                except Exception as e:
+                    LOG.error('Failed for object %s reason %s', name, e)
+                    ret['errors'][name] = str(e)
+            if objects_name is not None:
+                if ',' in objects_name:
+                    objects_name = objects_name.split(',')
+                else:
+                    objects_name = [objects_name]
+                for object_name in objects_name:
+                    process_single(object_name, config[object_name])
+            else:
+                for name, config_data in config.iteritems():
+                    process_single(name, config_data)
+        except Exception as e:
+            LOG.exception('Error Global')
+            raise
         if ret['errors']:
             raise Exception(ret)
         return ret
@@ -152,6 +185,7 @@
         new['id'] = str(old['id'])
         return new
 
+
 class Subnet(MaasObject):
     def __init__(self):
         super(Subnet, self).__init__()
@@ -159,7 +193,7 @@
         self._create_url = u'api/2.0/subnets/'
         self._update_url = u'api/2.0/subnets/{0}/'
         self._config_path = 'region.subnets'
-        self._extra_data_urls = {'fabrics':u'api/2.0/fabrics/'}
+        self._extra_data_urls = {'fabrics': u'api/2.0/fabrics/'}
 
     def fill_data(self, name, subnet, fabrics):
         data = {
@@ -201,10 +235,12 @@
         LOG.info('iprange %s', _format_data(data))
         if update:
             LOG.warn('UPDATING %s %s', data, old_data)
-            self._maas.put(u'api/2.0/ipranges/{0}/'.format(old_data['id']), **data)
+            self._maas.put(u'api/2.0/ipranges/{0}/'.format(old_data['id']),
+                           **data)
         else:
             self._maas.post(u'api/2.0/ipranges/', None, **data)
 
+
 class DHCPSnippet(MaasObject):
     def __init__(self):
         super(DHCPSnippet, self).__init__()
@@ -216,11 +252,11 @@
 
     def fill_data(self, name, snippet, subnets):
         data = {
-             'name': name,
-             'value': snippet['value'],
-             'description': snippet['description'],
-             'enabled': str(snippet['enabled'] and 1 or 0),
-             'subnet': str(subnets[snippet['subnet']]),
+            'name': name,
+            'value': snippet['value'],
+            'description': snippet['description'],
+            'enabled': str(snippet['enabled'] and 1 or 0),
+            'subnet': str(subnets[snippet['subnet']]),
         }
         return data
 
@@ -228,6 +264,7 @@
         new['id'] = str(old['id'])
         return new
 
+
 class PacketRepository(MaasObject):
     def __init__(self):
         super(PacketRepository, self).__init__()
@@ -254,6 +291,7 @@
         new['id'] = str(old['id'])
         return new
 
+
 class Device(MaasObject):
     def __init__(self):
         super(Device, self).__init__()
@@ -301,7 +339,7 @@
         if self._update:
             data['force'] = '1'
         LOG.info('interfaces link_subnet %s %s %s', system_id, interface_id,
-                _format_data(data))
+                 _format_data(data))
         self._maas.post(u'/api/2.0/nodes/{0}/interfaces/{1}/'
                         .format(system_id, interface_id), 'link_subnet',
                         **data)
@@ -318,12 +356,11 @@
         self._update_key = 'system_id'
 
     def fill_data(self, name, machine_data):
-        self._interface = machine_data['interface']
         power_data = machine_data['power_parameters']
         data = {
             'hostname': name,
             'architecture': machine_data.get('architecture', 'amd64/generic'),
-            'mac_addresses': self._interface['mac'],
+            'mac_addresses': machine_data['interface']['mac'],
             'power_type': machine_data.get('power_type', 'ipmi'),
             'power_parameters_power_address': power_data['power_address'],
         }
@@ -339,37 +376,78 @@
         if new['mac_addresses'].lower() not in old_macs:
             self._update = False
             LOG.info('Mac changed deleting old machine %s', old['system_id'])
-            self._maas.delete(u'api/2.0/machines/{0}/'.format(old['system_id']))
+            self._maas.delete(u'api/2.0/machines/{0}/'
+                              .format(old['system_id']))
         else:
             new[self._update_key] = str(old[self._update_key])
         return new
 
-    def _link_interface(self, system_id, interface_id):
-        if 'ip' not in self._interface:
+
+class AssignMachinesIP(MaasObject):
+    READY = 4
+
+    def __init__(self):
+        super(AssignMachinesIP, self).__init__()
+        self._all_elements_url = None
+        self._create_url = \
+            (u'/api/2.0/nodes/{system_id}/interfaces/{interface_id}/',
+             'link_subnet')
+        self._config_path = 'region.machines'
+        self._element_key = 'hostname'
+        self._update_key = 'system_id'
+        self._extra_data_urls = {'machines': (u'api/2.0/machines/',
+                                              None, 'hostname')}
+
+    def fill_data(self, name, data, machines):
+        interface = data['interface']
+        machine = machines[name]
+        if machine['status'] != self.READY:
+            raise Exception('Not in ready state')
+        if 'ip' not in interface:
             return
         data = {
             'mode': 'STATIC',
-            'subnet': self._interface.get('subnet'),
-            'ip_address': self._interface.get('ip'),
+            'subnet': str(interface.get('subnet')),
+            'ip_address': str(interface.get('ip')),
         }
-        if 'default_gateway' in self._interface:
-            data['default_gateway'] = self._interface.get('gateway')
-        if self._update:
-            data['force'] = '1'
-        LOG.info('interfaces link_subnet %s %s %s', system_id, interface_id,
-                _format_data(data))
-        self._maas.post(u'/api/2.0/nodes/{0}/interfaces/{1}/'
-                        .format(system_id, interface_id), 'link_subnet',
-                        **data)
+        if 'default_gateway' in interface:
+            data['default_gateway'] = interface.get('gateway')
+        data['force'] = '1'
+        data['system_id'] = str(machine['system_id'])
+        data['interface_id'] = str(machine['interface_set'][0]['id'])
+        return data
+
+
+class DeployMachines(MaasObject):
+    READY = 4
+
+    def __init__(self):
+        super(DeployMachines, self).__init__()
+        self._all_elements_url = None
+        self._create_url = (u'api/2.0/machines/{system_id}/', 'deploy')
+        self._config_path = 'region.machines'
+        self._element_key = 'hostname'
+        self._extra_data_urls = {'machines': (u'api/2.0/machines/',
+                                              None, 'hostname')}
+
+    def fill_data(self, name, machine_data, machines):
+        machine = machines[name]
+        if machine['status'] != self.READY:
+            raise Exception('Not in ready state')
+        data = {
+            'system_id': machine['system_id'],
+        }
+        if 'distro_series' in machine_data:
+            data['distro_series'] = machine_data['distro_series']
+        if 'hwe_kernel' in machine_data:
+            data['hwe_kernel'] = machine_data['hwe_kernel']
+        return data
 
     def send(self, data):
-        response = super(Machine, self).send(data)
-        resp_json = json.loads(response)
-        system_id = resp_json['system_id']
-        iface_id = resp_json['interface_set'][0]['id']
-        self._link_interface(system_id, iface_id)
-        return response
-
+        LOG.info('%s %s', self.__class__.__name__.lower(), _format_data(data))
+        self._maas.post(u'api/2.0/machines/', 'allocate', system_id=data['system_id']).read()
+        return self._maas.post(self._create_url[0].format(**data),
+                                *self._create_url[1:], **data).read()
 
 class BootResource(MaasObject):
     def __init__(self):
@@ -397,6 +475,7 @@
         self._update = False
         return new
 
+
 class CommissioningScripts(MaasObject):
     def __init__(self):
         super(CommissioningScripts, self).__init__()
@@ -416,6 +495,7 @@
     def update(self, new, old):
         return new
 
+
 class MaasConfig(MaasObject):
     def __init__(self):
         super(MaasConfig, self).__init__()
@@ -459,8 +539,8 @@
             key = 'id'
             if isinstance(url_call, tuple):
                 url_call, key = url_call[:]
-            extra[name] = {v['name']: v[key] for v in
-                            json.loads(self._maas.get(url_call).read())}
+            json_res = json.loads(self._maas.get(url_call).read())
+            extra[name] = {v['name']: v[key] for v in json_res}
         if self._all_elements_url:
             all_elements = {}
             elements = self._maas.get(self._all_elements_url).read()
@@ -494,6 +574,7 @@
             raise Exception(ret)
         return ret
 
+
 class Domain(MaasObject):
     def __init__(self):
         super(Domain, self).__init__()
@@ -515,6 +596,11 @@
         return new
 
     def process(self):
+        ret = {
+            'success': [],
+            'errors': {},
+            'updated': [],
+        }
         config = __salt__['config.get']('maas')
         for part in self._config_path.split('.'):
             config = config.get(part, {})
@@ -523,8 +609,8 @@
             key = 'id'
             if isinstance(url_call, tuple):
                 url_call, key = url_call[:]
-            extra[name] = {v['name']: v[key] for v in
-                            json.loads(self._maas.get(url_call).read())}
+            json_res = json.loads(self._maas.get(url_call).read())
+            extra[name] = {v['name']: v[key] for v in json_res}
         if self._all_elements_url:
             all_elements = {}
             elements = self._maas.get(self._all_elements_url).read()
@@ -536,11 +622,6 @@
                     all_elements[element[self._element_key]] = element
         else:
             all_elements = {}
-        ret = {
-            'success': [],
-            'errors': {},
-            'updated': [],
-        }
         try:
             data = self.fill_data(config, **extra)
             data = self.update(data, all_elements.values()[0])
@@ -558,35 +639,88 @@
         return ret
 
 
+class MachinesStatus(MaasObject):
+    @classmethod
+    def execute(cls, objects_name=None):
+        cls._maas = _create_maas_client()
+        result = cls._maas.get(u'api/2.0/machines/')
+        json_result = json.loads(result.read())
+        res = []
+        status_name_dict = dict([
+            (0, 'New'), (1, 'Commissioning'), (2, 'Failed commissioning'),
+            (3, 'Missing'), (4, 'Ready'), (5, 'Reserved'), (10, 'Allocated'),
+            (9, 'Deploying'), (6, 'Deployed'), (7, 'Retired'), (8, 'Broken'),
+            (11, 'Failed deployment'), (12, 'Releasing'),
+            (13, 'Releasing failed'), (14, 'Disk erasing'),
+            (15, 'Failed disk erasing')])
+        summary = collections.Counter()
+        if objects_name:
+            if ',' in objects_name:
+                objects_name = set(objects_name.split(','))
+            else:
+                objects_name = set([objects_name])
+        for machine in json_result:
+            if objects_name and machine['hostname'] not in objects_name:
+                continue
+            status = status_name_dict[machine['status']]
+            summary[status] += 1
+            res.append('hostname:{},system_id:{},status:{}'
+                       .format(machine['hostname'], machine['system_id'],
+                               status))
+        return {'machines': res, 'summary': summary}
+
+
 def process_fabrics():
     return Fabric().process()
 
+
 def process_subnets():
     return Subnet().process()
 
+
 def process_dhcp_snippets():
     return DHCPSnippet().process()
 
+
 def process_package_repositories():
     return PacketRepository().process()
 
-def process_devices():
-    return Device().process()
 
-def process_machines():
-    return Machine().process()
+def process_devices(*args):
+    return Device().process(*args)
+
+
+def process_machines(*args):
+    return Machine().process(*args)
+
+
+def process_assign_machines_ip(*args):
+    return AssignMachinesIP().process(*args)
+
+
+def machines_status(*args):
+    return MachinesStatus.execute(*args)
+
+
+def deploy_machines(*args):
+    return DeployMachines().process(*args)
+
 
 def process_boot_resources():
     return BootResource().process()
 
+
 def process_maas_config():
     return MaasConfig().process()
 
+
 def process_commissioning_scripts():
     return CommissioningScripts().process()
 
+
 def process_domain():
     return Domain().process()
 
+
 def process_sshprefs():
     return SSHPrefs().process()
diff --git a/maas/machines.sls b/maas/machines/assign_ip.sls
similarity index 78%
copy from maas/machines.sls
copy to maas/machines/assign_ip.sls
index 4c5504e..02fc7ba 100644
--- a/maas/machines.sls
+++ b/maas/machines/assign_ip.sls
@@ -4,8 +4,8 @@
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
 
-maas_machines:
+assign_ips_to_machines:
   module.run:
-  - name: maas.process_machines
+  - name: maas.process_assign_machines_ip
   - require:
     - cmd: maas_login_admin
diff --git a/maas/machines.sls b/maas/machines/deploy.sls
similarity index 83%
copy from maas/machines.sls
copy to maas/machines/deploy.sls
index 4c5504e..290036c 100644
--- a/maas/machines.sls
+++ b/maas/machines/deploy.sls
@@ -4,8 +4,8 @@
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
 
-maas_machines:
+deploy_machines:
   module.run:
-  - name: maas.process_machines
+  - name: maas.deploy_machines
   - require:
     - cmd: maas_login_admin
diff --git a/maas/machines.sls b/maas/machines/init.sls
similarity index 93%
rename from maas/machines.sls
rename to maas/machines/init.sls
index 4c5504e..c17f63f 100644
--- a/maas/machines.sls
+++ b/maas/machines/init.sls
@@ -4,7 +4,7 @@
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
 
-maas_machines:
+create__machines:
   module.run:
   - name: maas.process_machines
   - require:
diff --git a/maas/machines.sls b/maas/machines/status.sls
similarity index 81%
copy from maas/machines.sls
copy to maas/machines/status.sls
index 4c5504e..acfbbfc 100644
--- a/maas/machines.sls
+++ b/maas/machines/status.sls
@@ -4,8 +4,8 @@
   cmd.run:
   - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
 
-maas_machines:
+check_machines_status:
   module.run:
-  - name: maas.process_machines
+  - name: maas.machines_status
   - require:
     - cmd: maas_login_admin
diff --git a/maas/map.jinja b/maas/map.jinja
index 45c4eab..8c0a7d9 100644
--- a/maas/map.jinja
+++ b/maas/map.jinja
@@ -13,6 +13,7 @@
 Debian:
   pkgs:
   - maas-region-controller
+  - python-oauth
   services:
   - maas-regiond
   - bind9