Extend ironic formula

This patch updates ironicng salt module to be able of:

 * list vifs for a given node
 * attach vif for a given node
 * detach vif from a given node
 * deploy user image to node
 * generate configdrive

Fix some default values when enrolling nodes automatically.

Allow to download images from http to conductor http_root.

Change-Id: Id99ad955c8c7256ae10ece7a173242044692e713
diff --git a/_modules/configdrive.py b/_modules/configdrive.py
new file mode 100644
index 0000000..0a45b02
--- /dev/null
+++ b/_modules/configdrive.py
@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+
+
+import gzip
+import json
+import logging
+import os
+import StringIO
+import shutil
+import six
+import tempfile
+import yaml
+
+
+HAS_LIBS = False
+try:
+  from oslo_utils import uuidutils
+  from oslo_utils import fileutils
+  from oslo_concurrency import processutils
+  from oslo_serialization import base64
+  HAS_LIBS = True
+except ImportError:
+    pass
+
+LOG = logging.getLogger(__name__)
+
+
+def __virtual__():
+    '''
+    Only load this module if mkisofs is installed on this minion.
+    '''
+    if not HAS_LIBS:
+        return False
+
+    for path in os.environ["PATH"].split(os.pathsep):
+        if os.access(os.path.join(path, 'mkisofs'), os.X_OK):
+            return True
+
+    return False
+
+class ConfigDriveBuilder(object):
+    """Build config drives, optionally as a context manager."""
+
+    def __init__(self, image_file):
+        self.image_file = image_file
+        self.mdfiles=[] # List with (path, data)
+
+    def __enter__(self):
+        fileutils.delete_if_exists(self.image_file)
+        return self
+
+    def __exit__(self, exctype, excval, exctb):
+        self.make_drive()
+
+    def add_file(self, path, data):
+        self.mdfiles.append((path, data))
+
+    def _add_file(self, basedir, path, data):
+        filepath = os.path.join(basedir, path)
+        dirname = os.path.dirname(filepath)
+        fileutils.ensure_tree(dirname)
+        with open(filepath, 'wb') as f:
+            # the given data can be either text or bytes. we can only write
+            # bytes into files.
+            if isinstance(data, six.text_type):
+                data = data.encode('utf-8')
+            f.write(data)
+
+    def _write_md_files(self, basedir):
+        for data in self.mdfiles:
+            self._add_file(basedir, data[0], data[1])
+
+    def _make_iso9660(self, path, tmpdir):
+
+        processutils.execute('mkisofs',
+                      '-o', path,
+                      '-ldots',
+                      '-allow-lowercase',
+                      '-allow-multidot',
+                      '-l',
+                      '-V', 'config-2',
+                      '-r',
+                      '-J',
+                      '-quiet',
+                      tmpdir,
+                      attempts=1,
+                      run_as_root=False)
+
+    def make_drive(self):
+        """Make the config drive.
+        :raises ProcessExecuteError if a helper process has failed.
+        """
+        try:
+          tmpdir = tempfile.mkdtemp()
+          self._write_md_files(tmpdir)
+          self._make_iso9660(self.image_file, tmpdir)
+        finally:
+          shutil.rmtree(tmpdir)
+
+
+def generate(dst, hostname, domainname, instance_id=None, public_keys=None,
+             user_data=None, network_data=None, ironic_format=False):
+    ''' Generate config drive
+
+    :param dst: destination file to place config drive.
+    :param hostname: hostname of Instance.
+    :param domainname: instance domain.
+    :param instance_id: UUID of the instance.
+    :param public_keys: dict of public keys.
+    :param user_data: custom user data dictionary.
+    :param network_data: custom network info dictionary.
+    :param ironic_format: create base64 of gzipped ISO format
+
+    CLI Example:
+    .. code-block:: bash
+        salt '*' configdrive.generate dst=/tmp/my_cfgdrive.iso hostname=host1
+    '''
+    instance_md = {}
+    public_keys = public_keys or {}
+
+    instance_md['uuid'] = instance_id or uuidutils.generate_uuid()
+    instance_md['hostname'] = '%s.%s' % (hostname, domainname)
+    instance_md['name'] = hostname
+    instance_md['public_keys'] = public_keys
+
+    data = json.dumps(instance_md)
+
+    if user_data:
+        user_data = '#cloud-config\n\n' + yaml.dump(user_data, default_flow_style=False)
+
+    LOG.debug('Generating config drive for %s' % hostname)
+
+    with ConfigDriveBuilder(dst) as cfgdrive:
+        cfgdrive.add_file('openstack/latest/meta_data.json', data)
+        if user_data:
+            cfgdrive.add_file('openstack/latest/user_data', user_data)
+        if network_data:
+             cfgdrive.add_file('openstack/latest/network_data.json',
+                               json.dumps(network_data))
+        cfgdrive.add_file('openstack/latest/vendor_data.json', '{}')
+        cfgdrive.add_file('openstack/latest/vendor_data2.json', '{}')
+
+    b64_gzip = None
+    if ironic_format:
+        with open(dst) as f:
+            with tempfile.NamedTemporaryFile() as tmpzipfile:
+                g = gzip.GzipFile(fileobj=tmpzipfile, mode='wb')
+                shutil.copyfileobj(f, g)
+                g.close()
+                tmpzipfile.seek(0)
+                b64_gzip = base64.encode_as_bytes(tmpzipfile.read())
+        with open(dst, 'w') as f:
+            f.write(b64_gzip)
+
+    LOG.debug('Config drive was built %s' % dst)
+    res = {}
+    res['meta-data'] = data
+    if user_data:
+        res['user-data'] = user_data
+    if b64_gzip:
+        res['base64_gzip'] = b64_gzip
+    return res
diff --git a/_modules/ironicng.py b/_modules/ironicng.py
index 9ed55ef..d18a6d2 100644
--- a/_modules/ironicng.py
+++ b/_modules/ironicng.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import logging
+import json
 from functools import wraps
 LOG = logging.getLogger(__name__)
 
@@ -8,6 +9,7 @@
 HAS_IRONIC = False
 try:
     from ironicclient import client
+    from ironicclient.common import utils
     HAS_IRONIC = True
 except ImportError:
     pass
@@ -24,42 +26,59 @@
     return False
 
 
-def _autheticate(func_name):
-    '''
-    Authenticate requests with the salt keystone module and format return data
-    '''
-    @wraps(func_name)
-    def decorator_method(*args, **kwargs):
+def _get_keystone_endpoint_and_token(**connection_args):
+    if connection_args.get('connection_endpoint_type') == None:
+        endpoint_type = 'internalURL'
+    else:
+        endpoint_type = connection_args.get('connection_endpoint_type')
+
+    kstone = __salt__['keystone.auth'](**connection_args)
+    endpoint = kstone.service_catalog.url_for(
+        service_type='baremetal', endpoint_type=endpoint_type)
+    token = kstone.auth_token
+    return endpoint, token
+
+
+def _get_ironic_session(endpoint, token, api_version=None):
+    return client.get_client(1, ironic_url=endpoint,
+                             os_auth_token=token,
+                             os_ironic_api_version=api_version)
+
+
+def _get_function_attrs(**kwargs):
+    connection_args = {'profile': kwargs.pop('profile', None)}
+    nkwargs = {}
+    for kwarg in kwargs:
+        if 'connection_' in kwarg:
+            connection_args.update({kwarg: kwargs[kwarg]})
+        elif '__' not in kwarg:
+            nkwargs.update({kwarg: kwargs[kwarg]})
+    return connection_args, nkwargs
+
+
+def _autheticate(api_version=None):
+    def _auth(func_name):
         '''
-        Authenticate request and format return data
+        Authenticate requests with the salt keystone module and format return data
         '''
-        connection_args = {'profile': kwargs.pop('profile', None)}
-        nkwargs = {}
-        for kwarg in kwargs:
-            if 'connection_' in kwarg:
-                connection_args.update({kwarg: kwargs[kwarg]})
-            elif '__' not in kwarg:
-                nkwargs.update({kwarg: kwargs[kwarg]})
-        kstone = __salt__['keystone.auth'](**connection_args)
-        token = kstone.auth_token
+        @wraps(func_name)
+        def decorator_method(*args, **kwargs):
+            '''Authenticate request and format return data'''
+            connection_args, nkwargs = _get_function_attrs(**kwargs)
+            endpoint, token = _get_keystone_endpoint_and_token(**connection_args)
 
-        if kwargs.get('connection_endpoint_type') == None:
-            endpoint_type = 'internalURL'
-        else:
-            endpoint_type = kwargs.get('connection_endpoint_type')
+            ironic_interface = _get_ironic_session(
+                endpoint=endpoint,
+                token = token,
+                api_version=api_version)
 
-        endpoint = kstone.service_catalog.url_for(
-            service_type='baremetal',
-            endpoint_type=endpoint_type)
-        ironic_interface = client.get_client(
-            1,
-            ironic_url=endpoint, os_auth_token=token)
-        return func_name(ironic_interface, *args, **nkwargs)
-    return decorator_method
+            return func_name(ironic_interface, *args, **nkwargs)
+        return decorator_method
+    return _auth
 
 
-@_autheticate
-def list_nodes(ironic_interface, **kwargs):
+@_autheticate()
+def list_nodes(ironic_interface, *args, **kwargs):
     '''
     list all ironic nodes
     CLI Example:
@@ -67,21 +86,21 @@
         salt '*' ironic.list_nodes
     '''
     return {'nodes': [x.to_dict() for x
-                      in ironic_interface.node.list(**kwargs)]}
+                      in ironic_interface.node.list(*args, **kwargs)]}
 
 
-@_autheticate
-def create_node(ironic_interface, **kwargs):
+@_autheticate()
+def create_node(ironic_interface, *args, **kwargs):
     '''
     create ironic node
     CLI Example:
     .. code-block:: bash
         salt '*' ironic.create_node
     '''
-    return ironic_interface.node.create(**kwargs).to_dict()
+    return ironic_interface.node.create(*args, **kwargs).to_dict()
 
 
-@_autheticate
+@_autheticate()
 def delete_node(ironic_interface, node_id):
     '''
     delete ironic node
@@ -94,7 +113,7 @@
     ironic_interface.node.delete(node_id)
 
 
-@_autheticate
+@_autheticate()
 def show_node(ironic_interface, node_id):
     '''
     show info about ironic node
@@ -106,7 +125,7 @@
     return ironic_interface.node.get(node_id).to_dict()
 
 
-@_autheticate
+@_autheticate()
 def create_port(ironic_interface, address, node_name=None,
                 node_uuid=None, **kwargs):
     '''
@@ -121,8 +140,8 @@
         address=address, node_uuid=node_uuid, **kwargs).to_dict()
 
 
-@_autheticate
-def list_ports(ironic_interface, **kwargs):
+@_autheticate()
+def list_ports(ironic_interface, *args, **kwargs):
     '''
     list all ironic ports
     CLI Example:
@@ -131,6 +150,262 @@
     '''
 
     return {'ports': [x.to_dict() for x
-                      in ironic_interface.port.list(**kwargs)]}
+                      in ironic_interface.port.list(*args, **kwargs)]}
+
+@_autheticate()
+def node_set_provision_state(ironic_interface, *args, **kwargs):
+    '''Set the provision state for the node.
+
+    CLI Example:
+    .. code-block:: bash
+        salt '*' ironic.node_set_provision_state node_uuid=node-1 state=active profile=admin_identity
+    '''
+
+    ironic_interface.node.set_provision_state(*args, **kwargs)
+
+@_autheticate(api_version='1.28')
+def vif_attach(ironic_interface, *args, **kwargs):
+    '''Attach vif to a given node.
+
+    CLI Example:
+    .. code-block:: bash
+        salt '*' ironic.vif_attach node_ident=node-1 vif_id=vif1 profile=admin_identity
+    '''
+
+    ironic_interface.node.vif_attach(*args, **kwargs)
+
+@_autheticate(api_version='1.28')
+def vif_detach(ironic_interface, *args, **kwargs):
+    '''Detach vif from a given node.
+
+    CLI Example:
+    .. code-block:: bash
+        salt '*' ironic.vif_detach node_ident=node-1 vif_id=vif1 profile=admin_identity
+    '''
+
+    ironic_interface.node.vif_detach(*args, **kwargs)
+
+@_autheticate(api_version='1.28')
+def vif_list(ironic_interface, *args, **kwargs):
+    '''List vifs for a given node.
+
+    CLI Example:
+    .. code-block:: bash
+        salt '*' ironic.vif_list node_ident=node-1 profile=admin_identity
+    '''
+
+    return [vif.to_dict() for vif in ironic_interface.node.vif_list(*args, **kwargs)]
+
+def _merge_profiles(a, b, path=None):
+    """Merge b into a"""
+    if path is None: path = []
+    for key in b:
+        if key in a:
+            if isinstance(a[key], dict) and isinstance(b[key], dict):
+                _merge_profiles(a[key], b[key], path + [str(key)])
+            elif a[key] == b[key]:
+                pass # same leaf value
+            else:
+                raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
+        else:
+            a[key] = b[key]
+    return a
+
+def _get_node_deployment_profile(node_id, profile):
+    dp = {}
+    nodes = __salt__['pillar.get'](
+        'ironic:client:nodes:%s' % profile)
+
+    for node in nodes:
+        if node['name'] == node_id:
+            return node.get('deployment_profile')
 
 
+def deploy_node(node_id, image_source=None, root_gb=None,
+                image_checksum=None, configdrive=None, vif_id=None,
+                deployment_profile=None, partition_profile=None,
+                profile=None, **kwargs):
+    '''Deploy user image to ironic node
+
+    Deploy node with provided data. If deployment_profile is set,
+    try to get deploy data from pillar:
+
+      ironic:
+        client:
+          deployment_profiles:
+            profile1:
+              image_source:
+              image_checksum:
+              ...
+
+    :param node_id: UUID or Name of the node
+    :param image_source: URL/glance image uuid to deploy
+    :param root_gb: Size of root partition
+    :param image_checksum: md5 summ of image, only when image_source
+                           is URL
+    :param configdrive: URL to or base64 gzipped iso config drive
+    :param vif_id: UUID of VIF to attach to node.
+    :param deployment_profile: id of the profile to look nodes in.
+    :param partition_profile: id of the partition profile to apply.
+    :param profile: auth profile to use.
+
+    CLI Example:
+    .. code-block:: bash
+        salt '*' ironicng.deploy_node node_id=node01 image_source=aaa-bbb-ccc-ddd-eee-fff
+    '''
+    deploy_params = [image_source, image_checksum, configdrive, vif_id]
+    if deployment_profile and any(deploy_params):
+        err_msg = ("deployment_profile can' be specified with any "
+                   "of %s" % ', '.join(deploy_params))
+        LOG.error(err_msg)
+        return  _deploy_failed(name, err_msg)
+
+    if partition_profile:
+        partition_profile = __salt__['pillar.get'](
+          'ironic:client:partition_profiles:%s' % partition_profile)
+
+    if deployment_profile:
+        deployment_profile = __salt__['pillar.get'](
+            'ironic:client:deployment_profiles:%s' % deployment_profile)
+        node_deployment_profile = _get_node_deployment_profile(
+            node_id, profile=profile) or {}
+        if partition_profile:
+            image_properties = deployment_profile['instance_info'].get('image_properties', {})
+            image_properties.update(partition_profile)
+            deployment_profile['instance_info']['image_properties'] = image_properties
+        _merge_profiles(deployment_profile, node_deployment_profile)
+    else:
+        deployment_profile = {
+            'instance_info': {
+                'image_source': image_source,
+                'image_checksum': image_checksum,
+                'root_gb': root_gb,
+            },
+            'configdrive': configdrive,
+            'network': {
+                'vif_id': vif_id,
+            }
+        }
+
+    connection_args, nkwargs = _get_function_attrs(profile=profile, **kwargs)
+
+    endpoint, token = _get_keystone_endpoint_and_token(**connection_args)
+    ironic_interface = _get_ironic_session(
+            endpoint=endpoint,
+            token = token)
+
+    def _convert_to_uuid(resource, name, **connection_args):
+        resources =  __salt__['neutronng.list_%s' % resource](
+            name=name, **connection_args)
+
+        err_msg = None
+        if len(resources) == 0:
+            err_msg = "{0} with name {1} not found".format(
+                resource, network_name)
+        elif len(resources) > 1:
+            err_msg = "Multiple {0} with name {1} found.".format(
+                resource, network_name)
+        else:
+            return resources[resource][0]['id']
+
+        LOG.err(err_msg)
+        return _deploy_failed(name, err_msg)
+
+
+    def _prepare_node_for_deploy(ironic_interface,
+                                 node_id,
+                                 deployment_profile):
+
+        instance_info = deployment_profile.get('instance_info')
+        node_attr = []
+        for k,v in instance_info.iteritems():
+          node_attr.append('instance_info/%s=%s' % (k, json.dumps(v)))
+
+        net = deployment_profile.get('network')
+        vif_id = net.get('vif_id')
+        network_id = net.get('id')
+        network_name = net.get('name')
+        if (vif_id and any([network_name, network_id]) or
+                (network_name and network_id)):
+            err_msg = ("Only one of network:name or network:id or vif_id should be specified.")
+            LOG.error(err_msg)
+            return  _deploy_failed(name, err_msg)
+
+        if network_name:
+            network_id = _convert_to_uuid('networks', network_name, **connection_args)
+
+        if network_id:
+            create_port_args = {
+                'name': '%s_port' % node_id,
+                'network_id': network_id,
+            }
+            fixed_ips = []
+            for fixed_ip in net.get('fixed_ips', []):
+                subnet_name = fixed_ip.get('subnet_name')
+                subnet_id = fixed_ip.get('subnet_id')
+                if subnet_name and subnet_id:
+                    err_msg = ("Only one of subnet_name or subnet_id should be specified.")
+                    LOG.error(err_msg)
+                    return  _deploy_failed(name, err_msg)
+                if subnet_name:
+                    subnet_id = _convert_to_uuid('subnets', subnet_name, **connection_args)
+                if subnet_id:
+                    fixed_ips.append({'ip_address': fixed_ip['ip_address'],
+                                      'subnet_id': subnet_id})
+            if fixed_ips:
+                create_port_args['fixed_ips'] = fixed_ips
+            create_port_args.update(connection_args)
+
+            vif_id = __salt__['neutronng.create_port'](**create_port_args)
+
+        if vif_id:
+            __salt__['ironicng.vif_attach'](node_ident=node_id, vif_id=vif_id, **connection_args)
+
+        configdrive = deployment_profile.get('configdrive')
+        if not configdrive:
+            metadata = deployment_profile.get('metadata')
+            if metadata:
+                configdrive_args = {}
+                userdata = metadata.get('userdata')
+                instance = metadata.get('instance')
+                hostname = instance.pop('hostname', node_id)
+                if userdata:
+                    configdrive_args['user_data'] = userdata
+                if instance:
+                    configdrive_args.update(instance)
+                configdrive = __salt__['configdrive.generate'](
+                   dst='/tmp/%s' % node_id, hostname=hostname, ironic_format=True,
+                   **configdrive_args)['base64_gzip']
+
+        if configdrive:
+            node_attr.append('instance_info/configdrive=%s' % configdrive)
+
+        if node_attr:
+            patch = utils.args_array_to_patch('add', node_attr)
+            ironic_interface.node.update(node_id, patch).to_dict()
+
+
+    _prepare_node_for_deploy(ironic_interface, node_id, deployment_profile)
+
+    provision_args = {
+      'node_uuid': node_id,
+      'state': 'active'
+    }
+    provision_args.update(connection_args)
+
+    __salt__['ironicng.node_set_provision_state'](**provision_args)
+    return _deploy_started(node_id)
+
+
+def _deploy_failed(name, reason):
+    changes_dict = {'name': name,
+                    'comment': 'Deployment of node {0} failed to start due to {1}.'.format(name, reason),
+                    'result': False}
+    return changes_dict
+
+
+def _deploy_started(name):
+    changes_dict = {'name': name,
+                    'comment': 'Deployment of node {0} has been started.'.format(name),
+                    'result': True}
+    return changes_dict
diff --git a/ironic/client.sls b/ironic/client.sls
index 8a672d2..10faf63 100644
--- a/ironic/client.sls
+++ b/ironic/client.sls
@@ -13,10 +13,11 @@
   ironicng.node_present:
     - name: {{ node.name }}
     - driver: {{ node.driver }}
-    - properties: {{ node.properties }}
+    - properties: {{ node.properties|default({}) }}
     - profile: {{ identity_name }}
-    - driver_info: {{ node.driver_info }}
+    - driver_info: {{ node.driver_info|default({}) }}
 
+  {%- if node.ports is defined %}
   {%- for port in node.ports %}
 
 {{ node.name }}_port{{ loop.index }}_present:
@@ -26,6 +27,7 @@
     - profile: {{ identity_name }}
 
   {%- endfor %} # end for ports
+  {%- endif %} # end if node.ports defined
 
   {%- endfor %} # end for nodes
 {%- endfor %} # end client.nodes.iteritems
diff --git a/ironic/conductor.sls b/ironic/conductor.sls
index c4093dc..b8baa13 100644
--- a/ironic/conductor.sls
+++ b/ironic/conductor.sls
@@ -68,4 +68,18 @@
     - require:
       - file: ironic_dirs
 
+{%- if conductor.http_images is defined %}
+{%- for image in conductor.http_images %}
+
+image_{{ image.name }}:
+  file.managed:
+    - name: {{ conductor.http_root }}/{{ image.name }}
+    - source: {{ image.source }}
+    - source_hash: md5={{ image.md5summ }}
+    - user: 'ironic'
+    - group: 'ironic'
+
+{%- endfor %}
+{%- endif %}
+
 {%- endif %}
diff --git a/ironic/deploy.sls b/ironic/deploy.sls
new file mode 100644
index 0000000..71e7699
--- /dev/null
+++ b/ironic/deploy.sls
@@ -0,0 +1,33 @@
+{%- from "ironic/map.jinja" import client,deploy with context %}
+
+{%- if deploy.enabled %}
+
+{%- for identity_name, nodes in deploy.nodes.iteritems() %}
+  {%- for node in nodes %}
+    {%- if node.deployment_profile %}
+
+{%- set ir_n = salt['ironicng.show_node'](node_id=node.name, profile=identity_name) %}
+{%- if ir_n['provision_state'] == 'available' and ir_n['maintenance'] == False %}
+
+node_{{ node.name }}_deployment_started:
+  module.run:
+    - name: ironicng.deploy_node
+    - node_id: {{ node.name }}
+    - profile: {{ identity_name }}
+    - deployment_profile: {{ node.deployment_profile }}
+    - partition_profile: {{ node.partition_profile|default(None) }}
+
+{%- else %}
+node_{{ node.name }}_deployment_started:
+  test.show_notification:
+  - text: |
+      Didn't start deployment on node as node provision_state is
+      {{ ir_n['provision_state'] }} and maintenance is {{ ir_n['maintenance'] }}
+
+{%- endif %}
+
+    {%- endif %} {#- end if node.deployment_profile #}
+  {%- endfor %} {#- end for nodes #}
+{%- endfor %} {#- end client.nodes.iteritems #}
+
+{%- endif %} {#- end if deploy.enabled #}
diff --git a/ironic/files/ocata/ironic.conf b/ironic/files/ocata/ironic.conf
index a7d0fe4..ef0db7a 100644
--- a/ironic/files/ocata/ironic.conf
+++ b/ironic/files/ocata/ironic.conf
@@ -1392,7 +1392,11 @@
 # recommended to set an explicit value for this option.
 # (string value)
 # Allowed values: netboot, local
+{%- if conductor.default_boot_option is defined %}
+default_boot_option = {{ conductor.default_boot_option }}
+{%- else %}
 #default_boot_option = <None>
+{%- endif %}
 
 
 [dhcp]
diff --git a/ironic/map.jinja b/ironic/map.jinja
index 71344d0..3a2ad32 100644
--- a/ironic/map.jinja
+++ b/ironic/map.jinja
@@ -30,7 +30,9 @@
 {% set client = salt['grains.filter_by']({
     'Common': {
       'pkgs': ['python-ironicclient'],
+      'nodes': {}
     },
 }, base='Common', merge=pillar.ironic.get('client', {})) %}
 
+{%set deploy = pillar.ironic.get('deploy', {'enabled': false})%}
 {% set ironic = pillar.get('ironic', {}) %}
diff --git a/metadata/service/client.yml b/metadata/service/client.yml
new file mode 100644
index 0000000..8e33a4b
--- /dev/null
+++ b/metadata/service/client.yml
@@ -0,0 +1,6 @@
+applications:
+  - ironic
+parameters:
+  ironic:
+    client:
+      enabled: True