Implement boot-resources control
* Allow to manage boot-resources repo
* Allow to romove 'undefined' boot-resources repos
* Allow to select boot-res. selections. Including for unmanaged bs repos
* Misc: fix reuirment for non-related func
* Fix dep's for maas_config
* Disable rsyslog test pillar for kitchen
- Salt always fail tests with:
'Comment: Service rsyslog is already enabled, and is dead'
Partial-Bug: PROD-16412 (PROD:PROD-16412)
Change-Id: Idb86ffd35ef7e9fe6ce99ca5bcdd87570f8a70c4
diff --git a/.kitchen.yml b/.kitchen.yml
index ca02fdf..9d4e118 100644
--- a/.kitchen.yml
+++ b/.kitchen.yml
@@ -4,8 +4,7 @@
hostname: maas.ci.local
#socket: tcp://127.0.0.1:2376
use_sudo: false
-
-
+ run_options: -v /dev/log:/dev/log:ro
provisioner:
name: salt_solo
@@ -18,21 +17,21 @@
state_top:
base:
"*":
- - rsyslog
+# - rsyslog
- postgresql
- maas
pillars:
top.sls:
base:
"*":
- - rsyslog
+# - rsyslog
- postgresql
- linux
- maas
pillars-from-files:
postgresql.sls: tests/pillar/postgresql.sls
- rsyslog.sls: tests/pillar/rsyslog.sls
+# rsyslog.sls: tests/pillar/rsyslog.sls
linux.sls: tests/pillar/linux.sls
grains:
diff --git a/README.rst b/README.rst
index 0903a6e..2f85ea0 100644
--- a/README.rst
+++ b/README.rst
@@ -64,19 +64,19 @@
description: Test snippet
enabled: true
subnet: subnet1
+ boot_sources_delete_all_others: true
boot_sources:
- maas_mirror:
- url: http://images.maas.io/ephemeral-v3/daily/
+ resources_mirror:
+ url: http://images.maas.io/ephemeral-v3/
keyring_file: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg
- local_mirror:
- url: http://127.0.0.1/maas/images/ephemeral-v3/daily
- keyring_file: /usr/share/keyrings/ubuntu-cloudimage-keyring.gpg
- boot_resources:
- bootscript1:
- title: bootscript
- architecture: amd64/generic
- filetype: tgz
- content: /srv/salt/reclass/nodes/path_to_file
+ boot_sources_selections:
+ xenial:
+ url: "http://images.maas.io/ephemeral-v3/" # should be same in boot_sources, or other already defined.
+ os: "ubuntu"
+ release: "xenial"
+ arches: "amd64"
+ subarches: '"*"'
+ labels: '"*"'
package_repositories:
Saltstack:
url: http://repo.saltstack.com/apt/ubuntu/14.04/amd64/2016.3/
diff --git a/_modules/maasng.py b/_modules/maasng.py
index 8e4d1f2..16fe192 100644
--- a/_modules/maasng.py
+++ b/_modules/maasng.py
@@ -982,9 +982,266 @@
json_res = json.loads(maas.put(
u'api/2.0/fabrics/{0}/vlans/{1}/'.format(fabric_id, vid), **data).read())
- print(json_res)
+ LOG.debug("update_vlan:{}".format(json_res))
result["new"] = "Vlan {0} was updated".format(json_res["name"])
return result
# END NETWORKING
+
+# MAAS CONFIG SECTION
+
+
+def _get_boot_source_id_by_url(url):
+ # FIXME: fix ret\validation
+ try:
+ bs_id = get_boot_source(url=url)["id"]
+ except KeyError:
+ return {"error": "boot-source:{0} not exist!".format(url)}
+ return bs_id
+
+
+def get_boot_source(url=None):
+ """
+ Read a boot source by url. If url not specified - return all.
+
+ CLI Example:
+
+ .. code-block:: bash
+
+ salt 'maas-node' maasng.get_boot_source url
+
+ """
+ boot_sources = {}
+ maas = _create_maas_client()
+ json_res = json.loads(maas.get(u'api/2.0/boot-sources/').read() or 'null')
+ for item in json_res:
+ boot_sources[str(item["url"])] = item
+ if url:
+ return boot_sources.get(url, {})
+ return boot_sources
+
+
+def delete_boot_source(url, bs_id=None):
+ """
+ Delete a boot source by url.
+
+ CLI Example:
+
+ .. code-block:: bash
+
+ sal 'maas-node' maasng.delete url
+
+ """
+ result = {}
+ if not bs_id:
+ bs_id = _get_boot_source_id_by_url(url)
+ maas = _create_maas_client()
+ json_res = json.loads(maas.delete(
+ u'/api/2.0/boot-sources/{0}/'.format(bs_id)).read() or 'null')
+ LOG.debug("delete_boot_source:{}".format(json_res))
+ result["new"] = "Boot-resource {0} deleted".format(url)
+ return result
+
+
+def boot_sources_delete_all_others(except_urls=[]):
+ """
+ Delete all boot-sources, except defined in 'except_urls' list.
+ """
+ result = {}
+ maas_boot_sources = get_boot_source()
+ if 0 in [len(except_urls), len(maas_boot_sources)]:
+ result['result'] = None
+ result[
+ "comment"] = "Exclude or maas sources for delete empty. No changes goinng to be."
+ return result
+ for url in maas_boot_sources.keys():
+ if url not in except_urls:
+ LOG.info("Removing boot-source:{}".format(url))
+ boot_resources_import(action='stop_import', wait=True)
+ result["changes"] = delete_boot_source(url)
+ return result
+
+
+def create_boot_source(url, keyring_filename='', keyring_data='', wait=False):
+ """
+ Create and import maas boot-source: link to maas-ephemeral repo
+ Be aware, those step will import resource to rack ctrl, but you also need to import
+ them into the region!
+
+
+ :param url: The URL of the BootSource.
+ :param keyring_filename: The path to the keyring file for this BootSource.
+ :param keyring_data: The GPG keyring for this BootSource, base64-encoded data.
+
+ """
+
+ # TODO: not work with 'update' currently => keyring update may fail.
+ result = {}
+
+ data = {
+ "url": url,
+ "keyring_filename": keyring_filename,
+ "keyring_data": str(keyring_data),
+ }
+
+ maas = _create_maas_client()
+ ipdb.set_trace()
+ if url in get_boot_source():
+ result['result'] = None
+ result["comment"] = "boot resource already exist"
+ return result
+
+ # NOTE: maas.post will return 400, if url already defined.
+ json_res = json.loads(
+ maas.post(u'api/2.0/boot-sources/', None, **data).read())
+ if wait:
+ LOG.debug(
+ "Sleep for 5s,to get MaaS some time to process previous request")
+ time.sleep(5)
+ ret = boot_resources_is_importing(wait=True)
+ if ret is dict:
+ return ret
+ LOG.debug("create_boot_source:{}".format(json_res))
+ result["new"] = "boot resource {0} was created".format(json_res["url"])
+
+ return result
+
+
+def boot_resources_import(action='import', wait=False):
+ """
+ import/stop_import the boot resources.
+
+ :param action: import\stop_import
+ :param wait: True\False. Wait till process finished.
+
+ CLI Example:
+
+ .. code-block:: bash
+
+ salt 'maas-node' maasng.boot_resources_import action='import'
+
+ """
+ maas = _create_maas_client()
+ # Have no idea why, but usual jsonloads not work here..
+ imp = maas.post(u'api/2.0/boot-resources/', action)
+ if imp.code == 200:
+ LOG.debug('boot_resources_import:{}'.format(imp.readline()))
+ if wait:
+ boot_resources_is_importing(wait=True)
+ return True
+ else:
+ return False
+
+
+def boot_resources_is_importing(wait=False):
+ maas = _create_maas_client()
+ result = {}
+ if wait:
+ started_at = time.time()
+ poll_time = 5
+ timeout = 60 * 15
+ while boot_resources_is_importing(wait=False):
+ c_timeout = timeout - (time.time() - started_at)
+ if c_timeout <= 0:
+ result['result'] = False
+ result["comment"] = "Boot-resources import not finished in time"
+ return result
+ LOG.info(
+ "Waiting boot-resources import done\n"
+ "sleep for:{}s "
+ "Left:{}/{}s".format(poll_time, round(c_timeout), timeout))
+ time.sleep(poll_time)
+ return json.loads(
+ maas.get(u'api/2.0/boot-resources/', 'is_importing').read())
+ else:
+ return json.loads(
+ maas.get(u'api/2.0/boot-resources/', 'is_importing').read())
+
+#####
+#def boot_sources_selections_delete_all_others(except_urls=[]):
+# """
+# """
+# result = {}
+# return result
+
+
+def is_boot_source_selections_in(dict1, list1):
+ """
+ Check that requested boot-selection already in maas bs selections, if True- return bss id.
+ # FIXME: those hack check doesn't look good.
+ """
+ for bs in list1:
+ same = set(dict1.keys()) & set(bs.keys())
+ if all(elem in same for elem in
+ ['os', 'release', 'arches', 'subarches', 'labels']):
+ LOG.debug(
+ "boot-selection in maas:{0}\nlooks same to requested:{1}".format(
+ bs, dict1))
+ return bs['id']
+ return False
+
+
+def get_boot_source_selections(bs_url):
+ """
+ Get boot-source selections.
+ """
+ # check for key_error!
+ bs_id = _get_boot_source_id_by_url(bs_url)
+ maas = _create_maas_client()
+ json_res = json.loads(
+ maas.get(u'/api/2.0/boot-sources/{0}/selections/'.format(bs_id)).read())
+ LOG.debug(
+ "get_boot_source_selections for url:{} \n{}".format(bs_url, json_res))
+ return json_res
+
+
+def create_boot_source_selections(bs_url, os, release, arches="*",
+ subarches="*", labels="*", wait=True):
+ """
+ Create a new boot source selection for bs_url.
+ :param os: The OS (e.g. ubuntu, centos) for which to import resources.Required.
+ :param release: The release for which to import resources. Required.
+ :param arches: The architecture list for which to import resources.
+ :param subarches: The subarchitecture list for which to import resources.
+ :param labels: The label lists for which to import resources.
+ """
+
+ result = {}
+
+ data = {
+ "os": os,
+ "release": release,
+ "arches": arches,
+ "subarches": subarches,
+ "labels": labels,
+ }
+
+ maas = _create_maas_client()
+ bs_id = _get_boot_source_id_by_url(bs_url)
+ # TODO add pre-create verify
+ maas_bs_s = get_boot_source_selections(bs_url)
+ if is_boot_source_selections_in(data, maas_bs_s):
+ result["result"] = True
+ result[
+ "comment"] = 'Requested boot-source selection for {0} already exist.'.format(
+ bs_url)
+ return result
+
+ # NOTE: maas.post will return 400, if url already defined.
+ json_res = json.loads(
+ maas.post(u'api/2.0/boot-sources/{0}/selections/'.format(bs_id), None,
+ **data).read())
+ LOG.debug("create_boot_source_selections:{}".format(json_res))
+ if wait:
+ LOG.debug(
+ "Sleep for 5s,to get MaaS some time to process previous request")
+ time.sleep(5)
+ ret = boot_resources_import(action='import', wait=True)
+ if ret is dict:
+ return ret
+ result["new"] = "boot-source selection for {0} was created".format(bs_url)
+
+ return result
+
+# END MAAS CONFIG SECTION
diff --git a/_states/maasng.py b/_states/maasng.py
index 3d23311..1cc69f7 100644
--- a/_states/maasng.py
+++ b/_states/maasng.py
@@ -25,7 +25,9 @@
return 'maasng'
-def disk_layout_present(hostname, layout_type, root_size=None, root_device=None, volume_group=None, volume_name=None, volume_size=None, disk={}, **kwargs):
+def disk_layout_present(hostname, layout_type, root_size=None, root_device=None,
+ volume_group=None, volume_name=None, volume_size=None,
+ disk={}, **kwargs):
'''
Ensure that the disk layout does exist
@@ -74,7 +76,8 @@
return ret
-def raid_present(hostname, name, level, devices=[], partitions=[], partition_schema={}):
+def raid_present(hostname, name, level, devices=[], partitions=[],
+ partition_schema={}):
'''
Ensure that the raid does exist
@@ -346,7 +349,8 @@
'comment': 'Module function maasng.update_vlan executed'}
ret["changes"] = __salt__['maasng.update_vlan'](
- name=name, fabric=fabric, vid=vid, description=description, primary_rack=primary_rack, dhcp_on=dhcp_on)
+ name=name, fabric=fabric, vid=vid, description=description,
+ primary_rack=primary_rack, dhcp_on=dhcp_on)
if "error" in fabric:
ret['comment'] = "State execution failed for fabric {0}".format(fabric)
@@ -360,3 +364,75 @@
return ret
return ret
+
+
+def boot_source_present(url, keyring_file='', keyring_data=''):
+ """
+ Process maas boot-sources: link to maas-ephemeral repo
+
+
+ :param url: The URL of the BootSource.
+ :param keyring_file: The path to the keyring file for this BootSource.
+ :param keyring_data: The GPG keyring for this BootSource, base64-encoded data.
+ """
+ ret = {'name': url,
+ 'changes': {},
+ 'result': True,
+ 'comment': 'boot-source {0} presented'.format(url)}
+
+ if __opts__['test']:
+ ret['result'] = None
+ ret['comment'] = 'boot-source {0} will be updated'.format(url)
+
+ maas_boot_sources = __salt__['maasng.get_boot_source']()
+ # TODO imlpement check and update for keyrings!
+ if url in maas_boot_sources.keys():
+ ret["result"] = True
+ ret["comment"] = 'boot-source {0} alredy exist'.format(url)
+ return ret
+ ret["changes"] = __salt__['maasng.create_boot_source'](url,
+ keyring_filename=keyring_file,
+ keyring_data=keyring_data)
+ return ret
+
+
+def boot_sources_selections_present(bs_url, os, release, arches="*",
+ subarches="*", labels="*", wait=True):
+ """
+ Process maas boot-sources selection: set of resource configurathions, to be downloaded from boot-source bs_url.
+
+ :param bs_url: Boot-source url
+ :param os: The OS (e.g. ubuntu, centos) for which to import resources.Required.
+ :param release: The release for which to import resources. Required.
+ :param arches: The architecture list for which to import resources.
+ :param subarches: The subarchitecture list for which to import resources.
+ :param labels: The label lists for which to import resources.
+ :param wait: Initiate import and wait for done.
+
+ """
+ ret = {'name': bs_url,
+ 'changes': {},
+ 'result': True,
+ 'comment': 'boot-source {0} selection present'.format(bs_url)}
+
+ if __opts__['test']:
+ ret['result'] = None
+ ret['comment'] = 'boot-source {0} selection will be updated'.format(
+ bs_url)
+
+ maas_boot_sources = __salt__['maasng.get_boot_source']()
+ if bs_url not in maas_boot_sources.keys():
+ ret["result"] = False
+ ret[
+ "comment"] = 'Requested boot-source {0} not exist! Unable to proceed selection for it'.format(
+ bs_url)
+ return ret
+
+ ret["changes"] = __salt__['maasng.create_boot_source_selections'](bs_url,
+ os,
+ release,
+ arches=arches,
+ subarches=subarches,
+ labels=labels,
+ wait=wait)
+ return ret
diff --git a/maas/region.sls b/maas/region.sls
index 2047307..b0d9516 100644
--- a/maas/region.sls
+++ b/maas/region.sls
@@ -142,6 +142,9 @@
interval: 5
splay: 5
{%- endif %}
+ {%- if grains.get('kitchen-test') %}
+ - onlyif: /bin/false
+ {%- endif %}
maas_set_admin_password:
cmd.run:
@@ -162,6 +165,22 @@
- onlyif: /bin/false
{%- endif %}
+maas_wait_for_import_done:
+ module.run:
+ - name: maasng.boot_resources_import
+ - action: 'import'
+ - wait: True
+ - require:
+ - cmd: maas_login_admin
+ {% if region.get('boot_sources_delete_all_others', False) %}
+ - module: region_boot_sources_delete_all_others
+ {%- endif %}
+ - require_in:
+ - module: maas_config
+ {%- if grains.get('kitchen-test') %}
+ - onlyif: /bin/false
+ {%- endif %}
+
maas_config:
module.run:
- name: maas.process_maas_config
@@ -171,14 +190,61 @@
- onlyif: /bin/false
{%- endif %}
-{%- if region.get('boot_sources', False) %}
-maas_boot_sources:
+{##}
+{% if region.get('boot_sources_delete_all_others', False) %}
+ {# Collect exclude list, all other - will be removed #}
+ {% set exclude_list=[] %}
+ {%- for _, bs in region.boot_sources.iteritems() %} {% if bs.url is defined %} {% do exclude_list.append(bs.url) %} {% endif %} {%- endfor %}
+region_boot_sources_delete_all_others:
module.run:
- - name: maas.process_boot_sources
+ - name: maasng.boot_sources_delete_all_others
+ - except_urls: {{ exclude_list }}
- require:
- - cmd: maas_set_admin_password
+ - cmd: maas_login_admin
{%- endif %}
+{##}
+{% if region.get('boot_sources', False) %}
+ {%- for b_name, b_source in region.boot_sources.iteritems() %}
+maas_region_boot_source_{{ b_name }}:
+ maasng.boot_source_present:
+ - url: {{ b_source.url }}
+ {%- if b_source.keyring_data is defined %}
+ - keyring_data: {{ b_source.keyring_data }}
+ {%- endif %}
+ {%- if b_source.keyring_file is defined %}
+ - keyring_file: {{ b_source.keyring_file }}
+ {%- endif %}
+ - require:
+ - cmd: maas_login_admin
+ {%- endfor %}
+{%- endif %}
+
+{##}
+ {% if region.get('boot_sources_selections', False) %}
+ {%- for bs_name, bs_source in region.boot_sources_selections.iteritems() %}
+maas_region_boot_sources_selection_{{ bs_name }}:
+ maasng.boot_sources_selections_present:
+ - bs_url: {{ bs_source.url }}
+ - os: {{ bs_source.os }}
+ - release: {{ bs_source.release|string }}
+ - arches: {{ bs_source.arches|string }}
+ - subarches: {{ bs_source.subarches|string }}
+ - labels: {{ bs_source.labels }}
+ - require_in:
+ - module: maas_config
+ - module: maas_wait_for_import_done
+ - require:
+ - cmd: maas_login_admin
+ {% if region.get('boot_sources', False) %}
+ {%- for b_name, _ in region.boot_sources.iteritems() %}
+ - maas_region_boot_source_{{ b_name }}
+ {% endfor %}
+ {%- endif %}
+ {%- endfor %}
+ {%- endif %}
+{##}
+
{%- if region.get('commissioning_scripts', False) %}
/etc/maas/files/commisioning_scripts/:
file.directory:
@@ -202,7 +268,7 @@
module.run:
- name: maas.process_commissioning_scripts
- require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- endif %}
{%- if region.get('fabrics', False) %}
@@ -210,7 +276,7 @@
module.run:
- name: maas.process_fabrics
- require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- endif %}
{%- if region.get('subnets', False) %}
@@ -218,7 +284,7 @@
module.run:
- name: maas.process_subnets
- require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- if region.get('fabrics', False) %}
- module: maas_fabrics
{%- endif %}
@@ -229,7 +295,7 @@
module.run:
- name: maas.process_devices
- require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- if region.get('subnets', False) %}
- module: maas_subnets
{%- endif %}
@@ -240,7 +306,7 @@
module.run:
- name: maas.process_dhcp_snippets
- require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- endif %}
{%- if region.get('package_repositories', False) %}
@@ -248,22 +314,14 @@
module.run:
- name: maas.process_package_repositories
- require:
- - module: maas_config
-{%- endif %}
-
-{%- if region.get('boot_resources', False) %}
-maas_boot_resources:
- module.run:
- - name: maas.process_boot_resources
- - require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- endif %}
maas_domain:
module.run:
- name: maas.process_domain
- require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- if grains.get('kitchen-test') %}
- onlyif: /bin/false
{%- endif %}
@@ -289,7 +347,7 @@
module.run:
- name: maas.process_sshprefs
- require:
- - module: maas_config
+ - cmd: maas_login_admin
{%- endif %}
{%- endif %}