New command 'add-key'
also, fixed pep8
Change-Id: I526083d72b50dc99b7e945db8d3e95ddbb81459f
diff --git a/reclass_tools/cli.py b/reclass_tools/cli.py
index 2f1807d..898fd52 100644
--- a/reclass_tools/cli.py
+++ b/reclass_tools/cli.py
@@ -15,7 +15,6 @@
from __future__ import print_function
import argparse
-import os
import sys
import yaml
@@ -33,14 +32,22 @@
command_method()
def do_get_key(self):
- results = walk_models.remove_reclass_parameter(
+ walk_models.remove_reclass_parameter(
self.params.path,
self.params.key_name,
verbose=self.params.verbose,
pretend=True)
+ def do_add_key(self):
+ walk_models.add_reclass_parameter(
+ self.params.path,
+ self.params.key_name,
+ self.params.add_value,
+ verbose=self.params.verbose,
+ merge=self.params.merge)
+
def do_del_key(self):
- results = walk_models.remove_reclass_parameter(
+ walk_models.remove_reclass_parameter(
self.params.path,
self.params.key_name,
verbose=self.params.verbose,
@@ -62,7 +69,6 @@
reclass_storage = reclass_models.reclass_storage(inventory=inventory)
print('\n'.join(sorted(reclass_storage.keys())))
-
def do_list_nodes(self):
try:
from reclass_tools import reclass_models
@@ -119,10 +125,15 @@
action='store_const', const=True,
help='Show verbosed output', default=False)
+ merge_parser = argparse.ArgumentParser(add_help=False)
+ merge_parser.add_argument('--merge', dest='verbose',
+ action='store_const', const=True,
+ help='Show verbosed output', default=False)
+
key_parser = argparse.ArgumentParser(add_help=False)
key_parser_help = (
- 'Key name to find in reclass model files, for example:'
- ' parameters.linux.network.interface')
+ 'Key name to find in reclass model files, for example:'
+ ' parameters.linux.network.interface')
key_parser.add_argument('key_name', help=key_parser_help, default=None)
keys_parser = argparse.ArgumentParser(add_help=False)
@@ -130,6 +141,12 @@
'keys',
help='Key names to find in reclass model files', nargs='*')
+ add_value_parser = argparse.ArgumentParser(add_help=False)
+ add_value_parser.add_argument(
+ 'add_value',
+ help=('Value to add to the reclass model files, can be in the '
+ 'inline YAML format'))
+
path_parser = argparse.ArgumentParser(add_help=False)
path_parser.add_argument(
'path',
@@ -175,8 +192,6 @@
help=('YAML/JSON files with context data to render '
'the template'))
-
-
parser = argparse.ArgumentParser(
description="Manage virtual environments. "
"For additional help, use with -h/--help option")
@@ -184,13 +199,19 @@
help='available commands',
dest='command')
- # TODO: add-class NNN [to] MMM.yml # can be used with 'render'
subparsers.add_parser('get-key',
parents=[key_parser, path_parser,
verbose_parser],
help="Find a key in YAMLs found in <path>",
description=("Get a key collected from "
"different YAMLs"))
+ subparsers.add_parser('add-key',
+ parents=[key_parser, add_value_parser,
+ path_parser, verbose_parser,
+ merge_parser],
+ help="Find a key in YAMLs found in <path>",
+ description=("Get a key collected from "
+ "different YAMLs"))
subparsers.add_parser('del-key',
parents=[key_parser, path_parser,
verbose_parser],
diff --git a/reclass_tools/create_inventory.py b/reclass_tools/create_inventory.py
index a8e048b..8bb5840 100644
--- a/reclass_tools/create_inventory.py
+++ b/reclass_tools/create_inventory.py
@@ -1,13 +1,25 @@
-import yaml
-import json
-import sys
+# Copyright 2013 - 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
-from cookiecutter import generate
+import sys
+import yaml
+
from cookiecutter.exceptions import UndefinedVariableInTemplate
+from cookiecutter import generate
from reclass_tools import helpers
from reclass_tools import reclass_models
-from reclass_tools import walk_models
def create_inventory_context(domain=None, keys=None):
@@ -35,10 +47,12 @@
"""
inventory = reclass_models.inventory_list(domain=domain)
vcp_list = reclass_models.vcp_list(domain=domain, inventory=inventory)
- reclass_storage = reclass_models.reclass_storage(domain=domain, inventory=inventory)
+ reclass_storage = reclass_models.reclass_storage(domain=domain,
+ inventory=inventory)
if domain is None:
- sys.exit("Error: please specify a domain name from: \n{}".format('\n'.join(reclass_storage.keys())))
+ sys.exit("Error: please specify a domain name from: \n{}"
+ .format('\n'.join(reclass_storage.keys())))
for storage_domain, storage_nodes in reclass_storage.items():
if storage_domain != domain:
@@ -46,7 +60,8 @@
current_cluster_nodes = {}
for storage_node_name, storage_node in storage_nodes.items():
- inventory_node_name = "{0}.{1}".format(storage_node['name'], storage_node['domain'])
+ inventory_node_name = "{0}.{1}".format(storage_node['name'],
+ storage_node['domain'])
current_cluster_nodes[inventory_node_name] = {
'name': storage_node['name'],
'reclass_storage_name': storage_node_name,
@@ -56,7 +71,8 @@
if (storage_node['name'], storage_node['domain']) in vcp_list:
# Add role 'vcp' to mark the VM nodes.
- current_cluster_nodes[inventory_node_name]['roles'].append('vcp')
+ current_cluster_nodes[
+ inventory_node_name]['roles'].append('vcp')
if keys:
# Dump specified parameters for the node
@@ -68,7 +84,10 @@
key_path = key.split('.')
reclass_key = helpers.get_nested_key(node, path=key_path)
if reclass_key:
- helpers.create_nested_key(current_cluster_nodes[inventory_node_name], path=key_path, value=reclass_key)
+ helpers.create_nested_key(
+ current_cluster_nodes[inventory_node_name],
+ path=key_path,
+ value=reclass_key)
current_underlay_context = {
'cookiecutter': {
@@ -131,6 +150,7 @@
undefined_err.context,
default_flow_style=False
)
- print('='*15 + ' Context: '+ '='*15 + '\n{}'.format(context_str) + '='*40)
+ print('=' * 15 + ' Context: ' + '=' * 15 +
+ '\n{}'.format(context_str) + '='*40)
print('>>> {}'.format(undefined_err.message))
sys.exit('>>> Error message: {}'.format(undefined_err.error.message))
diff --git a/reclass_tools/helpers.py b/reclass_tools/helpers.py
index 322ac71..7ba9d2b 100644
--- a/reclass_tools/helpers.py
+++ b/reclass_tools/helpers.py
@@ -1,5 +1,19 @@
-import os
+# Copyright 2013 - 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
import json
+import os
import yaml
@@ -69,7 +83,8 @@
- Merges dicts and lists
- If a dict key has the suffix '__overwrite__' and boolean value,
then the key is assumed as a special keyword for merging:
- <key>__overwrite__: True # Overwrite the existing <key> content with <key> from obj_2
+ <key>__overwrite__: True # Overwrite the existing <key> content
+ # with <key> from obj_2
<key>__overwrite__: False # Keep the existing <key> content from obj_1
@@ -122,7 +137,8 @@
}
}
- Case #3: Use <key>__overwrite__: False to skip merging key if already exists
+ Case #3: Use <key>__overwrite__: False to skip merging key
+ if already exists
dict_a = {
'host': '1.1.1.1'
@@ -160,9 +176,9 @@
result[key] = value
else:
overwrite_key = key + '__overwrite__'
- if overwrite_key in obj_2 and obj_2[overwrite_key] == True:
+ if overwrite_key in obj_2 and obj_2[overwrite_key] is True:
result[key] = obj_2[key]
- elif overwrite_key in obj_2 and obj_2[overwrite_key] == False:
+ elif overwrite_key in obj_2 and obj_2[overwrite_key] is False:
result[key] = value
else:
result[key] = merge_nested_objects(value, obj_2[key])
diff --git a/reclass_tools/reclass_models.py b/reclass_tools/reclass_models.py
index 7ebb1f3..4f85e5e 100644
--- a/reclass_tools/reclass_models.py
+++ b/reclass_tools/reclass_models.py
@@ -1,11 +1,25 @@
+# Copyright 2013 - 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
import reclass
-from reclass.adapters import salt as reclass_salt
+# from reclass.adapters import salt as reclass_salt
from reclass import config as reclass_config
from reclass import core as reclass_core
from reclass_tools import helpers
-#import salt.cli.call
-#import salt.cli.caller
+# import salt.cli.call
+# import salt.cli.caller
def get_core():
@@ -23,23 +37,24 @@
return reclass_core.Core(storage, None, None)
-#def get_minion_domain():
-# """Try to get domain from the local salt minion"""
-# client = salt.cli.call.SaltCall()
-# client.parse_args(args=['pillar.items'])
-# caller = salt.cli.caller.Caller.factory(client.config)
-# result = caller.call()
-# # Warning! There is a model-related parameter
-# # TODO: move the path to the parameter to a settings/defaults
-# domain = result['return']['_param']['cluster_domain']
-# return domain
+# def get_minion_domain():
+# """Try to get domain from the local salt minion"""
+# client = salt.cli.call.SaltCall()
+# client.parse_args(args=['pillar.items'])
+# caller = salt.cli.caller.Caller.factory(client.config)
+# result = caller.call()
+# # Warning! There is a model-related parameter
+# # TODO(ddmitriev): move the path to the parameter to a settings/defaults
+# domain = result['return']['_param']['cluster_domain']
+# return domain
def inventory_list(domain=None):
core = get_core()
inventory = core.inventory()['nodes']
if domain is not None:
- inventory = {key:val for (key, val) in inventory.items() if key.endswith(domain)}
+ inventory = {key: val for (key, val) in inventory.items()
+ if key.endswith(domain)}
return inventory
@@ -65,9 +80,12 @@
vcp_nodes = helpers.get_nested_key(node, path=vcp_path)
if vcp_nodes is not None:
for vcp_node_name, vcp_node in vcp_nodes.items():
- vcp_node_names.add((vcp_node['name'], helpers.get_nested_key(node, path=domain_path)))
+ vcp_node_names.add((
+ vcp_node['name'],
+ helpers.get_nested_key(node, path=domain_path)))
return vcp_node_names
+
def reclass_storage(domain=None, inventory=None):
"""List VCP node names
@@ -77,12 +95,12 @@
inventory = inventory or inventory_list(domain=domain)
storage_path = 'parameters.reclass.storage.node'.split('.')
- result = dict()
+ res = dict()
for node_name, node in inventory.items():
storage_nodes = helpers.get_nested_key(node, path=storage_path)
if storage_nodes is not None:
for storage_node_name, storage_node in storage_nodes.items():
- if storage_node['domain'] not in result:
- result[storage_node['domain']] = dict()
- result[storage_node['domain']][storage_node_name] = storage_node
- return result
+ if storage_node['domain'] not in res:
+ res[storage_node['domain']] = dict()
+ res[storage_node['domain']][storage_node_name] = storage_node
+ return res
diff --git a/reclass_tools/walk_models.py b/reclass_tools/walk_models.py
index 8b4d91f..0a66033 100644
--- a/reclass_tools/walk_models.py
+++ b/reclass_tools/walk_models.py
@@ -1,11 +1,20 @@
+# Copyright 2013 - 2017 Mirantis, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
-import copy
-import hashlib
+# import copy
import os
-import re
-import tarfile
-import urllib2
import yaml
from reclass_tools import helpers
@@ -20,7 +29,7 @@
if isdir:
for dirName, subdirList, fileList in walker:
for filename in fileList:
- filepath = os.path.join(dirName,filename)
+ filepath = os.path.join(dirName, filename)
if verbose:
print (prefix + filepath)
with OpenFile(filepath, opener) as log:
@@ -32,12 +41,12 @@
yield (log)
-#def yaml_read(yaml_file):
-# if os.path.isfile(yaml_file):
-# with open(yaml_file, 'r') as f:
-# return yaml.load(f)
-# else:
-# print("\'{}\' is not a file!".format(yaml_file))
+# def yaml_read(yaml_file):
+# if os.path.isfile(yaml_file):
+# with open(yaml_file, 'r') as f:
+# return yaml.load(f)
+# else:
+# print("\'{}\' is not a file!".format(yaml_file))
class OpenFile(object):
@@ -53,14 +62,14 @@
def get_parser(self):
parsers = {'/lastlog': self.fake_parser,
- '/wtmp': self.fake_parser,
- '/btmp': self.fake_parser,
- '/atop.log': self.fake_parser,
- '/atop_': self.fake_parser,
- '/atop_current': self.fake_parser,
- '/supervisord.log': self.docker_parser,
- '.gz': self.gz_parser,
- '.bz2': self.gz_parser,
+ '/wtmp': self.fake_parser,
+ '/btmp': self.fake_parser,
+ '/atop.log': self.fake_parser,
+ '/atop_': self.fake_parser,
+ '/atop_current': self.fake_parser,
+ '/supervisord.log': self.docker_parser,
+ '.gz': self.gz_parser,
+ '.bz2': self.gz_parser,
}
for w in parsers.keys():
if w in self.fname:
@@ -73,7 +82,7 @@
print("Error opening file {0}: {1}".format(self.fname, e))
if self.fobj:
self.fobj.close()
- self.fobj = None
+ self.fobj = None
self.readlines = self.fake_parser
def plaintext_parser(self):
@@ -113,7 +122,9 @@
model = helpers.yaml_read(log.fname)
if model is not None:
# Collect all params from the models
- _param = helpers.get_nested_key(model, ['parameters', '_param'])
+ _param = helpers.get_nested_key(
+ model,
+ ['parameters', '_param'])
if _param:
for key, val in _param.items():
if key in _params:
@@ -125,6 +136,60 @@
return _params
+def add_reclass_parameter(paths, key, value, verbose=False, merge=False):
+ """Add a value to the specified key to all the files in the paths
+
+ if merge=False (default):
+ - new value replaces previous key content completely.
+
+ if merge=True:
+ - if the specified key type is list, then value will be appended
+ to the list. Value examples:
+ '1000'
+ 'new_lab_name'
+ 'cluster.virtual_cluster_name.infra'
+ 'http://archive.ubuntu.com'
+ '[a, b, c]' # a list in the list
+ '{a:1, b:2, c:3}' # a dict in the list
+ - if the specified key type is an existing dict, then the dict
+ will be extended with the dict in the value. Value example:
+ '{address: 192.168.1.1, netmask: 255.255.255.0}'
+
+ - If the specified key type is string/int/bool - it will replace previous
+ value
+ """
+ add_key = key.split('.')
+
+ for path in paths:
+ for fyml in walkfiles(path, verbose=verbose):
+ if fyml.fname.endswith('.yml'):
+ model = helpers.yaml_read(fyml.fname)
+ if model is not None:
+
+ nested_key = helpers.get_nested_key(model, add_key)
+ if nested_key:
+ if merge is False:
+ nested_key = value
+ else:
+ if type(nested_key) is list:
+ nested_key.append(value)
+ elif type(nested_key) is dict:
+ nested_key.update(value)
+ else:
+ helpers.create_nested_key(model, path=add_key,
+ value=value)
+ else:
+ helpers.create_nested_key(model, path=add_key,
+ value=value)
+
+ with open(fyml.fname, 'w') as f:
+ f.write(
+ yaml.dump(
+ model, default_flow_style=False
+ )
+ )
+
+
def remove_reclass_parameter(paths, key,
verbose=False,
pretend=False):
@@ -135,7 +200,7 @@
:rtype dict: { 'file path': {nested_key}, ...}
"""
remove_key = key.split('.')
- found_keys = {}
+ # found_keys = {}
for path in paths:
for fyml in walkfiles(path, verbose=verbose):
@@ -146,15 +211,17 @@
# Clear linux.network.interfaces
nested_key = helpers.get_nested_key(model, remove_key)
if nested_key:
- found_keys[fyml.fname] = copy.deepcopy(nested_key)
+ # found_keys[fyml.fname] = copy.deepcopy(nested_key)
if pretend:
- print("\n---\n# Found {0} in {1}".format('.'.join(remove_key),
- fyml.fname))
- print(yaml.dump(nested_key, default_flow_style=False))
+ print("\n---\n# Found {0} in {1}"
+ .format('.'.join(remove_key), fyml.fname))
+ print(yaml.dump(nested_key,
+ default_flow_style=False))
else:
- print("\n---\n# Removing {0} from {1}".format('.'.join(remove_key),
- fyml.fname))
- print(yaml.dump(nested_key, default_flow_style=False))
+ print("\n---\n# Removing {0} from {1}"
+ .format('.'.join(remove_key), fyml.fname))
+ print(yaml.dump(nested_key,
+ default_flow_style=False))
helpers.remove_nested_key(model, remove_key)
@@ -164,4 +231,4 @@
model, default_flow_style=False
)
)
- return found_keys
+ # return found_keys