blob: 533a120e533395ac1cf679c32f4ca51868362e3a [file] [log] [blame]
Dennis Dmitriev566db4b2017-07-18 18:13:07 +03001# Copyright 2013 - 2017 Mirantis, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030014
Dennis Dmitriev472159c2017-08-31 12:45:06 +030015from __future__ import print_function
16
Dennis Dmitriev566db4b2017-07-18 18:13:07 +030017# import copy
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030018import os
Dennis Dmitriev472159c2017-08-31 12:45:06 +030019import sys
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030020
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030021import yaml
22
Dennis Dmitrievde847d92017-06-26 18:58:05 +030023from reclass_tools import helpers
24
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030025
Dennis Dmitriev6bd44252017-06-22 17:33:40 +030026def walkfiles(topdir, verbose=False):
27 walker = os.walk(topdir)
28 opener = open
29 prefix = ''
30 isdir = os.path.isdir(topdir)
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030031
32 if isdir:
33 for dirName, subdirList, fileList in walker:
34 for filename in fileList:
Dennis Dmitriev566db4b2017-07-18 18:13:07 +030035 filepath = os.path.join(dirName, filename)
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030036 if verbose:
37 print (prefix + filepath)
38 with OpenFile(filepath, opener) as log:
39 yield (log)
40 else:
41 if verbose:
42 print (topdir)
43 with OpenFile(topdir, opener) as log:
44 yield (log)
45
46
Dennis Dmitriev566db4b2017-07-18 18:13:07 +030047# def yaml_read(yaml_file):
48# if os.path.isfile(yaml_file):
49# with open(yaml_file, 'r') as f:
50# return yaml.load(f)
51# else:
52# print("\'{}\' is not a file!".format(yaml_file))
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030053
54
55class OpenFile(object):
56
57 fname = None
58 opener = None
59 readlines = None
60 fobj = None
61
62 def __init__(self, fname, opener):
63 self.fname = fname
64 self.opener = opener
65
66 def get_parser(self):
67 parsers = {'/lastlog': self.fake_parser,
Dennis Dmitriev566db4b2017-07-18 18:13:07 +030068 '/wtmp': self.fake_parser,
69 '/btmp': self.fake_parser,
70 '/atop.log': self.fake_parser,
71 '/atop_': self.fake_parser,
72 '/atop_current': self.fake_parser,
73 '/supervisord.log': self.docker_parser,
74 '.gz': self.gz_parser,
75 '.bz2': self.gz_parser,
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030076 }
77 for w in parsers.keys():
78 if w in self.fname:
79 self.readlines = parsers[w]
80 return
81 try:
82 self.fobj = self.opener(self.fname, 'r')
83 self.readlines = self.plaintext_parser
84 except IOError as e:
85 print("Error opening file {0}: {1}".format(self.fname, e))
86 if self.fobj:
87 self.fobj.close()
Dennis Dmitriev566db4b2017-07-18 18:13:07 +030088 self.fobj = None
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +030089 self.readlines = self.fake_parser
90
91 def plaintext_parser(self):
92 try:
93 for s in self.fobj.readlines():
94 yield s
95 except IOError as e:
96 print("Error reading file {0}: {1}".format(self.fname, e))
97
98 def fake_parser(self):
99 yield ''
100
101 def docker_parser(self):
102 yield ''
103
104 def gz_parser(self):
105 yield ''
106
107 def bz2_parser(self):
108 yield ''
109
110 def __enter__(self):
111 self.get_parser()
112 return self
113
114 def __exit__(self, x, y, z):
115 if self.fobj:
116 self.fobj.close()
117
118
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300119def get_all_reclass_params(paths, verbose=False):
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300120 """Return dict with all used values for each param"""
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300121 _params = dict()
122 for path in paths:
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300123 for log in walkfiles(path, verbose):
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300124 if log.fname.endswith('.yml'):
Dennis Dmitriev65a80ee2017-06-30 17:30:37 +0300125 model = helpers.yaml_read(log.fname)
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300126 if model is not None:
127 # Collect all params from the models
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300128 _param = helpers.get_nested_key(
129 model,
130 ['parameters', '_param'])
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300131 if _param:
132 for key, val in _param.items():
133 if key in _params:
Dennis Dmitrievefaa1352017-06-19 12:56:11 +0300134 # Keep list values sorted
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300135 _params[key].append(val)
Dennis Dmitrievefaa1352017-06-19 12:56:11 +0300136 _params[key] = sorted(_params[key])
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300137 else:
138 _params[key] = [val]
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300139 return _params
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300140
141
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300142def add_reclass_parameter(paths, key, value, verbose=False, merge=False):
143 """Add a value to the specified key to all the files in the paths
144
145 if merge=False (default):
146 - new value replaces previous key content completely.
147
148 if merge=True:
149 - if the specified key type is list, then value will be appended
150 to the list. Value examples:
151 '1000'
152 'new_lab_name'
153 'cluster.virtual_cluster_name.infra'
154 'http://archive.ubuntu.com'
155 '[a, b, c]' # a list in the list
156 '{a:1, b:2, c:3}' # a dict in the list
157 - if the specified key type is an existing dict, then the dict
158 will be extended with the dict in the value. Value example:
159 '{address: 192.168.1.1, netmask: 255.255.255.0}'
160
161 - If the specified key type is string/int/bool - it will replace previous
162 value
163 """
164 add_key = key.split('.')
165
166 for path in paths:
167 for fyml in walkfiles(path, verbose=verbose):
168 if fyml.fname.endswith('.yml'):
169 model = helpers.yaml_read(fyml.fname)
170 if model is not None:
171
172 nested_key = helpers.get_nested_key(model, add_key)
Dennis Dmitriev03ad0882017-07-18 19:20:27 +0300173
Dennis Dmitriev686c2602018-03-08 17:17:35 +0200174 if nested_key is not None:
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300175 if merge is False:
176 nested_key = value
Dennis Dmitriev472159c2017-08-31 12:45:06 +0300177 else:
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300178 if type(nested_key) is list:
179 nested_key.append(value)
180 elif type(nested_key) is dict:
181 nested_key.update(value)
Dennis Dmitriev472159c2017-08-31 12:45:06 +0300182 else:
Dennis Dmitriev03ad0882017-07-18 19:20:27 +0300183 nested_key = value
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300184 else:
Dennis Dmitriev03ad0882017-07-18 19:20:27 +0300185 nested_key = value
186
187 helpers.create_nested_key(model, path=add_key,
188 value=nested_key)
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300189
190 with open(fyml.fname, 'w') as f:
191 f.write(
192 yaml.dump(
Dennis Dmitrieva7de8b52018-08-08 23:14:03 +0300193 model, default_flow_style=False, width=255
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300194 )
195 )
196
197
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300198def remove_reclass_parameter(paths, key,
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300199 verbose=False,
200 pretend=False):
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300201 """Removes specified key from parameters from all reclass models
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300202
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300203 :param key: string with point-separated nested objects, for
204 example: parameters.linux.network.interface
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300205 :rtype dict: { 'file path': {nested_key}, ...}
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300206 """
207 remove_key = key.split('.')
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300208 # found_keys = {}
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300209
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300210 for path in paths:
211 for fyml in walkfiles(path, verbose=verbose):
212 if fyml.fname.endswith('.yml'):
Dennis Dmitrieve56c8b92017-06-16 01:53:16 +0300213
Dennis Dmitriev472159c2017-08-31 12:45:06 +0300214 try:
215 model = helpers.yaml_read(fyml.fname)
216 except yaml.scanner.ScannerError as e:
217 print(e, file=sys.stderr)
218 continue
219
220 if model is not None:
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300221 # Clear linux.network.interfaces
Dennis Dmitrievde847d92017-06-26 18:58:05 +0300222 nested_key = helpers.get_nested_key(model, remove_key)
Dennis Dmitriev686c2602018-03-08 17:17:35 +0200223 if nested_key is not None:
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300224 # found_keys[fyml.fname] = copy.deepcopy(nested_key)
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300225 if pretend:
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300226 print("\n---\n# Found {0} in {1}"
227 .format('.'.join(remove_key), fyml.fname))
228 print(yaml.dump(nested_key,
Dennis Dmitrieva7de8b52018-08-08 23:14:03 +0300229 default_flow_style=False,
230 width=255))
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300231 else:
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300232 print("\n---\n# Removing {0} from {1}"
233 .format('.'.join(remove_key), fyml.fname))
234 print(yaml.dump(nested_key,
Dennis Dmitrieva7de8b52018-08-08 23:14:03 +0300235 default_flow_style=False,
236 width=255))
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300237
Dennis Dmitrievde847d92017-06-26 18:58:05 +0300238 helpers.remove_nested_key(model, remove_key)
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300239
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300240 with open(fyml.fname, 'w') as f:
241 f.write(
242 yaml.dump(
Dennis Dmitrieva7de8b52018-08-08 23:14:03 +0300243 model, default_flow_style=False,
244 width=255
Dennis Dmitriev6bd44252017-06-22 17:33:40 +0300245 )
Dennis Dmitriev672bd442017-06-16 18:13:54 +0300246 )
Dennis Dmitriev566db4b2017-07-18 18:13:07 +0300247 # return found_keys