Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 1 | # Copyright 2013 - 2017 Mirantis, Inc. |
| 2 | # |
| 3 | # Licensed under the Apache License, Version 2.0 (the "License"); you may |
| 4 | # not use this file except in compliance with the License. You may obtain |
| 5 | # a copy of the License at |
| 6 | # |
| 7 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | # |
| 9 | # Unless required by applicable law or agreed to in writing, software |
| 10 | # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 11 | # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 12 | # License for the specific language governing permissions and limitations |
| 13 | # under the License. |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 14 | |
Dennis Dmitriev | 472159c | 2017-08-31 12:45:06 +0300 | [diff] [blame] | 15 | from __future__ import print_function |
| 16 | |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 17 | # import copy |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 18 | import os |
Dennis Dmitriev | 472159c | 2017-08-31 12:45:06 +0300 | [diff] [blame] | 19 | import sys |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 20 | |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 21 | import yaml |
| 22 | |
Dennis Dmitriev | de847d9 | 2017-06-26 18:58:05 +0300 | [diff] [blame] | 23 | from reclass_tools import helpers |
| 24 | |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 25 | |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 26 | def walkfiles(topdir, verbose=False): |
| 27 | walker = os.walk(topdir) |
| 28 | opener = open |
| 29 | prefix = '' |
| 30 | isdir = os.path.isdir(topdir) |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 31 | |
| 32 | if isdir: |
| 33 | for dirName, subdirList, fileList in walker: |
| 34 | for filename in fileList: |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 35 | filepath = os.path.join(dirName, filename) |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 36 | if verbose: |
| 37 | print (prefix + filepath) |
| 38 | with OpenFile(filepath, opener) as log: |
| 39 | yield (log) |
| 40 | else: |
| 41 | if verbose: |
| 42 | print (topdir) |
| 43 | with OpenFile(topdir, opener) as log: |
| 44 | yield (log) |
| 45 | |
| 46 | |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 47 | # def yaml_read(yaml_file): |
| 48 | # if os.path.isfile(yaml_file): |
| 49 | # with open(yaml_file, 'r') as f: |
| 50 | # return yaml.load(f) |
| 51 | # else: |
| 52 | # print("\'{}\' is not a file!".format(yaml_file)) |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 53 | |
| 54 | |
| 55 | class OpenFile(object): |
| 56 | |
| 57 | fname = None |
| 58 | opener = None |
| 59 | readlines = None |
| 60 | fobj = None |
| 61 | |
| 62 | def __init__(self, fname, opener): |
| 63 | self.fname = fname |
| 64 | self.opener = opener |
| 65 | |
| 66 | def get_parser(self): |
| 67 | parsers = {'/lastlog': self.fake_parser, |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 68 | '/wtmp': self.fake_parser, |
| 69 | '/btmp': self.fake_parser, |
| 70 | '/atop.log': self.fake_parser, |
| 71 | '/atop_': self.fake_parser, |
| 72 | '/atop_current': self.fake_parser, |
| 73 | '/supervisord.log': self.docker_parser, |
| 74 | '.gz': self.gz_parser, |
| 75 | '.bz2': self.gz_parser, |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 76 | } |
| 77 | for w in parsers.keys(): |
| 78 | if w in self.fname: |
| 79 | self.readlines = parsers[w] |
| 80 | return |
| 81 | try: |
| 82 | self.fobj = self.opener(self.fname, 'r') |
| 83 | self.readlines = self.plaintext_parser |
| 84 | except IOError as e: |
| 85 | print("Error opening file {0}: {1}".format(self.fname, e)) |
| 86 | if self.fobj: |
| 87 | self.fobj.close() |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 88 | self.fobj = None |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 89 | self.readlines = self.fake_parser |
| 90 | |
| 91 | def plaintext_parser(self): |
| 92 | try: |
| 93 | for s in self.fobj.readlines(): |
| 94 | yield s |
| 95 | except IOError as e: |
| 96 | print("Error reading file {0}: {1}".format(self.fname, e)) |
| 97 | |
| 98 | def fake_parser(self): |
| 99 | yield '' |
| 100 | |
| 101 | def docker_parser(self): |
| 102 | yield '' |
| 103 | |
| 104 | def gz_parser(self): |
| 105 | yield '' |
| 106 | |
| 107 | def bz2_parser(self): |
| 108 | yield '' |
| 109 | |
| 110 | def __enter__(self): |
| 111 | self.get_parser() |
| 112 | return self |
| 113 | |
| 114 | def __exit__(self, x, y, z): |
| 115 | if self.fobj: |
| 116 | self.fobj.close() |
| 117 | |
| 118 | |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 119 | def get_all_reclass_params(paths, verbose=False): |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 120 | """Return dict with all used values for each param""" |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 121 | _params = dict() |
| 122 | for path in paths: |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 123 | for log in walkfiles(path, verbose): |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 124 | if log.fname.endswith('.yml'): |
Dennis Dmitriev | 65a80ee | 2017-06-30 17:30:37 +0300 | [diff] [blame] | 125 | model = helpers.yaml_read(log.fname) |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 126 | if model is not None: |
| 127 | # Collect all params from the models |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 128 | _param = helpers.get_nested_key( |
| 129 | model, |
| 130 | ['parameters', '_param']) |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 131 | if _param: |
| 132 | for key, val in _param.items(): |
| 133 | if key in _params: |
Dennis Dmitriev | efaa135 | 2017-06-19 12:56:11 +0300 | [diff] [blame] | 134 | # Keep list values sorted |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 135 | _params[key].append(val) |
Dennis Dmitriev | efaa135 | 2017-06-19 12:56:11 +0300 | [diff] [blame] | 136 | _params[key] = sorted(_params[key]) |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 137 | else: |
| 138 | _params[key] = [val] |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 139 | return _params |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 140 | |
| 141 | |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 142 | def add_reclass_parameter(paths, key, value, verbose=False, merge=False): |
| 143 | """Add a value to the specified key to all the files in the paths |
| 144 | |
| 145 | if merge=False (default): |
| 146 | - new value replaces previous key content completely. |
| 147 | |
| 148 | if merge=True: |
| 149 | - if the specified key type is list, then value will be appended |
| 150 | to the list. Value examples: |
| 151 | '1000' |
| 152 | 'new_lab_name' |
| 153 | 'cluster.virtual_cluster_name.infra' |
| 154 | 'http://archive.ubuntu.com' |
| 155 | '[a, b, c]' # a list in the list |
| 156 | '{a:1, b:2, c:3}' # a dict in the list |
| 157 | - if the specified key type is an existing dict, then the dict |
| 158 | will be extended with the dict in the value. Value example: |
| 159 | '{address: 192.168.1.1, netmask: 255.255.255.0}' |
| 160 | |
| 161 | - If the specified key type is string/int/bool - it will replace previous |
| 162 | value |
| 163 | """ |
| 164 | add_key = key.split('.') |
| 165 | |
| 166 | for path in paths: |
| 167 | for fyml in walkfiles(path, verbose=verbose): |
| 168 | if fyml.fname.endswith('.yml'): |
| 169 | model = helpers.yaml_read(fyml.fname) |
| 170 | if model is not None: |
| 171 | |
| 172 | nested_key = helpers.get_nested_key(model, add_key) |
Dennis Dmitriev | 03ad088 | 2017-07-18 19:20:27 +0300 | [diff] [blame] | 173 | |
Dennis Dmitriev | 686c260 | 2018-03-08 17:17:35 +0200 | [diff] [blame] | 174 | if nested_key is not None: |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 175 | if merge is False: |
| 176 | nested_key = value |
Dennis Dmitriev | 472159c | 2017-08-31 12:45:06 +0300 | [diff] [blame] | 177 | else: |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 178 | if type(nested_key) is list: |
| 179 | nested_key.append(value) |
| 180 | elif type(nested_key) is dict: |
| 181 | nested_key.update(value) |
Dennis Dmitriev | 472159c | 2017-08-31 12:45:06 +0300 | [diff] [blame] | 182 | else: |
Dennis Dmitriev | 03ad088 | 2017-07-18 19:20:27 +0300 | [diff] [blame] | 183 | nested_key = value |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 184 | else: |
Dennis Dmitriev | 03ad088 | 2017-07-18 19:20:27 +0300 | [diff] [blame] | 185 | nested_key = value |
| 186 | |
| 187 | helpers.create_nested_key(model, path=add_key, |
| 188 | value=nested_key) |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 189 | |
| 190 | with open(fyml.fname, 'w') as f: |
| 191 | f.write( |
| 192 | yaml.dump( |
Dennis Dmitriev | a7de8b5 | 2018-08-08 23:14:03 +0300 | [diff] [blame^] | 193 | model, default_flow_style=False, width=255 |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 194 | ) |
| 195 | ) |
| 196 | |
| 197 | |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 198 | def remove_reclass_parameter(paths, key, |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 199 | verbose=False, |
| 200 | pretend=False): |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 201 | """Removes specified key from parameters from all reclass models |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 202 | |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 203 | :param key: string with point-separated nested objects, for |
| 204 | example: parameters.linux.network.interface |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 205 | :rtype dict: { 'file path': {nested_key}, ...} |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 206 | """ |
| 207 | remove_key = key.split('.') |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 208 | # found_keys = {} |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 209 | |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 210 | for path in paths: |
| 211 | for fyml in walkfiles(path, verbose=verbose): |
| 212 | if fyml.fname.endswith('.yml'): |
Dennis Dmitriev | e56c8b9 | 2017-06-16 01:53:16 +0300 | [diff] [blame] | 213 | |
Dennis Dmitriev | 472159c | 2017-08-31 12:45:06 +0300 | [diff] [blame] | 214 | try: |
| 215 | model = helpers.yaml_read(fyml.fname) |
| 216 | except yaml.scanner.ScannerError as e: |
| 217 | print(e, file=sys.stderr) |
| 218 | continue |
| 219 | |
| 220 | if model is not None: |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 221 | # Clear linux.network.interfaces |
Dennis Dmitriev | de847d9 | 2017-06-26 18:58:05 +0300 | [diff] [blame] | 222 | nested_key = helpers.get_nested_key(model, remove_key) |
Dennis Dmitriev | 686c260 | 2018-03-08 17:17:35 +0200 | [diff] [blame] | 223 | if nested_key is not None: |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 224 | # found_keys[fyml.fname] = copy.deepcopy(nested_key) |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 225 | if pretend: |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 226 | print("\n---\n# Found {0} in {1}" |
| 227 | .format('.'.join(remove_key), fyml.fname)) |
| 228 | print(yaml.dump(nested_key, |
Dennis Dmitriev | a7de8b5 | 2018-08-08 23:14:03 +0300 | [diff] [blame^] | 229 | default_flow_style=False, |
| 230 | width=255)) |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 231 | else: |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 232 | print("\n---\n# Removing {0} from {1}" |
| 233 | .format('.'.join(remove_key), fyml.fname)) |
| 234 | print(yaml.dump(nested_key, |
Dennis Dmitriev | a7de8b5 | 2018-08-08 23:14:03 +0300 | [diff] [blame^] | 235 | default_flow_style=False, |
| 236 | width=255)) |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 237 | |
Dennis Dmitriev | de847d9 | 2017-06-26 18:58:05 +0300 | [diff] [blame] | 238 | helpers.remove_nested_key(model, remove_key) |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 239 | |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 240 | with open(fyml.fname, 'w') as f: |
| 241 | f.write( |
| 242 | yaml.dump( |
Dennis Dmitriev | a7de8b5 | 2018-08-08 23:14:03 +0300 | [diff] [blame^] | 243 | model, default_flow_style=False, |
| 244 | width=255 |
Dennis Dmitriev | 6bd4425 | 2017-06-22 17:33:40 +0300 | [diff] [blame] | 245 | ) |
Dennis Dmitriev | 672bd44 | 2017-06-16 18:13:54 +0300 | [diff] [blame] | 246 | ) |
Dennis Dmitriev | 566db4b | 2017-07-18 18:13:07 +0300 | [diff] [blame] | 247 | # return found_keys |