Log collector module
New:
- [Done] multiple namespace selector
- [Done] keyword-based pod selector
- [Done] per-pod logs syntax detection and parsing
- [Differed] in-place filtering for shorter logs
- [Done] individual logs timestamp detection
- [Done] Unix time bases Timestamp sorting
- [Done] Single file logs output using common format
- [Done] add all log types from all MOS namespaces and pods
Update:
- resource preparation can be skipped per module
- updated log collection using multiple threads
- new setting LOG_COLLECT_THREADS
Fixes:
- Network MTU fix
- Faster cmd execution on single pod
- Ceph benchmark validations
- Ceph benchmark report sorting
- Daemonset deployment with nodes skipped
- Network tree debugging script
- Tree depth limiter, i.e. stackoverflow prevention
Related-PROD: PROD-36845
Change-Id: Icf229ac62078c6418ab4dbdff12b0d27ed42af1d
diff --git a/debug_scripts/dnetwork.py b/debug_scripts/dnetwork.py
new file mode 100644
index 0000000..545eb94
--- /dev/null
+++ b/debug_scripts/dnetwork.py
@@ -0,0 +1,301 @@
+import json
+import ipaddress
+
+from copy import deepcopy
+from cfg_checker.modules.network.network_errors import NetworkErrors
+
+_network_item = {
+ "runtime": {},
+ "config": {},
+ "reclass": {}
+}
+errors = NetworkErrors()
+_console = []
+
+
+def cprint(_str):
+ print(_str)
+ _console.append(_str)
+
+
+# adding net data to tree
+def _add_data(_list, _n, _h, _d):
+ if _n not in _list:
+ _list[_n] = {}
+ _list[_n][_h] = [_d]
+ elif _h not in _list[_n]:
+ # there is no such host, just create it
+ _list[_n][_h] = [_d]
+ else:
+ # there is such host... this is an error
+ errors.add_error(
+ errors.NET_DUPLICATE_IF,
+ host=_h,
+ dup_if=_d['name']
+ )
+ _list[_n][_h].append(_d)
+
+
+def _map_network_for_host(host, if_class, net_list, data):
+ # filter networks for this IF IP
+ _nets = [n for n in net_list.keys() if if_class.ip in n]
+ _masks = [n.netmask for n in _nets]
+ if len(_nets) > 1:
+ # There a multiple network found for this IP, Error
+ errors.add_error(
+ errors.NET_SUBNET_INTERSECT,
+ host=host,
+ ip=str(if_class.exploded),
+ networks="; ".join([str(_n) for _n in _nets])
+ )
+ # check mask match
+ if len(_nets) > 0 and if_class.netmask not in _masks:
+ errors.add_error(
+ errors.NET_MASK_MISMATCH,
+ host=host,
+ if_name=data['name'],
+ if_cidr=if_class.exploded,
+ if_mapped_networks=", ".join([str(_n) for _n in _nets])
+ )
+
+ if len(_nets) < 1:
+ _add_data(net_list, if_class.network, host, data)
+ else:
+ # add all data
+ for net in _nets:
+ _add_data(net_list, net, host, data)
+
+ return net_list
+
+
+host = "tmp-node-00000"
+node_data = {}
+interfaces = {
+ host: {}
+}
+_runtime = {}
+
+with open("ifs_data.json", "rt") as ff:
+ jj = json.loads(ff.read())
+
+node_data['routes'] = jj.pop("routes")
+node_data['networks'] = jj
+
+cprint("# {} Networks".format(len(jj)))
+
+for net_name, net_data in node_data['networks'].items():
+ # cut net name
+ _i = net_name.find('@')
+ _name = net_name if _i < 0 else net_name[:_i]
+ # get ips and calculate subnets
+ if _name in ['lo']:
+ # skip the localhost
+ continue
+ else:
+ # add collected data to interface storage
+ if _name not in interfaces[host]:
+ interfaces[host][_name] = \
+ deepcopy(_network_item)
+ interfaces[host][_name]['runtime'] = \
+ deepcopy(net_data)
+
+ # get data and make sure that wide mask goes first
+ _ip4s = sorted(
+ net_data['ipv4'],
+ key=lambda s: s[s.index('/'):]
+ )
+ for _ip_str in _ip4s:
+ # create interface class
+ _if = ipaddress.IPv4Interface(_ip_str)
+ # check if this is a VIP
+ # ...all those will have /32 mask
+ net_data['vip'] = None
+ if _if.network.prefixlen == 32:
+ net_data['vip'] = str(_if.exploded)
+ if 'name' not in net_data:
+ net_data['name'] = _name
+ if 'ifs' not in net_data:
+ net_data['ifs'] = [_if]
+ # map it
+ _runtime = _map_network_for_host(
+ host,
+ _if,
+ _runtime,
+ net_data
+ )
+ else:
+ # data is already there, just add VIP
+ net_data['ifs'].append(_if)
+
+
+def process_interface(lvl, interface, tree, res):
+ if abs(lvl) > 50:
+ cprint("### ERROR: Probable cyclic dependency, exiting")
+ return
+ cprint("{}{}:{}".format(" "*(10+lvl), lvl, interface))
+ # get childs for each root
+ # tree row item (<if_name>, [<parents>], [<childs>])
+ if lvl not in tree:
+ # - no level - add it
+ tree[lvl] = {}
+ # there is such interface in this level?
+ if interface not in tree[lvl]:
+ # - IF not present
+ _n = ''
+ if interface not in res:
+ _n = 'unknown IF'
+ _p = None
+ _c = None
+ else:
+ # -- get parents, add
+ _p = res[interface]['lower']
+ # -- get childs, add
+ _c = res[interface]['upper']
+
+ # if None, put empty list
+ _p = _p if _p else []
+ # if None, put empty list
+ _c = _c if _c else []
+ tree[lvl].update({
+ interface: {
+ "note": _n,
+ "parents": _p,
+ "children": _c,
+ "size": len(_p) if len(_p) > len(_c) else len(_c)
+ }
+ })
+ for p_if in tree[lvl][interface]["parents"]:
+ # -- cycle: execute process for next parent, lvl-1
+ process_interface(lvl-1, p_if, tree, res)
+ for c_if in tree[lvl][interface]["children"]:
+ # -- cycle: execute process for next child, lvl+1
+ process_interface(lvl+1, c_if, tree, res)
+ else:
+ # - IF present - exit (been here already)
+ cprint("{}{}".format(" "*(10+lvl), "--branch-end--\n"))
+ return
+
+
+def _put(cNet, cIndex, _list):
+ _added = False
+ _actual_index = -1
+ # Check list len
+ _len = len(_list)
+ if cIndex >= _len:
+ # grow list to meet index
+ _list = _list + [''] * (cIndex - _len + 1)
+ _len = len(_list)
+
+ for _cI in range(cIndex, _len):
+ # add child per index
+ # if space is free
+ if not _list[_cI]:
+ _list[_cI] = cNet
+ _added = True
+ _actual_index = _cI
+ break
+ if not _added:
+ # grow list by one entry
+ _list = _list + [cNet]
+ _actual_index = len(_list) - 1
+ return _actual_index, _list
+
+
+# build network hierachy
+nr = node_data['networks']
+# walk interface tree
+for _ifname in node_data['networks']:
+ _tree = {}
+ _level = 0
+ cprint("# -> {}".format(_ifname))
+ process_interface(_level, _ifname, _tree, nr)
+ # save tree for node/if
+ node_data['networks'][_ifname]['tree'] = _tree
+
+ # debug, print built tree
+ cprint("# end '{}'".format(_ifname))
+ lvls = list(_tree.keys())
+ lvls.sort()
+ n = len(lvls)
+ m = max([len(_tree[k].keys()) for k in _tree.keys()])
+ matrix = [["" for i in range(m)] for j in range(n)]
+ x = 0
+ while True:
+ _lv = lvls.pop(0)
+ # get all interfaces on this level
+ nets = iter(_tree[_lv].keys())
+ while True:
+ y = 0
+ # get next interface
+ try:
+ _net = next(nets)
+ except StopIteration:
+ break
+ # all nets
+ _a = [_net]
+ # put current interface if this is only one left
+ if not _tree[_lv][_net]['children']:
+ if _net not in matrix[x]:
+ _, matrix[x] = _put(
+ _net,
+ y,
+ matrix[x]
+ )
+ y += 1
+ else:
+ # get all nets with same child
+ for _c in _tree[_lv][_net]['children']:
+ for _o_net in nets:
+ if _c in _tree[_lv][_o_net]['children']:
+ _a.append(_o_net)
+ # flush collected nets
+ for idx in range(len(_a)):
+ if _a[idx] in matrix[x]:
+ # there is such interface on this level
+ # get index
+ _nI = matrix[x].index(_a[idx])
+ _, matrix[x+1] = _put(
+ _c,
+ _nI,
+ matrix[x+1]
+ )
+ else:
+ # there is no such interface
+ # add it
+ _t, matrix[x] = _put(
+ _a[idx],
+ 0,
+ matrix[x]
+ )
+ # also, put child
+ _, matrix[x+1] = _put(
+ _c,
+ _t,
+ matrix[x+1]
+ )
+ # remove collected nets from processing
+ if _a[idx] in nets:
+ nets.remove(_a[idx])
+ y += len(_a)
+ if not nets:
+ x += 1
+ break
+ if not lvls:
+ break
+
+ lines = []
+ _columns = [len(max([i for i in li])) for li in matrix]
+ for idx_y in range(m):
+ line = ""
+ for idx_x in range(n):
+ _len = _columns[idx_x] if _columns[idx_x] else 1
+ _fmt = "{" + ":{}".format(_len) + "} "
+ line += _fmt.format(matrix[idx_x][idx_y])
+ lines.append(line)
+ node_data['networks'][_ifname]['matrix'] = matrix
+ node_data['networks'][_ifname]['lines'] = lines
+ cprint("#### Tree")
+ cprint("\n".join(lines))
+
+with open("debug_net_console.log", "w+") as ff:
+ ff.write("\n".join(_console))