Network check for MCC/MOS
- Network info gathering using DaemonSet with 'hostNetwork=True'
- DaemonSet handling routines
- Mapper and Checker refactoring for Kube
Fixes
- SSH timeouts handling using env vars
MCP_SSH_TIMEOUT when connecting
MCP_SCRIPT_RUN_TIMEOUT when running command
- Progress class supports 0 as an index
Related-PROD: PROD-36575
Change-Id: Ie03a9051007eeb788901acae3696ea2bfdfe33e2
diff --git a/cfg_checker/modules/network/__init__.py b/cfg_checker/modules/network/__init__.py
index 005ca38..2b806d6 100644
--- a/cfg_checker/modules/network/__init__.py
+++ b/cfg_checker/modules/network/__init__.py
@@ -6,7 +6,7 @@
command_help = "Network infrastructure checks and reports"
-supported_envs = [ENV_TYPE_SALT]
+supported_envs = [ENV_TYPE_SALT, ENV_TYPE_KUBE]
def _selectClass(_env, strClassHint="checker"):
@@ -96,7 +96,11 @@
# should not print map, etc...
# Just bare summary and errors
# Check if there is supported env found
- _env = args_utils.check_supported_env(ENV_TYPE_SALT, args, config)
+ _env = args_utils.check_supported_env(
+ [ENV_TYPE_SALT, ENV_TYPE_KUBE],
+ args,
+ config
+ )
# Start command
logger_cli.info("# Network check to console")
_skip, _skip_file = args_utils.get_skip_args(args)
@@ -122,7 +126,11 @@
def do_report(args, config):
# Network Report
# Check if there is supported env found
- _env = args_utils.check_supported_env(ENV_TYPE_SALT, args, config)
+ _env = args_utils.check_supported_env(
+ [ENV_TYPE_SALT],
+ args,
+ config
+ )
# Start command
logger_cli.info("# Network report (check, node map")
@@ -146,7 +154,11 @@
def do_map(args, config):
# Network Map
# Check if there is supported env found
- _env = args_utils.check_supported_env(ENV_TYPE_SALT, args, config)
+ _env = args_utils.check_supported_env(
+ [ENV_TYPE_SALT, ENV_TYPE_KUBE],
+ args,
+ config
+ )
# Start command
logger_cli.info("# Network report")
_skip, _skip_file = args_utils.get_skip_args(args)
@@ -156,7 +168,7 @@
skip_list=_skip,
skip_list_file=_skip_file
)
- networkMap.prepare_all_maps()
+ networkMap.map_networks()
networkMap.create_map()
networkMap.print_map()
@@ -166,7 +178,11 @@
def do_list(args, config):
# Network List
# Check if there is supported env found
- _env = args_utils.check_supported_env(ENV_TYPE_SALT, args, config)
+ _env = args_utils.check_supported_env(
+ [ENV_TYPE_SALT, ENV_TYPE_KUBE],
+ args,
+ config
+ )
# Start command
_skip, _skip_file = args_utils.get_skip_args(args)
_class = _selectClass(_env, strClassHint="mapper")
@@ -175,12 +191,14 @@
skip_list=_skip,
skip_list_file=_skip_file
)
- reclass = _map.map_network(_map.RECLASS)
- runtime = _map.map_network(_map.RUNTIME)
+ logger_cli.info("# Mapping networks")
+ if _env == ENV_TYPE_SALT:
+ reclass = _map.map_network(_map.RECLASS)
+ _s = [str(_n) for _n in reclass.keys()]
+ logger_cli.info("\n# Reclass networks list")
+ logger_cli.info("\n".join(_s))
- _s = [str(_n) for _n in reclass.keys()]
- logger_cli.info("\n# Reclass networks list")
- logger_cli.info("\n".join(_s))
+ runtime = _map.map_network(_map.RUNTIME)
_s = [str(_n) for _n in runtime.keys()]
logger_cli.info("\n# Runtime networks list")
logger_cli.info("\n".join(_s))
diff --git a/cfg_checker/modules/network/checker.py b/cfg_checker/modules/network/checker.py
index c2de426..b5809f3 100644
--- a/cfg_checker/modules/network/checker.py
+++ b/cfg_checker/modules/network/checker.py
@@ -11,9 +11,7 @@
self.errors = NetworkErrors()
def check_networks(self, map=True):
- self.mapper.map_network(self.mapper.RECLASS)
- self.mapper.map_network(self.mapper.RUNTIME)
-
+ self.mapper.map_networks()
self.mapper.create_map()
if map:
self.mapper.print_map()
diff --git a/cfg_checker/modules/network/mapper.py b/cfg_checker/modules/network/mapper.py
index 2e925f2..fff6bb6 100644
--- a/cfg_checker/modules/network/mapper.py
+++ b/cfg_checker/modules/network/mapper.py
@@ -4,6 +4,8 @@
from cfg_checker.common import logger_cli
from cfg_checker.common.exception import InvalidReturnException
+from cfg_checker.common.exception import ConfigException
+from cfg_checker.common.exception import KubeException
from cfg_checker.modules.network.network_errors import NetworkErrors
from cfg_checker.nodes import SaltNodes, KubeNodes
@@ -57,11 +59,6 @@
logger_cli.debug("... init error logs folder")
self.errors = NetworkErrors()
- def prepare_all_maps(self):
- self.map_network(self.RECLASS)
- self.map_network(self.RUNTIME)
- self.map_network(self.CONFIG)
-
# adding net data to tree
def _add_data(self, _list, _n, _h, _d):
if _n not in _list:
@@ -168,23 +165,17 @@
return _confs
- def _map_runtime_networks(self):
+ def _map_runtime_networks(self, result):
# class uses nodes from self.nodes dict
_runtime = {}
- logger_cli.info("# Mapping node runtime network data")
- self.master.prepare_script_on_active_nodes("ifs_data.py")
- _result = self.master.execute_script_on_active_nodes(
- "ifs_data.py",
- args=["json"]
- )
for key in self.master.nodes.keys():
# check if we are to work with this node
if not self.master.is_node_available(key):
continue
# due to much data to be passed from master,
# it is happening in order
- if key in _result:
- _text = _result[key]
+ if key in result:
+ _text = result[key]
if '{' in _text and '}' in _text:
_text = _text[_text.find('{'):]
else:
@@ -415,6 +406,40 @@
node_data['networks'][_ifname]['lines'] = lines
return _runtime
+
+class SaltNetworkMapper(NetworkMapper):
+ def __init__(
+ self,
+ config,
+ errors_class=None,
+ skip_list=None,
+ skip_list_file=None
+ ):
+ self.master = SaltNodes(config)
+ super(SaltNetworkMapper, self).__init__(
+ config,
+ errors_class=errors_class,
+ skip_list=skip_list,
+ skip_list_file=skip_list_file
+ )
+
+ def get_script_output(self):
+ """
+ Get runtime networks by executing script on nodes
+ """
+ logger_cli.info("# Mapping node runtime network data")
+ self.master.prepare_script_on_active_nodes("ifs_data.py")
+ _result = self.master.execute_script_on_active_nodes(
+ "ifs_data.py",
+ args="json"
+ )
+
+ return _result
+
+ def map_networks(self):
+ self.map_network(self.RECLASS)
+ self.map_network(self.RUNTIME)
+
def map_network(self, source):
# maps target network using given source
_networks = None
@@ -424,7 +449,8 @@
elif source == self.CONFIG:
_networks = self._map_configured_networks()
elif source == self.RUNTIME:
- _networks = self._map_runtime_networks()
+ _r = self.get_script_output()
+ _networks = self._map_runtime_networks(_r)
self.networks[source] = _networks
return _networks
@@ -662,9 +688,6 @@
# save map
self.map = _map
- # other runtime networks found
- # docker, etc
-
return
def print_map(self):
@@ -732,23 +755,7 @@
# " {0:8} {1}".format(hostname.split('.')[0], _text)
# )
# logger_cli.info("\n")
-
-
-class SaltNetworkMapper(NetworkMapper):
- def __init__(
- self,
- config,
- errors_class=None,
- skip_list=None,
- skip_list_file=None
- ):
- self.master = SaltNodes(config)
- super(SaltNetworkMapper, self).__init__(
- config,
- errors_class=errors_class,
- skip_list=skip_list,
- skip_list_file=skip_list_file
- )
+ return
class KubeNetworkMapper(NetworkMapper):
@@ -766,3 +773,156 @@
skip_list=skip_list,
skip_list_file=skip_list_file
)
+
+ def get_script_output(self, script, args=None):
+ """
+ Get runtime network by creating DaemonSet with Host network parameter
+ """
+ # prepare daemonset
+ logger_cli.info("-> Preparing daemonset to get node info")
+ _daemonset = self.master.prepare_daemonset(
+ "daemonset_template.yaml",
+ config_map=script
+ )
+
+ # wait for daemonset, normally less than 60 sec for all
+ # but still, let us give it 10 second per pod
+ _timeout = self.master.nodes.__len__() * 10
+ if not self.master.wait_for_daemonset(_daemonset, timeout=_timeout):
+ raise KubeException("Daemonset deployment fail")
+ logger_cli.info("-> Running script on daemonset")
+ # exec script on all pods in daemonset
+ _result = self.master.execute_script_on_daemon_set(
+ _daemonset,
+ script,
+ args=args
+ )
+
+ # delete daemonset
+ self.master.delete_daemonset(_daemonset)
+
+ return _result
+
+ def map_networks(self):
+ self.map_network(self.RUNTIME)
+
+ def map_network(self, source):
+ # maps target network using given source
+ _networks = None
+
+ if source == self.RUNTIME:
+ logger_cli.info("# Mapping node runtime network data")
+ _r = self.get_script_output("ifs_data.py", args="json")
+ _networks = self._map_runtime_networks(_r)
+ else:
+ raise ConfigException(
+ "Network type not supported in 'Kube': '{}'".format(source)
+ )
+
+ self.networks[source] = _networks
+ return _networks
+
+ def create_map(self):
+ """Create all needed elements for map output
+
+ :return: none
+ """
+ _runtime = self.networks[self.RUNTIME]
+
+ # main networks, target vars
+ _map = {}
+ # No matter of proto, at least one IP will be present for the network
+ # we interested in, since we are to make sure that L3 level
+ # is configured according to reclass model
+ for network in _runtime:
+ # shortcuts
+ _net = str(network)
+ _map[_net] = {}
+ # hostnames
+ names = sorted(_runtime[network].keys())
+ for hostname in names:
+ _notes = []
+ node = hostname.split('.')[0]
+ if not self.master.is_node_available(hostname, log=False):
+ logger_cli.info(
+ " {0:8} {1}".format(node, "node not available")
+ )
+ # add non-responsive node erorr
+ self.errors.add_error(
+ self.errors.NET_NODE_NON_RESPONSIVE,
+ host=hostname
+ )
+ _notes.append(
+ self.errors.get_error_type_text(
+ self.errors.NET_NODE_NON_RESPONSIVE
+ )
+ )
+ continue
+ # lookup interface name on node using network CIDR
+ _if_name = _runtime[network][hostname][0]["name"]
+ _raw = self.interfaces[hostname][_if_name]['runtime']
+ _if_name_suffix = ""
+ _a = _runtime[network][hostname]
+ for _host in _a:
+ for _if in _host['ifs']:
+ _ip_str = str(_if.exploded)
+
+ # Save all data
+ _values = {
+ "interface": _if_name,
+ "interface_note": _if_name_suffix,
+ "interface_map": "\n".join(_host['lines']),
+ "interface_matrix": _host['matrix'],
+ "ip_address": _ip_str,
+ "rt_mtu": _host['mtu'],
+ "status": _host['state'],
+ "raw_data": _raw,
+ }
+ if node in _map[_net]:
+ # add if to host
+ _map[_net][node].append(_values)
+ else:
+ _map[_net][node] = [_values]
+ _notes = []
+
+ # save map
+ self.map = _map
+ return
+
+ def print_map(self):
+ """
+ Create text report for CLI
+
+ :return: none
+ """
+ logger_cli.info("# Networks")
+ logger_cli.info(
+ " {0:8} {1:25} {2:25} {3:10} {4:10}".format(
+ "Host",
+ "IF",
+ "IP",
+ "MTU",
+ "State"
+ )
+ )
+ for network in self.map.keys():
+ logger_cli.info("-> {}".format(network))
+ for hostname in self.map[network].keys():
+ node = hostname.split('.')[0]
+ _n = self.map[network][hostname]
+ for _i in _n:
+ # Host IF IP Proto MTU State Gate Def.Gate
+ _text = "{:7} {:17} {:25} {:5} {:10}".format(
+ _i['interface'],
+ _i['interface_note'],
+ _i['ip_address'],
+ _i['rt_mtu'],
+ _i['status']
+ )
+ logger_cli.info(
+ " {0:8} {1}".format(
+ node,
+ _text
+ )
+ )
+ return