Refactor working with Networks and Pinger class
- Mapper moved to separate module
- Other modules can use Mapper to get desired networks
- salt_master is now a separate single instance
- Updated file handling on salt
- ping.py, an scripted flexible interface to ping command
multithreaded ping execution, 15 at once
- New commands in network: 'ping' and 'list'
- New error when runtime has no network listed in reclass
Fixes:
- Master node code handling
- Unknown node codes detection
- Proper node code search and handling
- File upload procedures updated
- Packages report fix
Change-Id: I5959210aed53b20b04b05ea880218e93239bb661
Related-PROD: PROD-28199
diff --git a/cfg_checker/nodes.py b/cfg_checker/nodes.py
index 5e47447..9a1fd48 100644
--- a/cfg_checker/nodes.py
+++ b/cfg_checker/nodes.py
@@ -1,9 +1,12 @@
+import json
import os
from copy import deepcopy
+from cfg_checker.clients import get_salt_remote, salt
from cfg_checker.common import config, const
from cfg_checker.common import logger, logger_cli
-from cfg_checker.common import salt_utils, utils
+from cfg_checker.common import utils
+from cfg_checker.common.exception import SaltException
from cfg_checker.common.settings import pkg_dir
node_tmpl = {
@@ -17,13 +20,18 @@
class SaltNodes(object):
def __init__(self):
- logger_cli.info("# Collecting nodes")
+ logger_cli.info("# Gathering environment information")
# simple salt rest client
- self.salt = salt_utils.SaltRemote()
+ self.salt = salt
+ self.nodes = None
+ def gather_node_info(self):
# Keys for all nodes
# this is not working in scope of 2016.8.3, will overide with list
logger_cli.debug("... collecting node names existing in the cloud")
+ if not self.salt:
+ self.salt = get_salt_remote(config)
+
try:
_keys = self.salt.list_keys()
_str = []
@@ -85,20 +93,26 @@
)
)
# get master node fqdn
- self.master_node = filter(
+ _filtered = filter(
lambda nd: self.nodes[nd]['role'] == const.all_roles_map['cfg'],
self.nodes
- )[0]
+ )
+ if len(_filtered) < 1:
+ raise SaltException(
+ "No master node detected! Check/Update node role map."
+ )
+ else:
+ self.salt.master_node = _filtered[0]
# OpenStack versions
self.mcp_release = self.salt.pillar_get(
- self.master_node,
+ self.salt.master_node,
"_param:apt_mk_version"
- )[self.master_node]
+ )[self.salt.master_node]
self.openstack_release = self.salt.pillar_get(
- self.master_node,
+ self.salt.master_node,
"_param:openstack_version"
- )[self.master_node]
+ )[self.salt.master_node]
def skip_node(self, node):
# Add node to skip list
@@ -113,6 +127,8 @@
return False
def get_nodes(self):
+ if not self.nodes:
+ self.gather_node_info()
return self.nodes
def get_specific_pillar_for_nodes(self, pillar_path):
@@ -156,7 +172,44 @@
else:
_data[_pillar_keys[-1]] = _result[node]
- def execute_script_on_active_nodes(self, script_filename, args=[]):
+ def prepare_json_on_node(self, node, _dict, filename):
+ # this function assumes that all folders are created
+ _dumps = json.dumps(_dict, indent=2).splitlines()
+ _storage_path = os.path.join(
+ config.salt_file_root, config.salt_scripts_folder
+ )
+ logger_cli.debug(
+ "... uploading data as '{}' "
+ "to master's file cache folder: '{}'".format(
+ filename,
+ _storage_path
+ )
+ )
+ _cache_path = os.path.join(_storage_path, filename)
+ _source_path = os.path.join(
+ 'salt://',
+ config.salt_scripts_folder,
+ filename
+ )
+ _target_path = os.path.join(
+ '/root',
+ config.salt_scripts_folder,
+ filename
+ )
+
+ logger_cli.debug("... creating file in cache '{}'".format(_cache_path))
+ self.salt.f_touch_master(_cache_path)
+ self.salt.f_append_master(_cache_path, _dumps)
+ logger.debug("... syncing file to '{}'".format(node))
+ self.salt.get_file(
+ node,
+ _source_path,
+ _target_path,
+ tgt_type="compound"
+ )
+ return _target_path
+
+ def prepare_script_on_active_nodes(self, script_filename):
# Prepare script
_p = os.path.join(pkg_dir, 'scripts', script_filename)
with open(_p, 'rt') as fd:
@@ -171,7 +224,7 @@
_storage_path
)
)
- self.salt.mkdir("cfg01*", _storage_path)
+ self.salt.mkdir(self.salt.master_node, _storage_path)
# Form cache, source and target path
_cache_path = os.path.join(_storage_path, script_filename)
_source_path = os.path.join(
@@ -202,7 +255,6 @@
),
tgt_type="compound"
)
- logger_cli.info("-> Running script to all active nodes")
logger.debug("... syncing file to nodes")
self.salt.get_file(
self.active_nodes_compound,
@@ -210,7 +262,42 @@
_target_path,
tgt_type="compound"
)
- # execute pkg collecting script
+ # return path on nodes, just in case
+ return _target_path
+
+ def execute_script_on_node(self, node, script_filename, args=[]):
+ # Prepare path
+ _target_path = os.path.join(
+ '/root',
+ config.salt_scripts_folder,
+ script_filename
+ )
+
+ # execute script
+ logger.debug("... running script on '{}'".format(node))
+ # handle results for each node
+ _script_arguments = " ".join(args) if args else ""
+ self.not_responded = []
+ _r = self.salt.cmd(
+ node,
+ 'cmd.run',
+ param='python {} {}'.format(_target_path, _script_arguments),
+ expr_form="compound"
+ )
+
+ # all false returns means that there is no response
+ self.not_responded = [_n for _n in _r.keys() if not _r[_n]]
+ return _r
+
+ def execute_script_on_active_nodes(self, script_filename, args=[]):
+ # Prepare path
+ _target_path = os.path.join(
+ '/root',
+ config.salt_scripts_folder,
+ script_filename
+ )
+
+ # execute script
logger.debug("... running script")
# handle results for each node
_script_arguments = " ".join(args) if args else ""
@@ -237,3 +324,6 @@
return False
else:
return True
+
+
+salt_master = SaltNodes()