Multi env support and Kube client integration
Kube friendly Beta
Package versions supports Kube env
Added:
- Env type detection
- New option: --use-env, for selecting env
when function supports multiple detected envs
- Updated config loading
- Each module and command type has supported env check
and stops execution if it is on unsupported env
- Functions can support multiple envs
- Kubernetes dependency
- Kubenernetes API detection: local and remote
- Package checking class hierachy for using Salt or Kube
- Remote pod execution routine
- Flexible SSH/SSH Forwarder classes: with, ssh,do(), etc
- Multithreaded SSH script execution
- Number of workers parameter, default 5
Fixed:
- Config dependency
- Command loading with supported envs list
- Unittests structure and execution flow updated
- Unittests fixes
- Fixed debug mode handling
- Unified command type/support routine
- Nested attrs getter/setter
Change-Id: I3ade693ac21536e2b5dcee4b24d511749dc72759
Related-PROD: PROD-35811
diff --git a/cfg_checker/common/settings.py b/cfg_checker/common/settings.py
index cca5142..deebbc0 100644
--- a/cfg_checker/common/settings.py
+++ b/cfg_checker/common/settings.py
@@ -1,11 +1,15 @@
import os
+import json
+import pwd
import sys
from cfg_checker.common.exception import ConfigException
-
from cfg_checker.common.log import logger_cli
-from cfg_checker.common.other import utils
+from cfg_checker.common.other import utils, shell
+from cfg_checker.common.ssh_utils import ssh_shell_p
+
+from cfg_checker.clients import get_kube_remote
pkg_dir = os.path.dirname(__file__)
pkg_dir = os.path.join(pkg_dir, os.pardir, os.pardir)
@@ -14,6 +18,31 @@
_default_work_folder = os.path.normpath(pkg_dir)
+ENV_TYPE_GLOB = "MCP"
+ENV_TYPE_SALT = "SALT"
+ENV_TYPE_KUBE = "KUBE"
+ENV_TYPE_LINUX = "LINUX"
+
+ENV_LOCAL = "local"
+
+supported_envs = [ENV_TYPE_LINUX, ENV_TYPE_SALT, ENV_TYPE_KUBE]
+
+
+def _extract_salt_return(_raw):
+ if not isinstance(_raw, str):
+ _json = _raw
+ logger_cli.debug("...ambigious return detected")
+ else:
+ try:
+ _json = json.loads(_raw)
+ except ValueError:
+ _json = _raw
+ logger_cli.debug(
+ "...return value is not a json: '{}'".format(_raw)
+ )
+
+ return _json
+
class CheckerConfiguration(object):
@staticmethod
@@ -28,10 +57,117 @@
else:
return None
- def _init_values(self):
+ def _detect(self, _type):
+ logger_cli.debug("...detecting '{}'".format(_type))
+ if _type is None:
+ raise ConfigException("# Unexpected supported env type")
+ elif _type == ENV_TYPE_SALT:
+ # Detect salt env
+ _detect_cmd = ["curl", "-s"]
+ _detect_cmd.append(
+ "http://" + self.mcp_host + ':' + self.salt_port
+ )
+ # Try to call salt API on target host
+ _r = None
+ logger_cli.debug("...trying to detect env type '{}'".format(_type))
+ if self.env_name == ENV_LOCAL:
+ _r = shell(" ".join(_detect_cmd))
+ else:
+ _r = ssh_shell_p(
+ " ".join(_detect_cmd),
+ self.ssh_host,
+ username=self.ssh_user,
+ keypath=self.ssh_key,
+ piped=False,
+ use_sudo=self.ssh_uses_sudo,
+ silent=True
+ )
+ # Parse return
+ _r = _extract_salt_return(_r)
+
+ if len(_r) < 1:
+ return False
+ elif _r["return"] == "Welcome":
+ return True
+ else:
+ return False
+ elif _type == ENV_TYPE_KUBE:
+ _kube = get_kube_remote(self)
+ try:
+ _vApi = _kube.get_versions_api()
+ _v = _vApi.get_code()
+ if hasattr(_v, "platform") and \
+ hasattr(_v, "major") and \
+ hasattr(_v, "minor"):
+ _host = "localhost" if _kube.is_local else _kube.kConf.host
+ logger_cli.info(
+ "# Kube server found: {}:{} on '{}'".format(
+ _v.major,
+ _v.minor,
+ _host
+ )
+ )
+ return True
+ else:
+ return False
+ except Exception as e:
+ logger_cli.warn(
+ "# Unexpected error finding Kube env: '{}' ".format(
+ str(e)
+ )
+ )
+ return False
+ elif _type == ENV_TYPE_LINUX:
+ # Detect Linux env
+ from platform import system, release
+ _s = system()
+ _r = release()
+ logger_cli.debug("...running on {} {}".format(_s, _r))
+ if _s in ['Linux', 'Darwin']:
+ return True
+ else:
+ return False
+ else:
+ raise ConfigException(
+ "# Env type of '{}' is not supported".format(
+ _type
+ )
+ )
+
+ def _detect_types(self):
+ """Try to detect env type based on the name
+ """
+ self.detected_envs = []
+ logger_cli.info('# Detecting env types')
+ for _env in supported_envs:
+ if self._detect(_env):
+ logger_cli.info("# '{}' found".format(_env))
+ self.detected_envs.append(_env)
+ else:
+ logger_cli.info("# '{}' not found".format(_env))
+
+ return
+
+ def _init_mcp_values(self):
"""Load values from environment variables or put default ones
"""
-
+ # filter vars and preload if needed
+ self.salt_vars = []
+ self.kube_vars = []
+ for _key, _value in self.vars:
+ if _key.startswith(ENV_TYPE_GLOB):
+ os.environ[_key] = _value
+ elif _key.startswith(ENV_TYPE_SALT):
+ self.salt_vars.append([_key, _value])
+ elif _key.startswith(ENV_TYPE_KUBE):
+ self.kube_vars.append([_key, _value])
+ else:
+ logger_cli.warn(
+ "Unsupported config variable: '{}={}'".format(
+ _key,
+ _value
+ )
+ )
self.name = "CheckerConfig"
self.working_folder = os.environ.get(
'CFG_TESTS_WORK_DIR',
@@ -43,24 +179,86 @@
self.pkg_versions_map = 'versions_map.csv'
self.ssh_uses_sudo = False
- self.ssh_key = os.environ.get('SSH_KEY', None)
- self.ssh_user = os.environ.get('SSH_USER', None)
- self.ssh_host = os.environ.get('SSH_HOST', None)
+ self.ssh_key = os.environ.get('MCP_SSH_KEY', None)
+ self.ssh_user = os.environ.get('MCP_SSH_USER', None)
+ self.ssh_host = os.environ.get('MCP_SSH_HOST', None)
- self.salt_host = os.environ.get('SALT_URL', None)
- self.salt_port = os.environ.get('SALT_PORT', '6969')
- self.salt_user = os.environ.get('SALT_USER', 'salt')
- self.salt_timeout = os.environ.get('SALT_TIMEOUT', 30)
- self.salt_file_root = os.environ.get('SALT_FILE_ROOT', None)
- self.salt_scripts_folder = os.environ.get(
- 'SALT_SCRIPTS_FOLDER',
- 'cfg_checker_scripts'
- )
+ self.mcp_host = os.environ.get('MCP_ENV_HOST', None)
+ self.salt_port = os.environ.get('MCP_SALT_PORT', '6969')
+ self.threads = int(os.environ.get('MCP_THREADS', "5"))
self.skip_nodes = utils.node_string_to_list(os.environ.get(
'CFG_SKIP_NODES',
None
))
+ # prebuild user data and folder path
+ self.pw_user = pwd.getpwuid(os.getuid())
+ if self.env_name == "local":
+ pass
+ else:
+ if not self.ssh_key and not self.force_no_key:
+ raise ConfigException(
+ "Please, supply a key for the cluster's master node. "
+ "Use MCP_SSH_KEY, see 'etc/example.env'"
+ )
+
+ def _init_env_values(self):
+ if ENV_TYPE_SALT in self.detected_envs:
+ for _key, _value in self.salt_vars:
+ os.environ[_key] = _value
+
+ self.salt_user = os.environ.get('SALT_USER', 'salt')
+ self.salt_timeout = os.environ.get('SALT_TIMEOUT', 30)
+ self.salt_file_root = os.environ.get('SALT_FILE_ROOT', None)
+ self.salt_scripts_folder = os.environ.get(
+ 'SALT_SCRIPTS_FOLDER',
+ 'cfg_checker_scripts'
+ )
+ elif ENV_TYPE_KUBE in self.detected_envs:
+ for _key, _value in self.kube_vars:
+ os.environ[_key] = _value
+
+ self.kube_config_root = os.environ.get('KUBE_CONFIG_ROOT', None)
+ self.kube_scripts_folder = os.environ.get(
+ 'KUBE_SCRIPTS_FOLDER',
+ None
+ )
+ self.kube_node_user = os.environ.get(
+ 'KUBE_NODE_USER',
+ 'ubuntu'
+ )
+ self.kube_node_keypath = os.environ.get(
+ 'KUBE_NODE_KEYPATH',
+ None
+ )
+ # Warn user only if Kube env is detected locally
+ if self.env_name == "local":
+ if not os.path.exists(self.kube_config_path):
+ logger_cli.warn(
+ "Kube config path not found on local env: '{}'".format(
+ self.kube_config_path
+ )
+ )
+ # On local envs, KUBE_NODE_KEYPATH is mandatory and is
+ # provided to give cfg-checker access to kube nodes
+ if not self.kube_node_keypath and not self.force_no_key:
+ raise ConfigException(
+ "Please, supply a key for the cluster nodes. "
+ "Use KUBE_NODE_KEYPATH, see 'etc/example.env'. "
+ "Consider checking KUBE_NODE_USER as well"
+ )
+ else:
+ # Init keys for nodes in case of remote env
+ # KUBE_NODE_KEYPATH is provided in case of nodes key would be
+ # different to master nodes key, which is supplied
+ # using MCP_SSH_KEY (mandatory) and, for the most cases,
+ # should be the same for remote envs
+ if not self.kube_node_keypath and not self.force_no_key:
+ logger_cli.debug(
+ "... using MCP_SSH_KEY as node keys. "
+ "Supply KUBE_NODE_KEYPATH to update."
+ )
+ self.kube_node_keypath = self.ssh_key
def _init_env(self, env_name=None):
"""Inits the environment vars from the env file
@@ -69,6 +267,7 @@
Keyword Arguments:
env_name {str} -- environment name to search configuration
files in etc/<env_name>.env (default: {None})
+ env_type {str} -- environment type to use: salt/kube
Raises:
ConfigException -- on IO error when loading env file
@@ -76,7 +275,7 @@
"""
# load env file as init os.environment with its values
if env_name is None:
- _env_name = 'local'
+ _env_name = ENV_LOCAL
else:
_env_name = env_name
_config_path = os.path.join(pkg_dir, 'etc', _env_name + '.env')
@@ -94,6 +293,7 @@
_config_path
)
)
+ self.vars = []
for index in range(len(_list)):
_line = _list[index]
# skip comments
@@ -101,13 +301,14 @@
continue
# validate
_errors = []
- if _line.find('=') < 0 or _line.count('=') > 1:
+ if len(_line) < 1:
+ _errors.append("Line {}: empty".format(index))
+ elif _line.find('=') < 0 or _line.count('=') > 1:
_errors.append("Line {}: {}".format(index, _line))
else:
# save values
_t = _line.split('=')
- _key, _value = _t[0], _t[1]
- os.environ[_key] = _value
+ self.vars.append([_t[0], _t[1]])
# if there was errors, report them
if _errors:
raise ConfigException(
@@ -121,11 +322,15 @@
len(_list)
)
)
- self.salt_env = _env_name
+ self.env_name = _env_name
- def __init__(self):
+ def __init__(self, args):
"""Base configuration class. Only values that are common for all scripts
"""
+ self.ssh_uses_sudo = args.sudo
+ self.kube_config_path = args.kube_config_path
+ self.debug = args.debug
+ self.force_no_key = args.force_no_key
# Make sure we running on Python 3
if sys.version_info[0] < 3 and sys.version_info[1] < 5:
logger_cli.error("# ERROR: Python 3.5+ is required")
@@ -136,9 +341,45 @@
sys.version_info[1]
))
- _env = os.getenv('SALT_ENV', None)
+ _env = os.getenv('MCP_ENV', None)
+
+ # Init environment variables from file, validate
self._init_env(_env)
- self._init_values()
+ # Load Common vars for any type of the env
+ self._init_mcp_values()
+ # Detect env types present
+ self._detect_types()
+ # handle forced env type var
+ _forced_type = os.getenv('MCP_TYPE_FORCE', None)
+ if _forced_type in supported_envs:
+ self.detected_envs.append(_forced_type)
+ elif _forced_type is not None:
+ logger_cli.warn(
+ "Unsupported forced type of '{}'".format(
+ _forced_type
+ )
+ )
+ # Check if any of the envs detected
+ if len(self.detected_envs) < 1:
+ if _env is None:
+ raise ConfigException("No environment types detected locally")
+ else:
+ raise ConfigException(
+ "No environment types detected at '{}'".format(
+ self.mcp_host
+ )
+ )
+ # Init vars that is specific to detected envs only
+ self._init_env_values()
-
-config = CheckerConfiguration()
+ # initialize path to folders
+ if self.env_name == "local":
+ # names and folders
+ self.user = self.pw_user.pw_name
+ self.homepath = self.pw_user.pw_dir
+ self.node_homepath = os.path.join('/home', self.kube_node_user)
+ else:
+ # names and folders in case of remote env
+ self.user = self.ssh_user
+ self.homepath = os.path.join('/home', self.ssh_user)
+ self.node_homepath = self.homepath