Updated env file and kube config file handling
New:
- Imported kaas/* repositories (1100+)
- Options for use of specific configs (env and kaas)
Fixes:
- updated header index search
- fixed package name handling
- fixed --force-tag option
Change-Id: Ib51ea9a47db93d6dd0868c32cb389d70053af26c
Related-PROD: PROD-35903
diff --git a/cfg_checker/cli/arguments.py b/cfg_checker/cli/arguments.py
index bd162a8..9ccf2f7 100644
--- a/cfg_checker/cli/arguments.py
+++ b/cfg_checker/cli/arguments.py
@@ -1,3 +1,11 @@
+import os
+
+from cfg_checker.common.settings import pkg_dir
+
+_env_name = "local"
+_config_path = os.path.join(pkg_dir, 'etc', _env_name + '.env')
+
+
def add_global_arguments(parser):
parser.add_argument(
"-d",
@@ -23,17 +31,30 @@
" Example: 'cmp*,ctl01'"
)
parser.add_argument(
- '--kube-config-path',
- metavar='skip_string', default="/root/.kube/config",
+ '--kube-config',
+ metavar='kube_config', default="/root/.kube/config",
help="Kube config path with certificates and keys. "
"Default: '/root/.kube/config'"
)
parser.add_argument(
- '--use-env',
- metavar='use_env',
+ '--force-env-type',
+ metavar='force_env_type',
help="Specify env to use if function supports multiple among detected"
" Example: SALT/KUBE"
)
+ parser.add_argument(
+ '--env-name',
+ metavar='env_name', default=_env_name,
+ help="Use of given env name instead of MCP_ENV var"
+ "Default: {}".format(_env_name)
+ )
+
+ parser.add_argument(
+ '--env-config',
+ metavar='env_config', default=_config_path,
+ help="Force use of given env file"
+ "Default: '{}'".format(_config_path)
+ )
parser.add_argument(
'--skip-nodes-file',
diff --git a/cfg_checker/common/const.py b/cfg_checker/common/const.py
index 2ae41f6..d17db08 100644
--- a/cfg_checker/common/const.py
+++ b/cfg_checker/common/const.py
@@ -80,7 +80,7 @@
"unk": "uknown"
}
-ubuntu_releases = ["trusty", "xenial", "ubuntu", "bionic"]
+ubuntu_releases = ["trusty", "xenial", "ubuntu", "bionic", "focal"]
all_arch = ["amd64"]
repo_types = {
"main": "Officially supported software",
@@ -134,5 +134,6 @@
"20": "train",
"19": "stein",
"18": "rocky",
- "17": "queens"
+ "17": "queens",
+ "00": "not installed"
}
diff --git a/cfg_checker/common/settings.py b/cfg_checker/common/settings.py
index 1a30ac0..b01d684 100644
--- a/cfg_checker/common/settings.py
+++ b/cfg_checker/common/settings.py
@@ -232,7 +232,7 @@
None
)
# Warn user only if Kube env is detected locally
- if self.env_name == "local":
+ if self.env_name == ENV_LOCAL:
if not os.path.exists(self.kube_config_path):
logger_cli.warn(
"Kube config path not found on local env: '{}'".format(
@@ -265,7 +265,7 @@
self.kube_node_keypath = self.ssh_key
self.kube_node_homepath = self.homepath
- def _init_env(self, env_name=None):
+ def _init_env(self, config_path, env_name=None):
"""Inits the environment vars from the env file
Uses simple validation for the values and names
@@ -279,23 +279,18 @@
ConfigException -- on env file failed validation
"""
# load env file as init os.environment with its values
- if env_name is None:
- _env_name = ENV_LOCAL
- else:
- _env_name = env_name
- _config_path = os.path.join(pkg_dir, 'etc', _env_name + '.env')
- if os.path.isfile(_config_path):
- with open(_config_path) as _f:
+ if os.path.isfile(config_path):
+ with open(config_path) as _f:
_list = _f.read().splitlines()
logger_cli.info(
"# Loading env vars from '{}'".format(
- _config_path
+ config_path
)
)
else:
raise ConfigException(
"# Failed to load enviroment vars from '{}'".format(
- _config_path
+ config_path
)
)
self.vars = []
@@ -327,13 +322,13 @@
len(_list)
)
)
- self.env_name = _env_name
+ self.env_name = env_name
def __init__(self, args):
"""Base configuration class. Only values that are common for all scripts
"""
self.ssh_uses_sudo = args.sudo
- self.kube_config_path = args.kube_config_path
+ self.kube_config_path = args.kube_config
self.debug = args.debug
self.force_no_key = args.force_no_key
# Make sure we running on Python 3
@@ -346,10 +341,15 @@
sys.version_info[1]
))
- _env = os.getenv('MCP_ENV', None)
+ # if env name is default, check var too
+ if args.env_name == ENV_LOCAL:
+ _env = os.getenv('MCP_ENV', None)
+ _env = _env if _env else args.env_name
+ else:
+ _env = args.env_name
# Init environment variables from file, validate
- self._init_env(_env)
+ self._init_env(args.env_config, env_name=_env)
# Load Common vars for any type of the env
self._init_mcp_values()
# Detect env types present
@@ -374,11 +374,8 @@
self.mcp_host
)
)
- # Init vars that is specific to detected envs only
- self._init_env_values()
-
# initialize path to folders
- if self.env_name == "local":
+ if self.env_name == ENV_LOCAL:
# names and folders
self.user = self.pw_user.pw_name
self.homepath = self.pw_user.pw_dir
@@ -386,3 +383,6 @@
# names and folders in case of remote env
self.user = self.ssh_user
self.homepath = os.path.join('/home', self.ssh_user)
+
+ # Init vars that is specific to detected envs only
+ self._init_env_values()
diff --git a/cfg_checker/helpers/args_utils.py b/cfg_checker/helpers/args_utils.py
index 7ac431c..68f22d5 100644
--- a/cfg_checker/helpers/args_utils.py
+++ b/cfg_checker/helpers/args_utils.py
@@ -85,9 +85,9 @@
)
def _check_target_vs_used(_tenv):
- if not hasattr(args, 'use_env'):
+ if not hasattr(args, 'force_env_type'):
return _tenv
- elif args.use_env == _tenv or not args.use_env:
+ elif args.force_env_type == _tenv or not args.force_env_type:
return _tenv
else:
raise CommandTypeNotSupportedException(
@@ -95,7 +95,7 @@
"for selected env of '{}'".format(
args.command,
args.type,
- args.use_env
+ args.force_env_type
)
)
@@ -104,14 +104,14 @@
if len(_set) < 1:
_raise_type_not_supported()
if len(_set) > 1:
- if not hasattr(args, 'use_env'):
+ if not hasattr(args, 'force_env_type'):
raise CheckerException(
"Multiple envs detected: {}, use --use-env option".format(
",".join(list(_set))
)
)
- elif args.use_env and args.use_env in _set:
- return args.use_env
+ elif args.force_env_type and args.force_env_type in _set:
+ return args.force_env_type
else:
_raise_type_not_supported()
else:
diff --git a/cfg_checker/modules/packages/checker.py b/cfg_checker/modules/packages/checker.py
index b2e8194..46e98be 100644
--- a/cfg_checker/modules/packages/checker.py
+++ b/cfg_checker/modules/packages/checker.py
@@ -453,7 +453,7 @@
self.master = SaltNodes(config)
super(SaltCloudPackageChecker, self).__init__(
config,
- force_tag=None,
+ force_tag=force_tag,
exclude_keywords=[],
skip_list=None,
skip_list_file=None
@@ -472,7 +472,7 @@
self.master = KubeNodes(config)
super(KubeCloudPackageChecker, self).__init__(
config,
- force_tag=None,
+ force_tag=force_tag,
exclude_keywords=[],
skip_list=None,
skip_list_file=None
diff --git a/cfg_checker/modules/packages/repos.py b/cfg_checker/modules/packages/repos.py
index 824275d..1a040ff 100644
--- a/cfg_checker/modules/packages/repos.py
+++ b/cfg_checker/modules/packages/repos.py
@@ -50,13 +50,9 @@
_val = value
if header:
try:
- _ = next(filter(lambda i: _di[i]["header"] == header, _di))
- # iterator not empty, find index
- for _k, _v in _di.items():
- if _v["header"] == header:
- _index = _k
+ _index = next(i for i in _di if header in _di[i]['header'])
except StopIteration:
- _index = str(len(_di.keys()) + 1)
+ _index = str(len(_di) + 1)
_di[_index] = {
"header": header,
"props": _val
@@ -65,13 +61,10 @@
return _index
else:
try:
- _ = next(filter(lambda i: _di[i] == _val, _di))
+ _index = next(i for i in _di if _val in _di[i])
# iterator not empty, find index
- for _k, _v in _di.items():
- if _v == _val:
- _index = _k
except StopIteration:
- _index = str(len(_di.keys()) + 1)
+ _index = str(len(_di) + 1)
# on save, cast it as str
_di[_index] = _val
finally:
@@ -260,7 +253,7 @@
_total_index = len(_sub_tags)
_index = 0
_progress = Progress(_total_index)
- logger.debug(
+ logger_cli.debug(
"... found {} subtags for '{}'".format(
len(_sub_tags),
_label
@@ -270,7 +263,7 @@
for _stag in _sub_tags:
_u = _baseurl + _stag
_index += 1
- logger.debug(
+ logger_cli.debug(
"... searching repos in '{}/{}'".format(
_label,
_stag
@@ -278,13 +271,13 @@
)
# Searching Package collections
- if _stag in ubuntu_releases:
+ if _stag in ubuntu_releases or _stag.startswith("ubuntu-2020"):
# if stag is the release, this is all packages
_repo["all"][_stag] = []
_repo["all"]["url"] = _n_url(_u)
_path_list = self.search_pkg(_n_url(_u), [])
self._map_repo(_path_list, _repo["all"][_stag])
- logger.info(
+ logger_cli.info(
"-> found {} dists".format(
len(_repo["all"][_stag])
)
@@ -299,7 +292,7 @@
_releases, _ = self._ls_repo_page(_n_url(_u))
for _rel in _releases:
if _rel not in ubuntu_releases:
- logger.debug(
+ logger_cli.debug(
"... skipped unknown ubuntu release: "
"'{}' in '{}'".format(
_rel,
@@ -314,7 +307,7 @@
_path_list,
_repo[_stag][_rel]
)
- logger.info(
+ logger_cli.info(
"-> found {} dists for '{}'".format(
len(_repo[_stag][_rel]),
_rel
@@ -579,7 +572,7 @@
_other_repos = 0
for _c, _d in _info.items():
for _ur, _l in _d.items():
- if _ur in ubuntu_releases:
+ if _ur in ubuntu_releases or _ur.startswith("ubuntu-2020"):
_ubuntu_package_repos += len(_l)
elif _ur != 'url':
_other_repos += len(_l)
@@ -741,10 +734,16 @@
elif _line.startswith(' '):
_desc[_key] += "\n{}".format(_line)
else:
- _key = _line[:_line.index(':')]
- _value = _line[_line.index(':')+1:]
- _key = _key.lower()
+ if _line.endswith(":"):
+ _key = _line[:-1]
+ _value = ""
+ else:
+ _key, _value = _line.split(": ", 1)
+ # _key = _line[:_line.index(':')]
+ # _value = _line[_line.index(':')+1:]
+ # _value = _value if _value[0] != ' ' else _value[1:]
+ _key = _key.lower()
_desc[_key] = _value
# save descriptions if needed
if descriptions:
diff --git a/cfg_checker/nodes.py b/cfg_checker/nodes.py
index 615a1a4..e6943a6 100644
--- a/cfg_checker/nodes.py
+++ b/cfg_checker/nodes.py
@@ -600,13 +600,17 @@
# OpenStack versions
self.mcp_release = ""
# Quick and Dirty way to detect OS release
- _nova_version = self.kube.exec_on_target_pod(
- "nova-manage --version",
- "nova-api-osapi",
- "openstack"
- )
- _nmajor = _nova_version.partition('.')[0]
- self.openstack_release = nova_openstack_versions[_nmajor]
+ try:
+ _nova_version = self.kube.exec_on_target_pod(
+ "nova-manage --version",
+ "nova-api-osapi",
+ "openstack"
+ )
+ _nmajor = _nova_version.partition('.')[0]
+ self.openstack_release = nova_openstack_versions[_nmajor]
+ except KubeException as e:
+ logger_cli.warn("Openstack not detected: {}".format(e.message))
+ self.openstack_release = nova_openstack_versions["00"]
return
@@ -667,7 +671,7 @@
def execute_script_on_node(self, node, script_filename, args=[]):
# Prepare path
_target_path = os.path.join(
- self.env_config.node_homepath,
+ self.env_config.kube_node_homepath,
self.env_config.kube_scripts_folder,
script_filename
)
@@ -747,7 +751,7 @@
_sh.do("apt install python3", sudo=True)
# check if script already there
_folder = os.path.join(
- self.env_config.node_homepath,
+ self.env_config.kube_node_homepath,
_conf.kube_scripts_folder
)
# check if folder exists
@@ -789,7 +793,7 @@
# Prepare script
_source_path = os.path.join(pkg_dir, 'scripts', script_filename)
_target_path = os.path.join(
- self.env_config.node_homepath,
+ self.env_config.kube_node_homepath,
self.env_config.kube_scripts_folder,
script_filename
)
@@ -852,12 +856,12 @@
_source_path = create_temp_file_with_content(_dumps)
_target_path = os.path.join(
- self.env_config.node_homepath,
+ self.env_config.kube_node_homepath,
self.env_config.kube_scripts_folder,
filename
)
_folder = os.path.join(
- self.env_config.node_homepath,
+ self.env_config.kube_node_homepath,
self.env_config.kube_scripts_folder
)
_check = "echo $(if [[ -s '{}' ]]; then echo True; " \