Refactored to include varios reports and checks
diff --git a/ci_checker/common/__init__.py b/ci_checker/common/__init__.py
new file mode 100644
index 0000000..e1c9f55
--- /dev/null
+++ b/ci_checker/common/__init__.py
@@ -0,0 +1,15 @@
+import os
+import const
+import log
+
+from base_settings import PKG_DIR, base_config
+from other import Utils
+
+
+utils = Utils()
+const = const
+
+logger, logger_cli = log.setup_loggers(
+ 'cee8_features',
+ log_fname=os.path.join(PKG_DIR, base_config.logfile_name)
+)
diff --git a/ci_checker/common/base_settings.py b/ci_checker/common/base_settings.py
new file mode 100644
index 0000000..f0a8f2b
--- /dev/null
+++ b/ci_checker/common/base_settings.py
@@ -0,0 +1,47 @@
+"""
+Base configuration module
+Gathers env values and supplies default ones
+
+Attributes:
+ base_config: class with all of the values prepared to work with env
+"""
+
+import os
+
+from ci_checker.common.other import utils
+
+PKG_DIR = os.path.dirname(__file__)
+PKG_DIR = os.path.join(PKG_DIR, os.pardir, os.pardir)
+PKG_DIR = os.path.normpath(PKG_DIR)
+
+_default_work_folder = os.path.normpath(PKG_DIR)
+
+
+class TestsConfigurationBase(object):
+ """
+ Base configuration class. Only values that are common for all scripts
+ """
+
+ name = "CiTestsBaseConfig"
+ logfile_name = 'ci_checker.log'
+ working_folder = os.environ.get('CI_TESTS_WORK_DIR', _default_work_folder)
+ salt_host = os.environ.get('SALT_URL', None)
+ salt_port = os.environ.get('SALT_PORT', '6969')
+ salt_user = os.environ.get('SALT_USER', 'salt')
+ salt_pass = os.environ.get('SALT_PASSWORD', None)
+
+ salt_timeout = os.environ.get('SALT_TIMEOUT', 30)
+ salt_file_root = os.environ.get('SALT_FILE_ROOT', None)
+ salt_scripts_folder = os.environ.get('SALT_SCRIPTS_FOLDER', 'test_scripts')
+
+ all_nodes = utils.get_nodes_list(
+ os.environ.get('CI_ALL_NODES', None),
+ os.environ.get('SALT_NODE_LIST_FILE', None)
+ )
+ skip_nodes = utils.node_string_to_list(os.environ.get(
+ 'CI_SKIP_NODES',
+ None
+ ))
+
+
+base_config = TestsConfigurationBase()
diff --git a/ci_checker/common/const.py b/ci_checker/common/const.py
new file mode 100644
index 0000000..4142ea7
--- /dev/null
+++ b/ci_checker/common/const.py
@@ -0,0 +1,32 @@
+"""Constants that is not to be changed and used in all other files
+"""
+
+from __future__ import print_function, absolute_import
+
+import itertools
+
+_cnt = itertools.count()
+NODE_DOWN = next(_cnt)
+NODE_UP = next(_cnt)
+
+del _cnt
+
+all_roles_map = {
+ "apt": "repository",
+ "bmk": "validation",
+ "cfg": "master",
+ "cid": "cicd",
+ "cmn": "storage_monitor",
+ "cmp": "compute",
+ "ctl": "openstack_controller",
+ "dbs": "database",
+ "gtw": "openstack_gateway",
+ "kvm": "foundation",
+ "log": "stacklight_logger",
+ "mon": "monitoring",
+ "msg": "messaging",
+ "mtr": "stacklight_metering",
+ "osd": "storage_node",
+ "prx": "proxy",
+ "rgw": "storage_rados"
+}
diff --git a/ci_checker/common/exception.py b/ci_checker/common/exception.py
new file mode 100644
index 0000000..82e2432
--- /dev/null
+++ b/ci_checker/common/exception.py
@@ -0,0 +1,21 @@
+from exceptions import Exception
+
+
+class CiCheckerBaseExceptions(Exception):
+ pass
+
+
+class CheckerException(CiCheckerBaseExceptions):
+ def __init__(self, message, *args, **kwargs):
+ super(CheckerException, self).__init__(message, *args, **kwargs)
+ # get the trace
+ # TODO: get and log traceback
+
+ # prettify message
+ self.message = "CheckerException: {}".format(message)
+
+
+class ConfigException(CheckerException):
+ def __init__(self, message, *args, **kwargs):
+ super(ConfigException, self).__init__(message, *args, **kwargs)
+ self.message = "Configuration error: {}".format(message)
diff --git a/ci_checker/common/log.py b/ci_checker/common/log.py
new file mode 100644
index 0000000..78f5ab3
--- /dev/null
+++ b/ci_checker/common/log.py
@@ -0,0 +1,80 @@
+import logging
+
+
+def color_me(color):
+ RESET_SEQ = "\033[0m"
+ COLOR_SEQ = "\033[1;%dm"
+
+ color_seq = COLOR_SEQ % (30 + color)
+
+ def closure(msg):
+ return color_seq + msg + RESET_SEQ
+ return closure
+
+
+class ColoredFormatter(logging.Formatter):
+ BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8)
+
+ colors = {
+ 'INFO': color_me(WHITE),
+ 'WARNING': color_me(YELLOW),
+ 'DEBUG': color_me(BLUE),
+ 'CRITICAL': color_me(YELLOW),
+ 'ERROR': color_me(RED)
+ }
+
+ def __init__(self, msg, use_color=True, datefmt=None):
+ logging.Formatter.__init__(self, msg, datefmt=datefmt)
+ self.use_color = use_color
+
+ def format(self, record):
+ orig = record.__dict__
+ record.__dict__ = record.__dict__.copy()
+ levelname = record.levelname
+
+ prn_name = levelname + ' ' * (8 - len(levelname))
+ if levelname in self.colors:
+ record.levelname = self.colors[levelname](prn_name)
+ else:
+ record.levelname = prn_name
+
+ # super doesn't work here in 2.6 O_o
+ res = logging.Formatter.format(self, record)
+
+ # res = super(ColoredFormatter, self).format(record)
+
+ # restore record, as it will be used by other formatters
+ record.__dict__ = orig
+ return res
+
+
+def setup_loggers(name, def_level=logging.DEBUG, log_fname=None):
+
+ # Stream Handler
+ sh = logging.StreamHandler()
+ sh.setLevel(def_level)
+ log_format = '%(message)s'
+ colored_formatter = ColoredFormatter(log_format, datefmt="%H:%M:%S")
+ sh.setFormatter(colored_formatter)
+
+ # File handler
+ if log_fname is not None:
+ fh = logging.FileHandler(log_fname)
+ log_format = '%(asctime)s - %(levelname)8s - %(name)-15s - %(message)s'
+ formatter = logging.Formatter(log_format, datefmt="%H:%M:%S")
+ fh.setFormatter(formatter)
+ fh.setLevel(logging.DEBUG)
+ else:
+ fh = None
+
+ logger = logging.getLogger(name)
+ logger.setLevel(logging.DEBUG)
+ if len(logger.handlers) == 0:
+ logger.addHandler(fh)
+
+ logger_cli = logging.getLogger(name + ".cli")
+ logger_cli.setLevel(logging.DEBUG)
+ if len(logger_cli.handlers) == 0:
+ logger_cli.addHandler(sh)
+
+ return logger, logger_cli
diff --git a/ci_checker/common/other.py b/ci_checker/common/other.py
new file mode 100644
index 0000000..14003ea
--- /dev/null
+++ b/ci_checker/common/other.py
@@ -0,0 +1,98 @@
+import os
+import re
+
+from ci_checker.common.const import all_roles_map
+
+from ci_checker.common.exception import ConfigException
+
+PKG_DIR = os.path.dirname(__file__)
+PKG_DIR = os.path.join(PKG_DIR, os.pardir, os.pardir)
+PKG_DIR = os.path.normpath(PKG_DIR)
+
+
+class Utils(object):
+ @staticmethod
+ def validate_name(fqdn, message=False):
+ """
+ Function that tries to validate node name.
+ Checks if code contains letters, has '.' in it,
+ roles map contains code's role
+
+ :param fqdn: node FQDN name to supply for the check
+ :param message: True if validate should return error check message
+ :return: False if checks failed, True if all checks passed
+ """
+ _message = "Validation passed"
+
+ def _result():
+ return (True, _message) if message else True
+
+ # node role code checks
+ _code = re.findall("[a-zA-Z]+", fqdn.split('.')[0])
+ if len(_code) > 0:
+ if _code[0] in all_roles_map:
+ return _result()
+ else:
+ # log warning here
+ _message = "Node code is unknown, '{}'. " \
+ "Please, update map".format(_code)
+ else:
+ # log warning here
+ _message = "Node name is invalid, '{}'".format(fqdn)
+
+ # put other checks here
+
+ # output result
+ return _result()
+
+ @staticmethod
+ def node_string_to_list(node_string):
+ # supplied var should contain node list
+ # if there is no ',' -> use space as a delimiter
+ if node_string is not None:
+ if node_string.find(',') < 0:
+ return node_string.split(' ')
+ else:
+ return node_string.split(',')
+ else:
+ return []
+
+ def get_node_code(self, fqdn):
+ # validate
+ _isvalid, _message = self.validate_name(fqdn, message=True)
+ _code = re.findall("[a-zA-Z]+", fqdn.split('.')[0])
+ # check if it is valid and raise if not
+ if _isvalid:
+ return _code[0]
+ else:
+ raise ConfigException(_message)
+
+ def get_nodes_list(self, env, nodes_list):
+ _list = []
+ if env is None:
+ # nothing supplied, use the one in repo
+ try:
+ if not nodes_list:
+ return []
+ with open(os.path.join(PKG_DIR, nodes_list)) as _f:
+ _list.extend(_f.read().splitlines())
+ except IOError as e:
+ raise ConfigException("Error while loading file, '{}': "
+ "{}".format(e.filename, e.strerror))
+ else:
+ _list.extend(self.node_string_to_list(env))
+
+ # validate names
+ _invalid = []
+ _valid = []
+ for idx in range(len(_list)):
+ _name = _list[idx]
+ if not self.validate_name(_name):
+ _invalid.append(_name)
+ else:
+ _valid.append(_name)
+
+ return _valid
+
+
+utils = Utils()
diff --git a/ci_checker/common/salt_utils.py b/ci_checker/common/salt_utils.py
new file mode 100644
index 0000000..cbf0321
--- /dev/null
+++ b/ci_checker/common/salt_utils.py
@@ -0,0 +1,372 @@
+"""
+Module to handle interaction with salt
+"""
+import os
+import requests
+import time
+
+from ci_checker.common.base_settings import base_config as config
+from ci_checker.common import logger
+
+
+def list_to_target_string(node_list, separator):
+ result = ''
+ for node in node_list:
+ result += node + ' ' + separator + ' '
+ return result[:-(len(separator)+2)]
+
+
+class SaltRest(object):
+ _host = config.salt_host
+ _port = config.salt_port
+ uri = "http://" + config.salt_host + ":" + config.salt_port
+ _auth = {}
+
+ default_headers = {
+ 'Accept': 'application/json',
+ 'Content-Type': 'application/json',
+ 'X-Auth-Token': None
+ }
+
+ def __init__(self):
+ self._token = self._login()
+ self.last_response = None
+
+ def get(self, path='', headers=default_headers, cookies=None):
+ _path = os.path.join(self.uri, path)
+ logger.debug("GET '{}'\nHeaders: '{}'\nCookies: {}".format(
+ _path,
+ headers,
+ cookies
+ ))
+ return requests.get(
+ _path,
+ headers=headers,
+ cookies=cookies
+ )
+
+ def post(self, data, path='', headers=default_headers, cookies=None):
+ if data is None:
+ data = {}
+ _path = os.path.join(self.uri, path)
+ if path == 'login':
+ _data = str(data).replace(config.salt_pass, "*****")
+ else:
+ _data = data
+ logger.debug("POST '{}'\nHeaders: '{}'\nCookies: {}\nBody: {}".format(
+ _path,
+ headers,
+ cookies,
+ _data
+ ))
+ return requests.post(
+ os.path.join(self.uri, path),
+ headers=headers,
+ json=data,
+ cookies=cookies
+ )
+
+ def _login(self):
+ login_payload = {
+ 'username': config.salt_user,
+ 'password': config.salt_pass,
+ 'eauth': 'pam'
+ }
+
+ logger.debug("Logging in to salt master...")
+ _response = self.post(login_payload, path='login')
+
+ if _response.ok:
+ self._auth['response'] = _response.json()['return'][0]
+ self._auth['cookies'] = _response.cookies
+ self.default_headers['X-Auth-Token'] = \
+ self._auth['response']['token']
+ return self._auth['response']['token']
+ else:
+ raise EnvironmentError(
+ "HTTP:{}, Not authorized?".format(_response.status_code)
+ )
+
+ def salt_request(self, fn, *args, **kwargs):
+ # if token will expire in 5 min, re-login
+ if self._auth['response']['expire'] < time.time() + 300:
+ self._auth['response']['X-Auth-Token'] = self._login()
+
+ _method = getattr(self, fn)
+ _response = _method(*args, **kwargs)
+ self.last_response = _response
+ _content = "..."
+ _len = len(_response.content)
+ if _len < 1024:
+ _content = _response.content
+ logger.debug(
+ "Response (HTTP {}/{}), {}: {}".format(
+ _response.status_code,
+ _response.reason,
+ _len,
+ _content
+ )
+ )
+ if _response.ok:
+ return _response.json()['return']
+ else:
+ raise EnvironmentError(
+ "Salt Error: HTTP:{}, '{}'".format(
+ _response.status_code,
+ _response.reason
+ )
+ )
+
+
+class SaltRemote(SaltRest):
+ def __init__(self):
+ super(SaltRemote, self).__init__()
+
+ def cmd(
+ self,
+ tgt,
+ fun,
+ param=None,
+ client='local',
+ kwarg=None,
+ expr_form=None,
+ tgt_type=None,
+ timeout=None
+ ):
+ _timeout = timeout if timeout is not None else config.salt_timeout
+ _payload = {
+ 'fun': fun,
+ 'tgt': tgt,
+ 'client': client,
+ 'timeout': _timeout
+ }
+
+ if expr_form:
+ _payload['expr_form'] = expr_form
+ if tgt_type:
+ _payload['tgt_type'] = tgt_type
+ if param:
+ _payload['arg'] = param
+ if kwarg:
+ _payload['kwarg'] = kwarg
+
+ _response = self.salt_request('post', [_payload])
+ if isinstance(_response, list):
+ return _response[0]
+ else:
+ raise EnvironmentError(
+ "Unexpected response from from salt-api/LocalClient: "
+ "{}".format(_response)
+ )
+
+ def run(self, fun, kwarg=None):
+ _payload = {
+ 'client': 'runner',
+ 'fun': fun,
+ 'timeout': config.salt_timeout
+ }
+
+ if kwarg:
+ _payload['kwarg'] = kwarg
+
+ _response = self.salt_request('post', [_payload])
+ if isinstance(_response, list):
+ return _response[0]
+ else:
+ raise EnvironmentError(
+ "Unexpected response from from salt-api/RunnerClient: "
+ "{}".format(_response)
+ )
+
+ def wheel(self, fun, arg=None, kwarg=None):
+ _payload = {
+ 'client': 'wheel',
+ 'fun': fun,
+ 'timeout': config.salt_timeout
+ }
+
+ if arg:
+ _payload['arg'] = arg
+ if kwarg:
+ _payload['kwarg'] = kwarg
+
+ _response = self.salt_request('post', _payload)['data']
+ if _response['success']:
+ return _response
+ else:
+ raise EnvironmentError(
+ "Salt Error: '{}'".format(_response['return']))
+
+ def pillar_request(self, node_target, pillar_submodule, argument):
+ # example cli: 'salt "ctl01*" pillar.keys rsyslog'
+ _type = "compound"
+ if isinstance(node_target, list):
+ _type = "list"
+ return self.cmd(
+ node_target,
+ "pillar." + pillar_submodule,
+ argument,
+ expr_form=_type
+ )
+
+ def pillar_keys(self, node_target, argument):
+ return self.pillar_request(node_target, 'keys', argument)
+
+ def pillar_get(self, node_target, argument):
+ return self.pillar_request(node_target, 'get', argument)
+
+ def pillar_data(self, node_target, argument):
+ return self.pillar_request(node_target, 'data', argument)
+
+ def pillar_raw(self, node_target, argument):
+ return self.pillar_request(node_target, 'raw', argument)
+
+ def list_minions(self):
+ """
+ Fails in salt version 2016.3.8
+ api returns dict of minions with grains
+ """
+ return self.salt_request('get', 'minions')
+
+ def list_keys(self):
+ """
+ Fails in salt version 2016.3.8
+ api should return dict:
+ {
+ 'local': [],
+ 'minions': [],
+ 'minions_denied': [],
+ 'minions_pre': [],
+ 'minions_rejected': [],
+ }
+ """
+ return self.salt_request('get', path='keys')
+
+ def get_status(self):
+ """
+ 'runner' client is the equivalent of 'salt-run'
+ Returns the
+ """
+ return self.run(
+ 'manage.status',
+ kwarg={'timeout': 10}
+ )
+
+ def get_active_nodes(self):
+ if config.skip_nodes:
+ logger.info("Nodes to be skipped: {0}".format(config.skip_nodes))
+ return self.cmd(
+ '* and not ' + list_to_target_string(
+ config.skip_nodes,
+ 'and not'
+ ),
+ 'test.ping',
+ expr_form='compound')
+ else:
+ return self.cmd('*', 'test.ping')
+
+ def get_monitoring_ip(self, param_name):
+ salt_output = self.cmd(
+ 'docker:client:stack:monitoring',
+ 'pillar.get',
+ param=param_name,
+ expr_form='pillar')
+ return salt_output[salt_output.keys()[0]]
+
+ def f_touch_master(self, path, makedirs=True):
+ _kwarg = {
+ "makedirs": makedirs
+ }
+ salt_output = self.cmd(
+ "cfg01*",
+ "file.touch",
+ param=path,
+ kwarg=_kwarg
+ )
+ return salt_output[salt_output.keys()[0]]
+
+ def f_append_master(self, path, strings_list, makedirs=True):
+ _kwarg = {
+ "makedirs": makedirs
+ }
+ _args = [path]
+ _args.extend(strings_list)
+ salt_output = self.cmd(
+ "cfg01*",
+ "file.write",
+ param=_args,
+ kwarg=_kwarg
+ )
+ return salt_output[salt_output.keys()[0]]
+
+ def mkdir(self, target, path, tgt_type=None):
+ salt_output = self.cmd(
+ target,
+ "file.mkdir",
+ param=path,
+ expr_form=tgt_type
+ )
+ return salt_output
+
+ def f_manage_file(self, target_path, source,
+ sfn='', ret='{}',
+ source_hash={},
+ user='root', group='root', backup_mode='755',
+ show_diff='base',
+ contents='', makedirs=True):
+ """
+ REST variation of file.get_managed
+ CLI execution goes like this (10 agrs):
+ salt cfg01\* file.manage_file /root/test_scripts/pkg_versions.py
+ '' '{}' /root/diff_pkg_version.py
+ '{hash_type: 'md5', 'hsum': <md5sum>}' root root '755' base ''
+ makedirs=True
+ param: name - target file placement when managed
+ param: source - source for the file
+ """
+ _source_hash = {
+ "hash_type": "md5",
+ "hsum": 000
+ }
+ _arg = [
+ target_path,
+ sfn,
+ ret,
+ source,
+ _source_hash,
+ user,
+ group,
+ backup_mode,
+ show_diff,
+ contents
+ ]
+ _kwarg = {
+ "makedirs": makedirs
+ }
+ salt_output = self.cmd(
+ "cfg01*",
+ "file.manage_file",
+ param=_arg,
+ kwarg=_kwarg
+ )
+ return salt_output[salt_output.keys()[0]]
+
+ def cache_file(self, target, source_path):
+ salt_output = self.cmd(
+ target,
+ "cp.cache_file",
+ param=source_path
+ )
+ return salt_output[salt_output.keys()[0]]
+
+ def get_file(self, target, source_path, target_path, tgt_type=None):
+ return self.cmd(
+ target,
+ "cp.get_file",
+ param=[source_path, target_path],
+ expr_form=tgt_type
+ )
+
+ @staticmethod
+ def compound_string_from_list(nodes_list):
+ return " or ".join(nodes_list)