Refactored to include varios reports and checks
diff --git a/check_versions/common/exception.py b/check_versions/common/exception.py
deleted file mode 100644
index 93edfff..0000000
--- a/check_versions/common/exception.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from exceptions import Exception
-
-
-class Cee8BaseExceptions(Exception):
- pass
-
-
-class Cee8Exception(Cee8BaseExceptions):
- def __init__(self, message, *args, **kwargs):
- super(Cee8Exception, self).__init__(message, *args, **kwargs)
- # get the trace
- # TODO: get and log traceback
-
- # prettify message
- self.message = "CEE8Exception: {}".format(message)
-
-
-class ConfigException(Cee8Exception):
- def __init__(self, message, *args, **kwargs):
- super(ConfigException, self).__init__(message, *args, **kwargs)
- self.message = "Configuration error: {}".format(message)
diff --git a/check_versions/__init__.py b/ci_checker/__init__.py
similarity index 100%
rename from check_versions/__init__.py
rename to ci_checker/__init__.py
diff --git a/check_versions/common/__init__.py b/ci_checker/common/__init__.py
similarity index 100%
rename from check_versions/common/__init__.py
rename to ci_checker/common/__init__.py
diff --git a/check_versions/common/base_settings.py b/ci_checker/common/base_settings.py
similarity index 93%
rename from check_versions/common/base_settings.py
rename to ci_checker/common/base_settings.py
index 8e10ea6..f0a8f2b 100644
--- a/check_versions/common/base_settings.py
+++ b/ci_checker/common/base_settings.py
@@ -8,7 +8,7 @@
import os
-from check_versions.common.other import utils
+from ci_checker.common.other import utils
PKG_DIR = os.path.dirname(__file__)
PKG_DIR = os.path.join(PKG_DIR, os.pardir, os.pardir)
@@ -23,7 +23,7 @@
"""
name = "CiTestsBaseConfig"
- logfile_name = 'ci_packages.log'
+ logfile_name = 'ci_checker.log'
working_folder = os.environ.get('CI_TESTS_WORK_DIR', _default_work_folder)
salt_host = os.environ.get('SALT_URL', None)
salt_port = os.environ.get('SALT_PORT', '6969')
diff --git a/check_versions/common/const.py b/ci_checker/common/const.py
similarity index 100%
rename from check_versions/common/const.py
rename to ci_checker/common/const.py
diff --git a/ci_checker/common/exception.py b/ci_checker/common/exception.py
new file mode 100644
index 0000000..82e2432
--- /dev/null
+++ b/ci_checker/common/exception.py
@@ -0,0 +1,21 @@
+from exceptions import Exception
+
+
+class CiCheckerBaseExceptions(Exception):
+ pass
+
+
+class CheckerException(CiCheckerBaseExceptions):
+ def __init__(self, message, *args, **kwargs):
+ super(CheckerException, self).__init__(message, *args, **kwargs)
+ # get the trace
+ # TODO: get and log traceback
+
+ # prettify message
+ self.message = "CheckerException: {}".format(message)
+
+
+class ConfigException(CheckerException):
+ def __init__(self, message, *args, **kwargs):
+ super(ConfigException, self).__init__(message, *args, **kwargs)
+ self.message = "Configuration error: {}".format(message)
diff --git a/check_versions/common/log.py b/ci_checker/common/log.py
similarity index 100%
rename from check_versions/common/log.py
rename to ci_checker/common/log.py
diff --git a/check_versions/common/other.py b/ci_checker/common/other.py
similarity index 94%
rename from check_versions/common/other.py
rename to ci_checker/common/other.py
index 00f67c2..14003ea 100644
--- a/check_versions/common/other.py
+++ b/ci_checker/common/other.py
@@ -1,9 +1,9 @@
import os
import re
-from check_versions.common.const import all_roles_map
+from ci_checker.common.const import all_roles_map
-from check_versions.common.exception import ConfigException
+from ci_checker.common.exception import ConfigException
PKG_DIR = os.path.dirname(__file__)
PKG_DIR = os.path.join(PKG_DIR, os.pardir, os.pardir)
@@ -72,6 +72,8 @@
if env is None:
# nothing supplied, use the one in repo
try:
+ if not nodes_list:
+ return []
with open(os.path.join(PKG_DIR, nodes_list)) as _f:
_list.extend(_f.read().splitlines())
except IOError as e:
diff --git a/check_versions/common/salt_utils.py b/ci_checker/common/salt_utils.py
similarity index 98%
rename from check_versions/common/salt_utils.py
rename to ci_checker/common/salt_utils.py
index 363e621..cbf0321 100644
--- a/check_versions/common/salt_utils.py
+++ b/ci_checker/common/salt_utils.py
@@ -5,8 +5,8 @@
import requests
import time
-from check_versions.common.base_settings import base_config as config
-from check_versions.common import logger
+from ci_checker.common.base_settings import base_config as config
+from ci_checker.common import logger
def list_to_target_string(node_list, separator):
diff --git a/check_versions/pkg_check.py b/ci_checker/pkg_check.py
similarity index 94%
rename from check_versions/pkg_check.py
rename to ci_checker/pkg_check.py
index 16fa090..b9d9ed3 100644
--- a/check_versions/pkg_check.py
+++ b/ci_checker/pkg_check.py
@@ -5,10 +5,10 @@
from copy import deepcopy
import common.const as const
-import pkg_reporter
-from check_versions.common import utils
-from check_versions.common import base_config, logger, logger_cli, PKG_DIR
-from check_versions.common import salt_utils
+import reporter
+from ci_checker.common import utils
+from ci_checker.common import base_config, logger, logger_cli, PKG_DIR
+from ci_checker.common import salt_utils
node_tmpl = {
'role': '',
@@ -163,11 +163,11 @@
:return: buff with html
"""
- _report = pkg_reporter.ReportToFile(
- pkg_reporter.HTMLPackageCandidates(),
+ _report = reporter.ReportToFile(
+ reporter.HTMLPackageCandidates(),
filename
)
- _report(self.nodes)
+ _report(nodes=self.nodes)
# init connection to salt and collect minion data
diff --git a/ci_checker/reclass_cmp.py b/ci_checker/reclass_cmp.py
new file mode 100644
index 0000000..d50064c
--- /dev/null
+++ b/ci_checker/reclass_cmp.py
@@ -0,0 +1,191 @@
+"""Model Comparer:
+- yaml parser
+- class tree comparison
+"""
+import itertools
+# import json
+import os
+import yaml
+
+import reporter
+from ci_checker.common import utils
+from ci_checker.common import base_config, logger, logger_cli, PKG_DIR
+
+
+class ModelComparer(object):
+ """Collection of functions to compare model data.
+ """
+ models = {}
+
+ @staticmethod
+ def load_yaml_class(fname):
+ """Loads a yaml from the file and forms a tree item
+
+ Arguments:
+ fname {string} -- full path to the yaml file
+ """
+ _yaml = {}
+ try:
+ _size = 0
+ with open(fname, 'r') as f:
+ _yaml = yaml.load(f)
+ _size = f.tell()
+ # TODO: do smth with the data
+ if not _yaml:
+ logger_cli.warning("WARN: empty file '{}'".format(fname))
+ _yaml = {}
+ else:
+ logger.debug("...loaded YAML '{}' ({}b)".format(fname, _size))
+ return _yaml
+ except yaml.YAMLError as exc:
+ logger_cli.error(exc)
+ except IOError as e:
+ logger_cli.error(
+ "Error loading file '{}': {}".format(fname, e.message)
+ )
+ raise Exception("CRITICAL: Failed to load YAML data: {}".format(
+ e.message
+ ))
+
+ def load_model_tree(self, name, root_path="/srv/salt/reclass"):
+ """Walks supplied path for the YAML filed and loads the tree
+
+ Arguments:
+ root_folder_path {string} -- Path to Model's root folder. Optional
+ """
+ logger_cli.info("Loading reclass tree from '{}'".format(root_path))
+ # prepare the file tree to walk
+ raw_tree = {}
+ # Credits to Andrew Clark@MIT. Original code is here:
+ # http://code.activestate.com/recipes/577879-create-a-nested-dictionary-from-oswalk/
+ root_path = root_path.rstrip(os.sep)
+ start = root_path.rfind(os.sep) + 1
+ root_key = root_path.rsplit(os.sep, 1)[1]
+ # Look Ma! I am walking the file tree with no recursion!
+ for path, dirs, files in os.walk(root_path):
+ # if this is a hidden folder, ignore it
+ _filders_list = path[start:].split(os.sep)
+ if any(item.startswith(".") for item in _filders_list):
+ continue
+ # cut absolute part of the path and split folder names
+ folders = path[start:].split(os.sep)
+ subdir = {}
+ # create generator of files that are not hidden
+ _subfiles = (file for file in files if not file.startswith("."))
+ for _file in _subfiles:
+ # cut file extension. All reclass files are '.yml'
+ _subnode = _file
+ # load all YAML class data into the tree
+ subdir[_subnode] = self.load_yaml_class(
+ os.path.join(path, _file)
+ )
+ # Save original filepath, just in case
+ subdir[_subnode]["_source"] = os.path.join(path[start:], _file)
+ # creating dict structure out of folder list. Pure python magic
+ parent = reduce(dict.get, folders[:-1], raw_tree)
+ parent[folders[-1]] = subdir
+ # save it as a single data object
+ self.models[name] = raw_tree[root_key]
+ return True
+
+ def generate_model_report_tree(self):
+ """Use all loaded models to generate comparison table with
+ values are groupped by YAML files
+ """
+ def find_changes(dict1, dict2, path=""):
+ _report = {}
+ for k in dict1.keys():
+ _new_path = path + ":" + k
+ if k == "_source":
+ continue
+ if k not in dict2:
+ # no key in dict2
+ _report[_new_path] = [dict1[k], "N/A"]
+ logger_cli.info(
+ "{}: {}, {}".format(_new_path, dict1[k], "N/A")
+ )
+ else:
+ if type(dict1[k]) is dict:
+ if path == "":
+ _new_path = k
+ _child_report = find_changes(
+ dict1[k],
+ dict2[k],
+ _new_path
+ )
+ _report.update(_child_report)
+ elif type(dict1[k]) is list:
+ # use ifilterfalse to compare lists of dicts
+ _removed = list(
+ itertools.ifilterfalse(
+ lambda x: x in dict2[k],
+ dict1[k]
+ )
+ )
+ _added = list(
+ itertools.ifilterfalse(
+ lambda x: x in dict1[k],
+ dict2[k]
+ )
+ )
+ if _removed or _added:
+ _removed_str_lst = ["- {}".format(item)
+ for item in _removed]
+ _added_str_lst = ["+ {}".format(item)
+ for item in _added]
+ _report[_new_path] = [
+ dict1[k],
+ _removed_str_lst + _added_str_lst
+ ]
+ logger_cli.info(
+ "{}:\n"
+ "{} original items total".format(
+ _new_path,
+ len(dict1[k])
+ )
+ )
+ if _removed:
+ logger_cli.info(
+ "{}".format('\n'.join(_removed_str_lst))
+ )
+ if _added:
+ logger_cli.info(
+ "{}".format('\n'.join(_added_str_lst))
+ )
+ else:
+ if dict1[k] != dict2[k]:
+ _report[_new_path] = [dict1[k], dict2[k]]
+ logger_cli.info("{}: {}, {}".format(
+ _new_path,
+ dict1[k],
+ dict2[k]
+ ))
+ return _report
+ # tmp report for keys
+ diff_report = find_changes(
+ self.models["inspur_Aug"],
+ self.models['inspur_Dec']
+ )
+ return diff_report
+
+
+# temporary executing the parser as a main prog
+if __name__ == '__main__':
+ mComparer = ModelComparer()
+ mComparer.load_model_tree(
+ 'inspur_Aug',
+ '/Users/savex/proj/inspur_hc/reclass_cmp/reclass-20180810'
+ )
+ mComparer.load_model_tree(
+ 'inspur_Dec',
+ '/Users/savex/proj/inspur_hc/reclass_cmp/reclass-20181210'
+ )
+ diffs = mComparer.generate_model_report_tree()
+
+ report = reporter.ReportToFile(
+ reporter.HTMLModelCompare(),
+ './mdl_diff.html'
+ )
+ report(mdl_diff=diffs)
+ # with open("./gen_tree.json", "w+") as _out:
+ # _out.write(json.dumps(mComparer.generate_model_report_tree))
diff --git a/check_versions/pkg_reporter.py b/ci_checker/reporter.py
similarity index 91%
rename from check_versions/pkg_reporter.py
rename to ci_checker/reporter.py
index 8b14a8a..326969e 100644
--- a/check_versions/pkg_reporter.py
+++ b/ci_checker/reporter.py
@@ -3,7 +3,7 @@
import abc
import os
-from check_versions.common import const
+from ci_checker.common import const
pkg_dir = os.path.dirname(__file__)
pkg_dir = os.path.join(pkg_dir, os.pardir)
@@ -56,11 +56,12 @@
def _count_totals(data):
data['counters']['total_nodes'] = len(data['nodes'])
- def __call__(self, nodes):
+ def __call__(self, nodes={}, mdl_diff={}):
# init data structures
data = self.common_data()
data.update({
- "nodes": nodes
+ "nodes": nodes,
+ "compare": data
})
# add template specific data
@@ -143,6 +144,19 @@
data['counters']['total_packages'] = _all_pkg
+# Package versions report
+class HTMLModelCompare(_TMPLBase):
+ tmpl = "model_tree_cmp_tmpl.j2"
+
+ def _extend_data(self, data):
+ # extend data with the compare dict
+
+ # counters - mdl_diff
+ data['counters']['mdl_diff'] = 51
+
+ pass
+
+
class ReportToFile(object):
def __init__(self, report, target):
self.report = report
diff --git a/etc/example._env b/etc/example._env
index f22f18f..477296c 100644
--- a/etc/example._env
+++ b/etc/example._env
@@ -15,7 +15,7 @@
SALT_URL=127.0.0.1
# Salt master port.
-# Note that you can safely execute second master
+# Note that you can safely execute second master (not on production)
# on the same cfg node with diferent port and set logging to ALL
SALT_PORT=16969
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..eaae064
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+six
+pyyaml
+jinja2
\ No newline at end of file
diff --git a/scripts/ifs_data.py b/scripts/ifs_data.py
index 1b6143f..f49bd67 100644
--- a/scripts/ifs_data.py
+++ b/scripts/ifs_data.py
@@ -69,7 +69,7 @@
# _ifs = sorted(ifs_data.keys())
# _ifs.remove("lo")
# for _idx in range(len(_ifs)):
-# print("{}: {}, {}, {}".format(
+# print("\t{}:\t{},\t\t{},\t{}".format(
# _ifs[_idx],
# " ".join(ifs_data[_ifs[_idx]]['ipv4'].keys()),
# ifs_data[_ifs[_idx]]['mtu'],
diff --git a/scripts/net_checks.py b/scripts/net_checks.py
new file mode 100644
index 0000000..dab896e
--- /dev/null
+++ b/scripts/net_checks.py
@@ -0,0 +1,57 @@
+# From dstremkovsky
+"""```root@cfg01:/srv/salt/reclass# salt kvm01* net_checks.get_nics
+kvm01.multinode-ha.int:
+ |_
+ - bond0
+ - None
+ - 00:25:90:e7:46:d0
+ - 1
+ - 1500
+ |_
+ - bond0.1306
+ - None
+ - 00:25:90:e7:46:d0
+ - 1
+ - 1500
+ |_
+ - enp2s0f0
+ - None
+ - 00:25:90:e7:46:d0
+ - 1
+ - 1500
+ |_
+ - enp2s0f1
+ - None
+ - 00:25:90:e7:46:d1
+ - 0
+ - 1500```
+
+```Generate csv report for the env
+
+.. code-block:: bash
+
+ salt -C 'kvm* or cmp* or osd*' net_checks.get_nics_csv \
+ | grep '^\ ' | sed 's/\ *//g' | grep -Ev ^server \
+ | sed '1 i\server,nic_name,ip_addr,mac_addr,link,mtu,chassis_id,chassis_name,port_mac,port_descr'
+
+**Example of system output:**
+
+.. code-block:: bash
+
+ server,nic_name,ip_addr,mac_addr,link,mtu,chassis_id,chassis_name,port_mac,port_descr
+ cmp010.domain.com,bond0,None,b4:96:91:10:5b:3a,1,1500,,,,
+ cmp010.domain.com,bond0.21,10.200.178.110,b4:96:91:10:5b:3a,1,1500,,,,
+ cmp010.domain.com,bond0.22,10.200.179.110,b4:96:91:10:5b:3a,1,1500,,,,
+ cmp010.domain.com,bond1,None,3c:fd:fe:34:ad:22,0,1500,,,,
+ cmp010.domain.com,bond1.24,10.200.181.110,3c:fd:fe:34:ad:22,0,1500,,,,
+ cmp010.domain.com,fourty5,None,3c:fd:fe:34:ad:20,0,9000,,,,
+ cmp010.domain.com,fourty6,None,3c:fd:fe:34:ad:22,0,9000,,,,
+ cmp010.domain.com,one1,None,b4:96:91:10:5b:38,0,1500,,,,
+ cmp010.domain.com,one2,None,b4:96:91:10:5b:39,1,1500,f0:4b:3a:8f:75:40,exnfvaa18-20,548,ge-0/0/22
+ cmp010.domain.com,one3,None,b4:96:91:10:5b:3a,1,1500,f0:4b:3a:8f:75:40,exnfvaa18-20,547,ge-0/0/21
+ cmp010.domain.com,one4,10.200.177.110,b4:96:91:10:5b:3b,1,1500,f0:4b:3a:8f:75:40,exnfvaa18-20,546,ge-0/0/20
+ cmp011.domain.com,bond0,None,b4:96:91:13:6c:aa,1,1500,,,,
+ cmp011.domain.com,bond0.21,10.200.178.111,b4:96:91:13:6c:aa,1,1500,,,,
+ cmp011.domain.com,bond0.22,10.200.179.111,b4:96:91:13:6c:aa,1,1500,,,,
+ ...```
+"""
diff --git a/templates/model_tree_cmp_tmpl.j2 b/templates/model_tree_cmp_tmpl.j2
new file mode 100644
index 0000000..f7633de
--- /dev/null
+++ b/templates/model_tree_cmp_tmpl.j2
@@ -0,0 +1,215 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <meta charset="UTF-8">
+ <title>Model Tree Changes</title>
+ <style>
+ body {
+ font-family: Verdana, Geneva, Tahoma, sans-serif;
+ font-size: 90% !important;
+ }
+ .dot_green {
+ float: left;
+ color: green;
+ margin-right: 0.5em;
+ margin-top: 0.2em;
+ }
+ .dot_red {
+ float: left;
+ color: red;
+ margin-right: 0.5em;
+ margin-top: 0.2em;
+ }
+ .dot_empty {
+ float: left;
+ color: darkgray;
+ margin-right: 0.5em;
+ margin-top: 0.2em;
+ }
+ /* Style the tab */
+ .tab {
+ float: left;
+ width: 130px;
+ border: 1px solid #fff;
+ background-color: #efe;
+ }
+
+ /* Style the buttons that are used to open the tab content */
+ .tab button {
+ display: block;
+ background-color: inherit;
+ color: Black;
+ border: none;
+ outline: none;
+ font-family: "Lucida Console", Monaco, monospace;
+ text-align: left;
+ cursor: pointer;
+ transition: 0.3s;
+ font-size: 1.3em;
+ width: 100%;
+ padding: 1px;
+ margin: 1px;
+ }
+
+ button > div.node_name {
+ float: left;
+ font-size: 1.3em;
+ }
+
+ .smallgreytext {
+ float: right;
+ font-size: 0.7em;
+ color: gray;
+ }
+
+ /* Change background color of buttons on hover */
+ .tab button:hover {
+ background-color: #7b7;
+ }
+
+ /* Create an active/current "tab button" class */
+ .tab button.active {
+ background-color: #8c8;
+ color: white;
+ }
+
+ /* Style the tab content */
+ .tabcontent {
+ display: none;
+ position: absolute;
+ font-size: 1em;
+ padding: 0.5em;
+ right: -10%;
+ top: 0%;
+ transform: translateX(-12%);
+ width: calc(100% - 170px);
+ overflow-x: scroll;
+ overflow-wrap: break-word;
+ }
+
+ table {
+ border: 0 hidden;
+ width: 100%;
+ }
+ tr:nth-child(even) {
+ background-color: #fff;
+ }
+ tr:nth-child(odd) {
+ background-color: #ddd;
+ }
+ .Header {
+ background-color: #bbb;
+ color: Black;
+ width: 30%;
+ text-align: center;
+ }
+ .pkgName {
+ font-size: 1em;
+ padding-left: 10px;
+ }
+
+ .version {
+ font-size: 1.1em;
+ text-align: left;
+ }
+
+ .differ {
+ background-color: #eaa;
+ }
+ /* Tooltip container */
+ .tooltip {
+ position: relative;
+ display: inline-block;
+ border-bottom: 1px dotted black;
+ }
+
+ .tooltip .tooltiptext {
+ visibility: hidden;
+ background-color: black;
+ font-family: "Lucida Console", Monaco, monospace;
+ font-size: 0.5em;
+ width: auto;
+ color: #fff;
+ border-radius: 6px;
+ padding: 5px 5px;
+
+ /* Position the tooltip */
+ position: absolute;
+ z-index: 1;
+ }
+
+ .tooltip:hover .tooltiptext {
+ visibility: visible;
+ }
+
+ </style>
+ <script language="JavaScript">
+ function init() {
+ // Declare all variables
+ var i, tabcontent, tablinks;
+
+ // Get all elements with class="tabcontent" and hide them
+ tabcontent = document.getElementsByClassName("tabcontent");
+ for (i = 1; i < tabcontent.length; i++) {
+ tabcontent[i].style.display = "none";
+ }
+ tabcontent[0].style.display = "block";
+
+ // Get all elements with class="tablinks" and remove the class "active"
+ tablinks = document.getElementsByClassName("tablinks");
+ for (i = 1; i < tablinks.length; i++) {
+ tablinks[i].className = tablinks[i].className.replace(" active", "");
+ }
+ tablinks[0].className += " active";
+
+ }
+ function openTab(evt, tabName) {
+ // Declare all variables
+ var i, tabcontent, tablinks;
+
+ // Get all elements with class="tabcontent" and hide them
+ tabcontent = document.getElementsByClassName("tabcontent");
+ for (i = 0; i < tabcontent.length; i++) {
+ tabcontent[i].style.display = "none";
+ }
+
+ // Get all elements with class="tablinks" and remove the class "active"
+ tablinks = document.getElementsByClassName("tablinks");
+ for (i = 0; i < tablinks.length; i++) {
+ tablinks[i].className = tablinks[i].className.replace(" active", "");
+ }
+
+ // Show the current tab, and add an "active" class to the link that opened the tab
+ document.getElementById(tabName).style.display = "block";
+ evt.currentTarget.className += " active";
+ }
+ </script>
+</head>
+<body onload="init()">
+<div class="tab">
+ <button class="tablinks" onclick="openTab(event, 'mdl_changes')">
+ <div class="node_name">Model changes</div>
+ <div class="smallgreytext">({{ counters['mdl_diff'] }})</div>
+ </button>
+</div>
+<div id="mdl_changes" class="tabcontent">
+ <table class="pkgversions">
+ <tbody>
+ <tr>
+ <td class="Header">Package name</td>
+ <td class="Header">Installed</td>
+ <td class="Header">Candidate</td>
+ </tr>
+ <tr><td colspan=3>Package with different versions uniq for this node</td></tr>
+ <td class="pkgName">class:cluster:param1</td>
+ <td class="version differ">
+ <div class="tooltip">Value1
+ <pre class="tooltiptext">raw text from the diff | linebreaks }}</pre>
+ </div>
+ </td>
+ <td class="version">Value2</td>
+ </tbody>
+ </table>
+</div>
+</body>
+</html>
\ No newline at end of file