Merge "Imlement result searching based on type tests and by test run id"
diff --git a/log_helper/README.md b/log_helper/README.md
new file mode 100644
index 0000000..a07c948
--- /dev/null
+++ b/log_helper/README.md
@@ -0,0 +1,74 @@
+CLI helper tool
+======================
+
+This CLI helper tool gathers all log lines related to resource | req-id from particular folder with logs.
+All related resources and their IDs used in Tempest tests are stored in machine-readable YAML file with the following
+format:
+
+```
+<test identifier>:
+ status: PASSED|FAILED
+ resources:
+ <openstack-service name (nova|neutron|glance|keystone)>:
+ <resource-name (port|server|security-group|router)>:
+ <request-id (req-xxxx) >:
+ name: <resource name (test-port-mytest|test-vm-)>
+ id/uuid: <resource id>
+ request: <exact request>
+ http:
+ error: <if exists>
+ status_code: <>
+```
+
+This machine-readable YAML file can be generated using _report_parcer_ tool:
+`https://gerrit.mcp.mirantis.com/plugins/gitiles/mcp/osccore-qa-testing-tools/+/refs/heads/master/tempest_tests_resources`
+
+
+## Preparing the environment
+
+1. Clone the repository `log_helper`:
+* Clone with SSH
+ ~~~~
+ git clone ssh://gerrit.mcp.mirantis.com:29418/mcp/osccore-qa-testing-tools
+ ~~~~
+
+2. Navigate to the `log_helper/` folder:
+ ~~~
+ cd log_helper/
+ ~~~
+
+3. Edit 'config.py' file and provide it with required values:
+
+ * `RESULTS_DIR`, `LOG_DIR` and `TEMPEST_REPORT_YAML`
+
+
+4. In the `log_helper/` folder install log_helper tool:
+ ~~~~
+ python3 -m pip install -e .
+ ~~~~
+
+
+How to use
+----------
+
+- Extract/save all openstack logs to folder (ex. pod-logs)
+
+- Generate a tempest report using the report_parcer tool with the format as descideb above
+
+- Run log_helper tool using two options:
+
+1. with no parameters (LOG_DIR and TEMPEST_REPORT_YAML file as config parameters):
+
+``log_helper.py``
+
+2. with resource-id and log-level (optional) as input parameters (LOG_DIR as config parameter):
+
+``log_helper.py <resource-id>``
+
+ or
+
+``log_helper.py <resource-id> <log-level>``
+
+Example:
+
+``log_helper.py req-aabbb391-ff60-4893-9b20-60f6a29749c9 ERROR``
diff --git a/log_helper/__init__.py b/log_helper/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/log_helper/__init__.py
diff --git a/log_helper/config.py b/log_helper/config.py
new file mode 100644
index 0000000..ed2dd8a
--- /dev/null
+++ b/log_helper/config.py
@@ -0,0 +1,8 @@
+# Path to directory with logs (e.x. pod-logs)
+LOG_DIR = '/home/roman/Downloads/Test_logs/pod-logs'
+
+# Path to machine-readable YAML file, generated by report_parcer tool
+TEMPEST_REPORT_YAML = '/home/roman/Downloads/Test_logs/tempest_new.yaml'
+
+# Path to directory with results of log_helper execution
+RESULTS_DIR = '/home/roman/Downloads/Test_logs/log_helper_result'
diff --git a/log_helper/log_helper.py b/log_helper/log_helper.py
new file mode 100755
index 0000000..48190c2
--- /dev/null
+++ b/log_helper/log_helper.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+import subprocess
+import sys
+import os
+import yaml
+from os import path
+import config
+
+
+param_is_yaml = False
+
+if len(sys.argv) == 1:
+ param_is_yaml = path.isfile(config.TEMPEST_REPORT_YAML)
+ if param_is_yaml is False:
+ print('TEMPEST_REPORT_YAML config parameter is not a file')
+ raise Exception('TEMPEST_REPORT_YAML config parameter is not a file')
+
+
+def log_gather(resource_id, sub_resource, log_level=None):
+ """ Get all log lines related to resource-id
+ :param resource_id: ID resource, e.g. request-id, server-id
+ :param sub_resource: name of sub_resource log file, e.g subnet.log for neutron resource
+ :param log_level: substring for resource_id log: e.g. get only ERROR log level messages, optional
+ """
+ try:
+ directory = os.walk(config.LOG_DIR)
+ except IndexError:
+ print('Parameter <LOG_DIR> is not provided')
+ raise ValueError('Parameter <LOG_DIR> is not provided')
+
+ if param_is_yaml:
+ for dirs in directory:
+ run_cmd = f"grep -a {resource_id} {dirs[0]}/* >> {config.RESULTS_DIR}/{sub_resource}"
+ subprocess.run(run_cmd, shell=True)
+
+ else:
+ for dirs in directory:
+ if log_level:
+ run_cmd = f"grep -lE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {dirs[0]}/* >> '{config.RESULTS_DIR}/tmp.log'"
+ else:
+ run_cmd = f"grep -l {resource_id} {dirs[0]}/* >> '{config.RESULTS_DIR}/tmp.log'"
+ subprocess.run(run_cmd, shell=True)
+
+ with open(config.RESULTS_DIR + '/tmp.log') as f:
+ files = f.readlines()
+
+ for file in files:
+ subd = file.split("/")
+ log_dir = subd[-4] + "." + subd[-3] + "." + subd[-2]
+ log_name = subd[-1].replace('\n', '')
+ os.makedirs(os.path.join(config.RESULTS_DIR, sys.argv[1], log_dir), exist_ok=True)
+ path = os.path.join(config.RESULTS_DIR, sys.argv[1], log_dir, log_name)
+ if log_level:
+ run_cmd = f"grep -aE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {file} >> {path}"
+ else:
+ run_cmd = f"grep -a {resource_id} {file} >> {path}"
+ subprocess.run(run_cmd.replace('\n', ''), shell=True)
+
+ os.remove(config.RESULTS_DIR + '/tmp.log')
+
+
+if param_is_yaml:
+ print('Find all the failed tempest tests from YAML file')
+ with open(config.TEMPEST_REPORT_YAML) as f:
+ test_resources = yaml.safe_load(f)
+
+ for test in test_resources.items():
+ # Find all the failed tempest tests from YAML file and gather logs for
+ # related resources in corresponded folders
+ if test[1]['status'] == 'failure':
+ print('Collecting logs for ' + test[0])
+ os.makedirs(os.path.join(config.RESULTS_DIR, test[0]), exist_ok=True)
+ for resource in test[1]['resources']:
+ os.makedirs(os.path.join(config.RESULTS_DIR, test[0], resource), exist_ok=True)
+ for sub_resource in test[1]['resources'][resource]:
+ log_gather(list(test[1]['resources'][resource][sub_resource])[0],
+ os.path.join(test[0], resource, sub_resource + '.' + 'log'))
+
+else:
+ print('Find all the related log for one specific test or id with error')
+ os.makedirs(os.path.join(config.RESULTS_DIR, sys.argv[1]), exist_ok=True)
+ if len(sys.argv) == 3:
+ log_gather(sys.argv[1], os.path.join(sys.argv[1], 'test' + '.' + 'log'), log_level=sys.argv[2])
+ else:
+ log_gather(sys.argv[1], os.path.join(sys.argv[1], 'test' + '.' + 'log'))
+
+print('The logger is finished')
diff --git a/log_helper/requirements.txt b/log_helper/requirements.txt
new file mode 100644
index 0000000..bee6c14
--- /dev/null
+++ b/log_helper/requirements.txt
@@ -0,0 +1 @@
+PyYAML==6.0
diff --git a/log_helper/setup.py b/log_helper/setup.py
new file mode 100644
index 0000000..5e73800
--- /dev/null
+++ b/log_helper/setup.py
@@ -0,0 +1,13 @@
+from distutils.core import setup
+
+setup(
+ name='log_helper',
+ version='0.2',
+ py_modules=["log_helper", "config"],
+ install_requires=['pyyaml'],
+ python_requires='>=3.6',
+ author='Roman Bubyr',
+ author_email='rbubyr@gmail.com',
+ description='Openstack log helper tool',
+ scripts=['log_helper.py'],
+)
diff --git a/tempest_tests_resources/README.md b/tempest_tests_resources/README.md
index 551104d..40e7574 100644
--- a/tempest_tests_resources/README.md
+++ b/tempest_tests_resources/README.md
@@ -2,11 +2,27 @@
======================
This tool creating machine readable YAML file with all resources used in Tempest tests
+The machine readable YAML file has the following format:
+
+ """
+ <test identifier>:
+ status: PASSED|FAILED
+ resources:
+ <openstack-service name (nova|neutron|glance|keystone)>:
+ <resource-name (port|server|security-group|router)>:
+ <request-id (req-xxxx) >:
+ name: <resource name (test-port-mytest|test-vm-)>
+ id/uuid: <resource id>
+ requst: <exact request>
+ http:
+ error: <if exists>
+ status_code: <>
+ """
How to use
----------
-Update your env variabled or add report and result file to artifacts dir
+Update your env variables or add report and result file to artifacts dir
```
export REPORT_NAME='' \
export TEMPORARY_FILE_NAME='' \
@@ -14,6 +30,29 @@
export TEMPEST_REPORT_XML=''
```
-Run report parser script:
+where REPORT_NAME is a tempest log file from your tempest run, TEMPORARY_FILE_NAME is any name of temporary file,
+RESULT_FILE_NAME is name of the machine readable YAML file and TEMPEST_REPORT_XML is tempest report XML from your
+tempest run.
+
+Run report parser script to create a Yaml report for all test cases:
``python3 report_parser.py``
+
+or to process requests from Setup/Tear Down Classes if the argument:
+
+``python3 report_parser.py class``
+
+or to create a Yaml report for one specific test case:
+
+``python3 report_parser.py <test case name>``
+
+ for example:
+
+ ``python3 report_parser.py test_show_hypervisor_with_non_admin_user``
+
+
+or to create a Yaml report for failed only test cases:
+
+``python3 report_parser.py failed``
+
+
diff --git a/tempest_tests_resources/report_parser.py b/tempest_tests_resources/report_parser.py
index d697d32..0e09f39 100644
--- a/tempest_tests_resources/report_parser.py
+++ b/tempest_tests_resources/report_parser.py
@@ -1,5 +1,6 @@
import re
import subprocess
+import sys
import json
import yaml
import os
@@ -69,6 +70,7 @@
:return:
"""
# Skip list to process requests from tests only
+ # Process requests from Setup/Tear Down Classes if the argument 'class' exist
try:
# regex for: (ClassName:test_name)
@@ -170,12 +172,22 @@
result = {}
for request in get_request_response(TEMPORARY_FILENAME):
-
# Get test name from request
- test_name = _get_test_name(request[0])
+
+ # Process requests from Setup/Tear Down Classes if the argument 'class' exist
+ if len(sys.argv) == 2 and sys.argv[1] == 'class':
+ methods_skip_list = ['_run_cleanups', ]
+ test_name = _get_test_name(request[0], methods_skip_list)
+ else:
+ test_name = _get_test_name(request[0])
+
if not test_name:
continue
+ # Generate test recourses only for specific test case if the argument 'test case name' exist
+ if len(sys.argv) == 2 and sys.argv[1] != 'class' and sys.argv[1] != 'failed' and sys.argv[1] not in test_name:
+ continue
+
if not result.get(test_name):
result[test_name] = {"status": None,
"resources": {}}
@@ -275,7 +287,7 @@
for child in root:
classname = child.attrib['classname']
name = child.attrib['name']
- if classname and name:
+ if classname or name:
short_classname = classname.split('.')[-1]
short_name = name.split('[')[0]
short_test_name = f"{short_classname}.{short_name}"
@@ -288,6 +300,17 @@
except IndexError:
test_status = 'passed'
+ # Generate test recourses only for failed test cases if the argument 'failed' exist
+ if len(sys.argv) == 2 and sys.argv[1] == 'failed':
+ if test_status != 'failure':
+ # we try to remove not 'failed' cases from report with 'partial' name of short_test_name
+ rz = result.pop(short_test_name, None)
+ if not rz:
+ for item in list(result.keys()):
+ if item in short_test_name:
+ result.pop(item)
+ continue
+
if short_test_name and short_test_name in result:
# TODO(imenkov): how to avoid issue
# we can see in report 2 tests:
@@ -296,11 +319,24 @@
#
# but they logged as one:
# RecordsetValidationTest:test_cannot_create_MX_with
+ # rbubyr: it happens because in tempest.log these both tests are
+ # logged as RecordsetValidationTest:test_cannot_create_MX_with
+ # should be fixed in tempest tests
if not result[short_test_name].get('full_test_name'):
result[short_test_name]['full_test_name'] = []
result[short_test_name]['full_test_name'].append(full_test_name)
result[short_test_name]['status'] = test_status
+ # TODO(rbubyr): some test cases are absent in result dic, these tests won't have resources mapping
+ # because they are skipped in TEMPORARY_FILE_NAME
+ # for now we just add to final report only 'failure' test cases which are absent in result dic
+ # it might be possible to add resources from tempest xml
+ elif test_status == 'failure' and len(sys.argv) == 2 and sys.argv[1] == 'failed':
+ result[short_test_name] = {}
+ result[short_test_name]['full_test_name'] = []
+ result[short_test_name]['full_test_name'].append(full_test_name)
+ result[short_test_name]['status'] = test_status
+
def delete_temporary_file(path_to_temp_file):
os.remove(path_to_temp_file)