log_helper improvements.
- Move some arguments to config file
- add possibility to install tool from setup.py file
- run the tool as script (without python statement)
Related-prod: PRODX-32778
Change-Id: Ibc1ef0a75f8ea04b017e1362f24b5b06b78a09eb
diff --git a/log_helper/README.md b/log_helper/README.md
index 64410e4..a07c948 100644
--- a/log_helper/README.md
+++ b/log_helper/README.md
@@ -2,8 +2,10 @@
======================
This CLI helper tool gathers all log lines related to resource | req-id from particular folder with logs.
-All related resources and their IDs used in Tempest tests are stored in machine-readable YAML file with the following format:
+All related resources and their IDs used in Tempest tests are stored in machine-readable YAML file with the following
+format:
+```
<test identifier>:
status: PASSED|FAILED
resources:
@@ -16,6 +18,35 @@
http:
error: <if exists>
status_code: <>
+```
+
+This machine-readable YAML file can be generated using _report_parcer_ tool:
+`https://gerrit.mcp.mirantis.com/plugins/gitiles/mcp/osccore-qa-testing-tools/+/refs/heads/master/tempest_tests_resources`
+
+
+## Preparing the environment
+
+1. Clone the repository `log_helper`:
+* Clone with SSH
+ ~~~~
+ git clone ssh://gerrit.mcp.mirantis.com:29418/mcp/osccore-qa-testing-tools
+ ~~~~
+
+2. Navigate to the `log_helper/` folder:
+ ~~~
+ cd log_helper/
+ ~~~
+
+3. Edit 'config.py' file and provide it with required values:
+
+ * `RESULTS_DIR`, `LOG_DIR` and `TEMPEST_REPORT_YAML`
+
+
+4. In the `log_helper/` folder install log_helper tool:
+ ~~~~
+ python3 -m pip install -e .
+ ~~~~
+
How to use
----------
@@ -26,18 +57,18 @@
- Run log_helper tool using two options:
-1. with folder name and tempest report YAML file as input parameters:
+1. with no parameters (LOG_DIR and TEMPEST_REPORT_YAML file as config parameters):
-``python3 log_helper.py <folder-with-logs> <tempest-report-yaml>``
+``log_helper.py``
-2. with folder name , resource-id and log-level (optional) as input parameters:
+2. with resource-id and log-level (optional) as input parameters (LOG_DIR as config parameter):
-``python3 log_helper.py <folder-with-logs> <resource-id>``
+``log_helper.py <resource-id>``
or
-``python3 log_helper.py <folder-with-logs> <resource-id> <log-level>``
+``log_helper.py <resource-id> <log-level>``
Example:
-``python3 log_helper.py pod-logs req-aabbb391-ff60-4893-9b20-60f6a29749c9 ERROR``
+``log_helper.py req-aabbb391-ff60-4893-9b20-60f6a29749c9 ERROR``
diff --git a/log_helper/config.py b/log_helper/config.py
new file mode 100644
index 0000000..ed2dd8a
--- /dev/null
+++ b/log_helper/config.py
@@ -0,0 +1,8 @@
+# Path to directory with logs (e.x. pod-logs)
+LOG_DIR = '/home/roman/Downloads/Test_logs/pod-logs'
+
+# Path to machine-readable YAML file, generated by report_parcer tool
+TEMPEST_REPORT_YAML = '/home/roman/Downloads/Test_logs/tempest_new.yaml'
+
+# Path to directory with results of log_helper execution
+RESULTS_DIR = '/home/roman/Downloads/Test_logs/log_helper_result'
diff --git a/log_helper/log_helper.py b/log_helper/log_helper.py
old mode 100644
new mode 100755
index 86a324a..48190c2
--- a/log_helper/log_helper.py
+++ b/log_helper/log_helper.py
@@ -1,62 +1,68 @@
+#!/usr/bin/env python3
+
import subprocess
import sys
import os
import yaml
from os import path
+import config
-try:
- param_is_yaml = path.isfile(sys.argv[2])
-except IndexError:
- print('Second argument <tempest-report-yaml>/<resource-ID> is not provided')
- raise ValueError('ASecond argument <tempest-report-yaml>/<resource-ID> is not provided')
+
+param_is_yaml = False
+
+if len(sys.argv) == 1:
+ param_is_yaml = path.isfile(config.TEMPEST_REPORT_YAML)
+ if param_is_yaml is False:
+ print('TEMPEST_REPORT_YAML config parameter is not a file')
+ raise Exception('TEMPEST_REPORT_YAML config parameter is not a file')
def log_gather(resource_id, sub_resource, log_level=None):
- """ Get all log lines related to resource-id with error
+ """ Get all log lines related to resource-id
:param resource_id: ID resource, e.g. request-id, server-id
:param sub_resource: name of sub_resource log file, e.g subnet.log for neutron resource
- :param log_level: substring for resource_id log: e.g. get only ERROR log level, optional
+ :param log_level: substring for resource_id log: e.g. get only ERROR log level messages, optional
"""
try:
- directory = os.walk(sys.argv[1])
+ directory = os.walk(config.LOG_DIR)
except IndexError:
- print('Argument <folder-with-logs> is not provided')
- raise ValueError('Argument <folder-with-logs> is not provided')
+ print('Parameter <LOG_DIR> is not provided')
+ raise ValueError('Parameter <LOG_DIR> is not provided')
if param_is_yaml:
for dirs in directory:
- run_cmd = f"grep -a {resource_id} {dirs[0]}/* >> {sub_resource}"
+ run_cmd = f"grep -a {resource_id} {dirs[0]}/* >> {config.RESULTS_DIR}/{sub_resource}"
subprocess.run(run_cmd, shell=True)
else:
for dirs in directory:
if log_level:
- run_cmd = f"grep -lE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {dirs[0]}/* >> 'tmp.log'"
+ run_cmd = f"grep -lE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {dirs[0]}/* >> '{config.RESULTS_DIR}/tmp.log'"
else:
- run_cmd = f"grep -l {resource_id} {dirs[0]}/* >> 'tmp.log'"
+ run_cmd = f"grep -l {resource_id} {dirs[0]}/* >> '{config.RESULTS_DIR}/tmp.log'"
subprocess.run(run_cmd, shell=True)
- with open('tmp.log') as f:
+ with open(config.RESULTS_DIR + '/tmp.log') as f:
files = f.readlines()
for file in files:
subd = file.split("/")
log_dir = subd[-4] + "." + subd[-3] + "." + subd[-2]
log_name = subd[-1].replace('\n', '')
- os.makedirs(os.path.join(sys.argv[2], log_dir), exist_ok=True)
- path = os.path.join(sys.argv[2], log_dir, log_name)
+ os.makedirs(os.path.join(config.RESULTS_DIR, sys.argv[1], log_dir), exist_ok=True)
+ path = os.path.join(config.RESULTS_DIR, sys.argv[1], log_dir, log_name)
if log_level:
run_cmd = f"grep -aE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {file} >> {path}"
else:
run_cmd = f"grep -a {resource_id} {file} >> {path}"
subprocess.run(run_cmd.replace('\n', ''), shell=True)
- os.remove('tmp.log')
+ os.remove(config.RESULTS_DIR + '/tmp.log')
if param_is_yaml:
print('Find all the failed tempest tests from YAML file')
- with open(sys.argv[2]) as f:
+ with open(config.TEMPEST_REPORT_YAML) as f:
test_resources = yaml.safe_load(f)
for test in test_resources.items():
@@ -64,20 +70,19 @@
# related resources in corresponded folders
if test[1]['status'] == 'failure':
print('Collecting logs for ' + test[0])
- os.makedirs(test[0], exist_ok=True)
+ os.makedirs(os.path.join(config.RESULTS_DIR, test[0]), exist_ok=True)
for resource in test[1]['resources']:
- os.makedirs(os.path.join(test[0], resource), exist_ok=True)
+ os.makedirs(os.path.join(config.RESULTS_DIR, test[0], resource), exist_ok=True)
for sub_resource in test[1]['resources'][resource]:
log_gather(list(test[1]['resources'][resource][sub_resource])[0],
os.path.join(test[0], resource, sub_resource + '.' + 'log'))
else:
print('Find all the related log for one specific test or id with error')
- os.makedirs(sys.argv[2], exist_ok=True)
- if len(sys.argv) == 4:
- log_gather(sys.argv[2], os.path.join(sys.argv[2], 'test' + '.' + 'log'), log_level=sys.argv[3])
+ os.makedirs(os.path.join(config.RESULTS_DIR, sys.argv[1]), exist_ok=True)
+ if len(sys.argv) == 3:
+ log_gather(sys.argv[1], os.path.join(sys.argv[1], 'test' + '.' + 'log'), log_level=sys.argv[2])
else:
- log_gather(sys.argv[2], os.path.join(sys.argv[2], 'test' + '.' + 'log'))
+ log_gather(sys.argv[1], os.path.join(sys.argv[1], 'test' + '.' + 'log'))
print('The logger is finished')
-
diff --git a/log_helper/requirements.txt b/log_helper/requirements.txt
new file mode 100644
index 0000000..bee6c14
--- /dev/null
+++ b/log_helper/requirements.txt
@@ -0,0 +1 @@
+PyYAML==6.0
diff --git a/log_helper/setup.py b/log_helper/setup.py
new file mode 100644
index 0000000..5e73800
--- /dev/null
+++ b/log_helper/setup.py
@@ -0,0 +1,13 @@
+from distutils.core import setup
+
+setup(
+ name='log_helper',
+ version='0.2',
+ py_modules=["log_helper", "config"],
+ install_requires=['pyyaml'],
+ python_requires='>=3.6',
+ author='Roman Bubyr',
+ author_email='rbubyr@gmail.com',
+ description='Openstack log helper tool',
+ scripts=['log_helper.py'],
+)