Update log_helper tool to accept ResourceID/requestID as input parameters
Related-prod: PRODX-30241
Change-Id: I33d1c48834087ba05f9bcdf1c818020059a12bc0
diff --git a/log_helper/README.md b/log_helper/README.md
index 8957b7f..64410e4 100644
--- a/log_helper/README.md
+++ b/log_helper/README.md
@@ -20,11 +20,24 @@
How to use
----------
-- Extract/save all openstack pod logs to folder (ex. pod-logs)
+- Extract/save all openstack logs to folder (ex. pod-logs)
-- Generate tempest report using report_parcer tool
+- Generate a tempest report using the report_parcer tool with the format as descideb above
-- Run log_helper tool with folder name and tempest report YAML file as input parameters:
+- Run log_helper tool using two options:
+
+1. with folder name and tempest report YAML file as input parameters:
``python3 log_helper.py <folder-with-logs> <tempest-report-yaml>``
+2. with folder name , resource-id and log-level (optional) as input parameters:
+
+``python3 log_helper.py <folder-with-logs> <resource-id>``
+
+ or
+
+``python3 log_helper.py <folder-with-logs> <resource-id> <log-level>``
+
+Example:
+
+``python3 log_helper.py pod-logs req-aabbb391-ff60-4893-9b20-60f6a29749c9 ERROR``
diff --git a/log_helper/log_helper.py b/log_helper/log_helper.py
index 076de3f..86a324a 100644
--- a/log_helper/log_helper.py
+++ b/log_helper/log_helper.py
@@ -2,12 +2,20 @@
import sys
import os
import yaml
+from os import path
+
+try:
+ param_is_yaml = path.isfile(sys.argv[2])
+except IndexError:
+ print('Second argument <tempest-report-yaml>/<resource-ID> is not provided')
+ raise ValueError('ASecond argument <tempest-report-yaml>/<resource-ID> is not provided')
-def log_gather(resource_id, sub_resource):
- """ Get all log lines related to req-id
- :param resource_id: ID of Request
- :param: name of sub_resource log file, e.g subnet.log for neutron resource
+def log_gather(resource_id, sub_resource, log_level=None):
+ """ Get all log lines related to resource-id with error
+ :param resource_id: ID resource, e.g. request-id, server-id
+ :param sub_resource: name of sub_resource log file, e.g subnet.log for neutron resource
+ :param log_level: substring for resource_id log: e.g. get only ERROR log level, optional
"""
try:
directory = os.walk(sys.argv[1])
@@ -15,25 +23,61 @@
print('Argument <folder-with-logs> is not provided')
raise ValueError('Argument <folder-with-logs> is not provided')
- for dirs in directory:
- run_cmd = f"grep {resource_id} {dirs[0]}/* >> {sub_resource}"
- subprocess.run(run_cmd, shell=True)
+ if param_is_yaml:
+ for dirs in directory:
+ run_cmd = f"grep -a {resource_id} {dirs[0]}/* >> {sub_resource}"
+ subprocess.run(run_cmd, shell=True)
+
+ else:
+ for dirs in directory:
+ if log_level:
+ run_cmd = f"grep -lE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {dirs[0]}/* >> 'tmp.log'"
+ else:
+ run_cmd = f"grep -l {resource_id} {dirs[0]}/* >> 'tmp.log'"
+ subprocess.run(run_cmd, shell=True)
+
+ with open('tmp.log') as f:
+ files = f.readlines()
+
+ for file in files:
+ subd = file.split("/")
+ log_dir = subd[-4] + "." + subd[-3] + "." + subd[-2]
+ log_name = subd[-1].replace('\n', '')
+ os.makedirs(os.path.join(sys.argv[2], log_dir), exist_ok=True)
+ path = os.path.join(sys.argv[2], log_dir, log_name)
+ if log_level:
+ run_cmd = f"grep -aE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {file} >> {path}"
+ else:
+ run_cmd = f"grep -a {resource_id} {file} >> {path}"
+ subprocess.run(run_cmd.replace('\n', ''), shell=True)
+
+ os.remove('tmp.log')
-with open(sys.argv[2]) as f:
- test_resources = yaml.safe_load(f)
+if param_is_yaml:
+ print('Find all the failed tempest tests from YAML file')
+ with open(sys.argv[2]) as f:
+ test_resources = yaml.safe_load(f)
-for test in test_resources.items():
- # Find all the failed tempest test from YAML file and gather logs for
- # related resources in corresponded folders
- if test[1]['status'] == 'failure':
- print('Collecting logs for ' + test[0])
- os.makedirs(test[0], exist_ok=True)
- for resource in test[1]['resources']:
- os.makedirs(os.path.join(test[0], resource), exist_ok=True)
- for sub_resource in test[1]['resources'][resource]:
- log_gather(list(test[1]['resources'][resource][sub_resource])[0],
- os.path.join(test[0], resource, sub_resource + '.' + 'log'))
+ for test in test_resources.items():
+ # Find all the failed tempest tests from YAML file and gather logs for
+ # related resources in corresponded folders
+ if test[1]['status'] == 'failure':
+ print('Collecting logs for ' + test[0])
+ os.makedirs(test[0], exist_ok=True)
+ for resource in test[1]['resources']:
+ os.makedirs(os.path.join(test[0], resource), exist_ok=True)
+ for sub_resource in test[1]['resources'][resource]:
+ log_gather(list(test[1]['resources'][resource][sub_resource])[0],
+ os.path.join(test[0], resource, sub_resource + '.' + 'log'))
+
+else:
+ print('Find all the related log for one specific test or id with error')
+ os.makedirs(sys.argv[2], exist_ok=True)
+ if len(sys.argv) == 4:
+ log_gather(sys.argv[2], os.path.join(sys.argv[2], 'test' + '.' + 'log'), log_level=sys.argv[3])
+ else:
+ log_gather(sys.argv[2], os.path.join(sys.argv[2], 'test' + '.' + 'log'))
print('The logger is finished')