log_helper/log_helper.py

import subprocess
import sys
import os
import yaml
from os import path

try:
    param_is_yaml = path.isfile(sys.argv[2])
except IndexError:
    print('Second argument <tempest-report-yaml>/<resource-ID> is not provided')
    raise ValueError('ASecond argument <tempest-report-yaml>/<resource-ID> is not provided')


def log_gather(resource_id, sub_resource, log_level=None):
    """ Get all log lines related to resource-id with error
       :param resource_id: ID resource, e.g. request-id, server-id
       :param sub_resource: name of sub_resource log file, e.g subnet.log for neutron resource
       :param log_level: substring for resource_id log: e.g. get only ERROR log level, optional
       """
    try:
        directory = os.walk(sys.argv[1])
    except IndexError:
        print('Argument <folder-with-logs> is not provided')
        raise ValueError('Argument <folder-with-logs> is not provided')

    if param_is_yaml:
        for dirs in directory:
            run_cmd = f"grep -a {resource_id} {dirs[0]}/* >> {sub_resource}"
            subprocess.run(run_cmd, shell=True)

    else:
        for dirs in directory:
            if log_level:
                run_cmd = f"grep -lE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {dirs[0]}/* >> 'tmp.log'"
            else:
                run_cmd = f"grep -l {resource_id} {dirs[0]}/* >> 'tmp.log'"
            subprocess.run(run_cmd, shell=True)

        with open('tmp.log') as f:
            files = f.readlines()

        for file in files:
            subd = file.split("/")
            log_dir = subd[-4] + "." + subd[-3] + "." + subd[-2]
            log_name = subd[-1].replace('\n', '')
            os.makedirs(os.path.join(sys.argv[2], log_dir), exist_ok=True)
            path = os.path.join(sys.argv[2], log_dir, log_name)
            if log_level:
                run_cmd = f"grep -aE '{resource_id}.*{log_level}|{log_level}.*{resource_id}' {file} >> {path}"
            else:
                run_cmd = f"grep -a {resource_id} {file} >> {path}"
            subprocess.run(run_cmd.replace('\n', ''), shell=True)

        os.remove('tmp.log')


if param_is_yaml:
    print('Find all the failed tempest tests from YAML file')
    with open(sys.argv[2]) as f:
        test_resources = yaml.safe_load(f)

    for test in test_resources.items():
        # Find all the failed tempest tests from YAML file and gather logs for
        # related resources in corresponded folders
        if test[1]['status'] == 'failure':
            print('Collecting logs for ' + test[0])
            os.makedirs(test[0], exist_ok=True)
            for resource in test[1]['resources']:
                os.makedirs(os.path.join(test[0], resource), exist_ok=True)
                for sub_resource in test[1]['resources'][resource]:
                    log_gather(list(test[1]['resources'][resource][sub_resource])[0],
                               os.path.join(test[0], resource, sub_resource + '.' + 'log'))

else:
    print('Find all the related log for one specific test or id with error')
    os.makedirs(sys.argv[2], exist_ok=True)
    if len(sys.argv) == 4:
        log_gather(sys.argv[2], os.path.join(sys.argv[2], 'test' + '.' + 'log'), log_level=sys.argv[3])
    else:
        log_gather(sys.argv[2], os.path.join(sys.argv[2], 'test' + '.' + 'log'))

print('The logger is finished')