log_helper/log_helper.py

import subprocess
import sys
import os
import yaml


def log_gather(resource_id, sub_resource):
    """ Get all log lines related to req-id
       :param resource_id: ID of Request
       :param: name of sub_resource log file, e.g subnet.log for neutron resource
       """
    try:
        directory = os.walk(sys.argv[1])
    except IndexError:
        print('Argument <folder-with-logs> is not provided')
        raise ValueError('Argument <folder-with-logs> is not provided')

    for dirs in directory:
        run_cmd = f"grep {resource_id} {dirs[0]}/* >> {sub_resource}"
        subprocess.run(run_cmd, shell=True)


with open(sys.argv[2]) as f:
    test_resources = yaml.safe_load(f)

for test in test_resources.items():
    # Find all the failed tempest test from YAML file and gather logs for
    # related resources in corresponded folders
    if test[1]['status'] == 'failure':
        print('Collecting logs for ' + test[0])
        os.makedirs(test[0], exist_ok=True)
        for resource in test[1]['resources']:
            os.makedirs(os.path.join(test[0], resource), exist_ok=True)
            for sub_resource in test[1]['resources'][resource]:
                log_gather(list(test[1]['resources'][resource][sub_resource])[0],
                           os.path.join(test[0], resource, sub_resource + '.' + 'log'))

print('The logger is finished')