utils/helpers.py

import csv
import datetime
import logging
import os
import time

from jinja2 import Environment, FileSystemLoader
import pandas as pd

import utils

logger = logging.getLogger(__name__)


def get_tests_configuration_data():
    """
    Gets the values from the config file and generates the dictionary with the
    human-readable configuration.
    """
    config = utils.get_configuration()
    glance_test_file_size = config.get("IMAGE_SIZE_MB", 9000)
    image = config.get('image_name', 'cvp.ubuntu.2004')
    ext_net = config.get('external_network') or ''
    utility_single_thread = "iperf3"  # not configurable, just for info
    utility_multiple_threads = str(
        config.get('multiple_threads_iperf_utility', 10))
    iperf_time = int(config.get('iperf_time', 60))

    tests_data = {
        "File size in the Glance test": "{} MB".format(glance_test_file_size),
        "Image used in the vm2vm tests": image,
        "Floating network used": ext_net,
        "Network performance tool used for "
        "single thread tests": utility_single_thread,
        "Network performance tool used for "
        "multiple threads tests": utility_multiple_threads,
        "iperf time in seconds "
        "to transmit for each test (iperf -t)": iperf_time
    }
    return tests_data


def create_test_result_table_csv_file(text_table_rows, test_name):
    """
    Gets the text-table and saves to the separate CSV file.
    """
    dir_path = 'reports'
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)
    csv_name = "{}.csv".format(test_name)
    csv_report_path = "{}/{}".format(dir_path, csv_name)
    with open(csv_report_path, 'w', newline='') as csvfile:
        writer = csv.writer(csvfile)
        writer.writerows(text_table_rows)


def read_csv_files_to_dataframes(csv_files_list):
    """
    The pandas module reads the list of the CSV files and saves them to the
    dataframe objects. Returns the dictionary with DFs, title and time for
    each result table.
    """
    df_tables = []
    for csv_file in csv_files_list:
        # Read the CSVs to the DataFrames
        df = pd.read_csv(csv_file)
        # Get the file age (to prevent the cases when there are some old CSV
        # files, the tests are executed later, got the error, but the test
        # report is generated from the old files).
        # Also, if the tests are executed one by one in different time, all
        # results are collected with the proper age of each file.
        executed_at = os.path.getctime(csv_file)
        df_tables.append({
            'df': df,  # the results table of a test case
            'title': csv_file[8:-4],  # remove "reports/", ".csv" to get names
            'executed_at': time.ctime(executed_at)  # time of test execution
        })
    return df_tables


def convert_csvs_to_single_html_report():
    """
    Generates the single HTML report from the CSV files at reports/ dir.
    Uses the template and CSS styles from the templates/.
    """
    dir_path = 'reports/'
    if not os.path.exists(dir_path):
        logger.warning("Could not generate the HTML report since there is no "
                       "{} folder.".format(dir_path))
        return
    datetime_now = datetime.datetime.now()
    timestamp = datetime_now.strftime("%d%m%Y_%H%M%S")
    report_file = "spt_report_{}.html".format(timestamp)
    csv_files = ["{}{}".format(dir_path, f)
                 for f in sorted(os.listdir(path=dir_path))
                 if f.endswith('.csv')]
    if not csv_files:
        logger.warning("Could not generate the HTML report since the CSV files"
                       " are absent in the {} folder.".format(dir_path))
        return

    env = Environment(loader=FileSystemLoader('.'))
    template = env.get_template('templates/template.j2')

    tests_config_data = get_tests_configuration_data()
    human_timestamp = datetime_now.strftime("%Y-%m-%d %H:%M:%S")
    df_tables = read_csv_files_to_dataframes(csv_files)
    html = template.render(
        tables=df_tables,
        tests_config_data=tests_config_data,
        human_timestamp=human_timestamp
    )

    with open(report_file, 'w') as f:
        f.write(html)
    logger.info("The HTML report {} is created.".format(report_file))