Added simple HTML reporting
Simple HTML reporting for executed tests
Added the simple HTML reporting for the executed tests:
firstly the text result tables are saved to the CSV files,
and then they are converted to a single HTML report.
The generation of the report is using “pandas” and “jinja2"
modules. The CSS styles are using Mirantis style guides.
The Jijna template and the CSS styles are stored in the
‘templates/’ folder.
The ‘glance_speed_test’ output is improved and is using
a similar text table as the other tests.
At the end of each test case, the results are saved to
a new CSV file named after the test case in the ‘reports/’
folder. If the test is rerun, the new CSV is created
instead of the old one. In the end of the execution of
the tests, a single HTML report is created from all CSV
files in the ‘reports/’ folder.
Related-PROD: PROD-36943
Change-Id: Iceff8b168364219a01b60546bb9908e01d61c434
diff --git a/fixtures/base.py b/fixtures/base.py
index ed00d80..fb4fb43 100644
--- a/fixtures/base.py
+++ b/fixtures/base.py
@@ -5,6 +5,7 @@
import time
import logging
+from utils import helpers
from utils import os_client
@@ -211,3 +212,9 @@
else:
logger.info("Project {} is not empty, skip deleting".format(
alt_project.name))
+
+
+@pytest.fixture(scope="session")
+def html_report():
+ yield
+ helpers.convert_csvs_to_single_html_report()
diff --git a/requirements.txt b/requirements.txt
index 6898f80..556da96 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,5 @@
+jinja2 # BSD License (BSD-3-Clause)
+pandas==1.5.3 # BSD License (BSD-3-Clause)
paramiko==2.7.2 # LGPLv2.1+
pytest==4.6.11 # MIT
python-cinderclient==6.0.0 # Apache-2.0
diff --git a/templates/table_styles.css.j2 b/templates/table_styles.css.j2
new file mode 100644
index 0000000..afad011
--- /dev/null
+++ b/templates/table_styles.css.j2
@@ -0,0 +1,106 @@
+<style>
+ h3 {
+ font-size: 12px;
+ font-weight: 900;
+ color: #214666;
+ line-height: 1.2;
+
+ }
+
+ h1 {
+ font-size: 22px;
+ font-weight: 900;
+ color: #06203a;
+ line-height: 1.2;
+ margin-bottom: 18px;
+ }
+
+ body {
+ padding: 0 50px;
+ font-family: Verdana, "Helvetica Neue", Arial, Helvetica, Geneva, sans-serif;
+ margin-bottom: 30px;
+ }
+
+ table {
+ border-collapse: collapse;
+ font-size: 11px;
+ width: 100%;
+ }
+
+ th, td {
+ text-align: left;
+ padding: 6px;
+ }
+
+ td {
+ color: black;
+ }
+
+ th {
+ background-color: #214666;
+ color: white;
+ text-align: center;
+ }
+
+ table.dataframe th:first-child, table.dataframe td:first-child {
+ width: 27%;
+ }
+
+ td:nth-child(n+2):nth-child(-n+6) {
+ color: #66707A;
+ }
+
+ td:last-child {
+ font-weight: bolder;
+ font-size: 13px;
+ color: black !important;
+ }
+
+ table.dataframe th:last-child, table.dataframe td:last-child {
+ width: 27%;
+ }
+
+ tr:nth-child(even) {
+ background-color: #EFF2F4;
+ }
+
+ ul {
+ list-style: none;
+ font-size: 11px;
+ color: #66707A;
+ line-height: 1.5;
+ }
+
+ ul li::before {
+ content: "";
+ display: inline-block;
+ width: 8px;
+ height: 8px;
+ margin-right: 10px;
+ background-color: #428CCB;
+ }
+
+ .timestamp {
+ text-align: right;
+ color: #66707A;
+ top: 0;
+ right: 0;
+ margin: 20px;
+ font-size: 11px;
+ position: absolute;
+ }
+
+ .test-header {
+ display: flex;
+ justify-content: space-between;
+ align-items: center;
+ margin-top: 15px;
+ margin-bottom: -7px;
+ }
+
+ .note {
+ font-size: 11px;
+ color: #66707A;
+ }
+
+</style>
diff --git a/templates/template.j2 b/templates/template.j2
new file mode 100644
index 0000000..5003170
--- /dev/null
+++ b/templates/template.j2
@@ -0,0 +1,25 @@
+<!DOCTYPE html>
+<html>
+ <head>
+ <title>SPT tests</title>
+ {% include 'templates/table_styles.css.j2' %}
+ </head>
+ <body>
+ <h1>Simplified Performance Tests (SPT)</h1>
+ <ul><strong>Configuration used in tests:</strong>
+ {% for key, value in tests_config_data.items() %}
+ <li>{{ key }}: <i>{{ value }}</i></li>
+ {% endfor %}
+ </ul>
+
+ {% for table in tables %}
+ <div class="test-header">
+ <h3>{{ table.title }}</h3>
+ <p class="note">Executed at: <i>{{ table.executed_at }}</i></p>
+ </div>
+ {{ table.df.to_html(index=False) }}
+ {% endfor %}
+
+ <p class="timestamp">Report generated at: {{ human_timestamp }}</p>
+ </body>
+</html>
diff --git a/tests/test_glance.py b/tests/test_glance.py
index 85d070d..1b6f973 100644
--- a/tests/test_glance.py
+++ b/tests/test_glance.py
@@ -6,6 +6,9 @@
import time
import utils
+from utils import helpers
+
+from texttable import Texttable
logger = logging.getLogger(__name__)
@@ -40,7 +43,8 @@
subprocess.call('rm -f /tmp/image_mk_framework.download', shell=True)
-def test_speed_glance(create_image, openstack_clients, record_property):
+def test_speed_glance(create_image, openstack_clients,
+ request, html_report):
"""
Simplified Performance Tests Download / upload Glance
1. Create file with random data (dd)
@@ -48,6 +52,9 @@
3. Download image.
4. Measure download/upload speed and print them into stdout
"""
+ result_table = Texttable(max_width=120)
+ table_rows = [["Test Speed Glance", "Image Size", "Time Consumed",
+ "Result"]]
image_size_megabytes = utils.get_configuration().get("IMAGE_SIZE_MB")
if not is_parsable(image_size_megabytes, int):
pytest.fail("Can't convert IMAGE_SIZE_MB={} to 'int'".format(
@@ -80,7 +87,12 @@
"Occurred error: {}".format(e))
end_time = time.time()
- speed_upload = image_size_megabytes / (end_time - start_time)
+ time_diff = end_time - start_time
+ speed_upload = image_size_megabytes / time_diff
+ table_rows.append(["Upload",
+ "{} MB".format(image_size_megabytes),
+ "{} s".format(round(time_diff, 3)),
+ "{} MB/s".format(round(speed_upload, 2))])
logger.info("Testing download file speed...")
start_time = time.time()
@@ -89,13 +101,18 @@
image_file.write(item)
end_time = time.time()
- speed_download = image_size_megabytes / (end_time - start_time)
+ time_diff = end_time - start_time
+ speed_download = image_size_megabytes / time_diff
+ table_rows.append(["Download",
+ "{} MB".format(image_size_megabytes),
+ "{} s".format(round(time_diff, 3)),
+ "{} MB/s".format(round(speed_download, 2))])
logger.info("Deleted image {}.".format(image.id))
openstack_clients.image.images.delete(image.id)
- record_property("Upload", speed_upload)
- record_property("Download", speed_download)
- sys.stdout.write("\n++++++++++++++++++++++++++++++++++++++++")
- sys.stdout.write(('\nupload - {} MB/s'.format(speed_upload)))
- sys.stdout.write(('\ndownload - {} MB/s'.format(speed_download)))
- sys.stdout.write("\n++++++++++++++++++++++++++++++++++++++++\n")
+ result_table.add_rows(table_rows)
+ sys.stdout.write('\n{}\n'.format(result_table.draw()))
+
+ # Send the results to CSV file at reports/ directory
+ helpers.create_test_result_table_csv_file(
+ table_rows, request.node.name)
diff --git a/tests/test_vm2vm.py b/tests/test_vm2vm.py
index 44b584e..8a94600 100644
--- a/tests/test_vm2vm.py
+++ b/tests/test_vm2vm.py
@@ -6,6 +6,7 @@
from texttable import Texttable
import utils
+from utils import helpers
from utils import os_client
from utils import ssh
@@ -13,7 +14,7 @@
logger = logging.getLogger(__name__)
-def test_vm2vm(openstack_clients, pair, os_resources, record_property):
+def test_vm2vm(openstack_clients, pair, os_resources, request, html_report):
"""
Simplified Performance Tests VM to VM test in different topologies
1. Create 4 VMs admin project
@@ -208,6 +209,10 @@
result_table.add_rows(table_rows)
sys.stdout.write('\n{}\n'.format(result_table.draw()))
+ # Send the results to CSV file at reports/ directory
+ helpers.create_test_result_table_csv_file(
+ table_rows, request.node.name)
+
logger.info("Removing VMs and FIPs...")
for vm in vms:
openstack_clients.compute.servers.delete(vm)
diff --git a/tests/test_vm2vm_different_routers.py b/tests/test_vm2vm_different_routers.py
index 8d20090..31c4478 100644
--- a/tests/test_vm2vm_different_routers.py
+++ b/tests/test_vm2vm_different_routers.py
@@ -6,6 +6,7 @@
from texttable import Texttable
import utils
+from utils import helpers
from utils import os_client
from utils import ssh
@@ -15,8 +16,7 @@
def test_vm2vm_different_project_different_routers(
openstack_clients, openstack_alt_clients, pair,
- os_resources, os_resources_alt_project,
- record_property):
+ os_resources, os_resources_alt_project, request, html_report):
"""
Simplified Performance Tests VM to VM test in different projects, different
networks, different routers, measure by Floating IPs (common floating net):
@@ -226,6 +226,10 @@
result_table.add_rows(table_rows)
sys.stdout.write('\n{}\n'.format(result_table.draw()))
+ # Send the results to CSV file at reports/ directory
+ helpers.create_test_result_table_csv_file(
+ table_rows, request.node.name)
+
logger.info("Removing VMs and FIPs...")
for vm in vms:
openstack_clients.compute.servers.delete(vm)
diff --git a/utils/helpers.py b/utils/helpers.py
index add9e87..92ab0c0 100644
--- a/utils/helpers.py
+++ b/utils/helpers.py
@@ -1,26 +1,116 @@
-import sys
+import csv
+import datetime
+import logging
+import os
+import time
-import texttable as tt
+from jinja2 import Environment, FileSystemLoader
+import pandas as pd
+
+import utils
+
+logger = logging.getLogger(__name__)
-class helpers(object):
+def get_tests_configuration_data():
+ """
+ Gets the values from the config file and generates the dictionary with the
+ human-readable configuration.
+ """
+ config = utils.get_configuration()
+ glance_test_file_size = config.get("IMAGE_SIZE_MB", 9000)
+ image = config.get('image_name', 'cvp.ubuntu.2004')
+ ext_net = config.get('external_network') or ''
+ utility_single_thread = "iperf3" # not configurable, just for info
+ utility_multiple_threads = str(
+ config.get('multiple_threads_iperf_utility', 10))
+ iperf_time = int(config.get('iperf_time', 60))
- def __init__(self):
- pass
+ tests_data = {
+ "File size in the Glance test": "{} MB".format(glance_test_file_size),
+ "Image used in the vm2vm tests": image,
+ "Floating network used": ext_net,
+ "Network performance tool used for "
+ "single thread tests": utility_single_thread,
+ "Network performance tool used for "
+ "multiple threads tests": utility_multiple_threads,
+ "iperf time in seconds "
+ "to transmit for each test (iperf -t)": iperf_time
+ }
+ return tests_data
- def draw_table_with_results(self, global_results):
- tab = tt.Texttable()
- header = [
- 'node name 1',
- 'node name 2',
- 'network',
- 'bandwidth >',
- 'bandwidth <',
- ]
- tab.set_cols_align(['l', 'l', 'l', 'l', 'l'])
- tab.set_cols_width([27, 27, 15, 20, '20'])
- tab.header(header)
- for row in global_results:
- tab.add_row(row)
- s = tab.draw()
- sys.stdout.write(s)
+
+def create_test_result_table_csv_file(text_table_rows, test_name):
+ """
+ Gets the text-table and saves to the separate CSV file.
+ """
+ dir_path = 'reports'
+ if not os.path.exists(dir_path):
+ os.makedirs(dir_path)
+ csv_name = "{}.csv".format(test_name)
+ csv_report_path = "{}/{}".format(dir_path, csv_name)
+ with open(csv_report_path, 'w', newline='') as csvfile:
+ writer = csv.writer(csvfile)
+ writer.writerows(text_table_rows)
+
+
+def read_csv_files_to_dataframes(csv_files_list):
+ """
+ The pandas module reads the list of the CSV files and saves them to the
+ dataframe objects. Returns the dictionary with DFs, title and time for
+ each result table.
+ """
+ df_tables = []
+ for csv_file in csv_files_list:
+ # Read the CSVs to the DataFrames
+ df = pd.read_csv(csv_file)
+ # Get the file age (to prevent the cases when there are some old CSV
+ # files, the tests are executed later, got the error, but the test
+ # report is generated from the old files).
+ # Also, if the tests are executed one by one in different time, all
+ # results are collected with the proper age of each file.
+ executed_at = os.path.getctime(csv_file)
+ df_tables.append({
+ 'df': df, # the results table of a test case
+ 'title': csv_file[8:-4], # remove "reports/", ".csv" to get names
+ 'executed_at': time.ctime(executed_at) # time of test execution
+ })
+ return df_tables
+
+
+def convert_csvs_to_single_html_report():
+ """
+ Generates the single HTML report from the CSV files at reports/ dir.
+ Uses the template and CSS styles from the templates/.
+ """
+ dir_path = 'reports/'
+ if not os.path.exists(dir_path):
+ logger.warning("Could not generate the HTML report since there is no "
+ "{} folder.".format(dir_path))
+ return
+ datetime_now = datetime.datetime.now()
+ timestamp = datetime_now.strftime("%d%m%Y_%H%M%S")
+ report_file = "spt_report_{}.html".format(timestamp)
+ csv_files = ["{}{}".format(dir_path, f)
+ for f in sorted(os.listdir(path=dir_path))
+ if f.endswith('.csv')]
+ if not csv_files:
+ logger.warning("Could not generate the HTML report since the CSV files"
+ " are absent in the {} folder.".format(dir_path))
+ return
+
+ env = Environment(loader=FileSystemLoader('.'))
+ template = env.get_template('templates/template.j2')
+
+ tests_config_data = get_tests_configuration_data()
+ human_timestamp = datetime_now.strftime("%Y-%m-%d %H:%M:%S")
+ df_tables = read_csv_files_to_dataframes(csv_files)
+ html = template.render(
+ tables=df_tables,
+ tests_config_data=tests_config_data,
+ human_timestamp=human_timestamp
+ )
+
+ with open(report_file, 'w') as f:
+ f.write(html)
+ logger.info("The HTML report {} is created.".format(report_file))