Add generate_report project
Change-Id: I062dc72dcbd9de70f09029df9d666ef7494417d5
PROD-28569
diff --git a/daily_jenkins_job_report/README.md b/daily_jenkins_job_report/README.md
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/daily_jenkins_job_report/README.md
diff --git a/daily_jenkins_job_report/daily_report/config.py b/daily_jenkins_job_report/daily_report/config.py
new file mode 100644
index 0000000..d49b71f
--- /dev/null
+++ b/daily_jenkins_job_report/daily_report/config.py
@@ -0,0 +1,29 @@
+# Jenkins API credantials
+USERNAME = 'mcp-oscore-jenkins'
+PASSWORD = 'ahvoNg4mae'
+JENKINS_URL = 'https://ci.mcp.mirantis.net'
+
+
+# For get_jobs_results.py save_results_to_html method
+# GENERATED_REPORT = '/var/www/oscore_jobs.com/html/reports/'
+GENERATED_REPORT = \
+ '/home/serhii/my_projects/osccore-qa-testing-tools/' \
+ 'daily_jenkins_job_report/daily_report'
+REPORT_TEMPLATE = 'templates/report_template.html'
+
+# For generating report
+MULTIJOBS = ['oscore-oscc-ci',
+ 'oscore-promote-openstack-pike-xenial',
+ 'oscore-promote-openstack-queens-xenial'
+ ]
+
+SINGLEJOBS = ['oscore-test-openstack-upgrade-ocata-pike-core',
+ 'oscore-test-openstack-upgrade-pike-queens-core-barbican',
+ 'oscore-test-openstack-upgrade-pike-queens-core-ssl',
+ 'oscore-test-openstack-upgrade-pike-queens-core-extra-ssl'
+ ]
+
+# Logging
+LOG_FOLDER = '/tmp/'
+LOG_FILENAME = 'daily_jenkins_jobs_report.log'
+LOGGER = 'generate_report'
diff --git a/daily_jenkins_job_report/daily_report/generate_report.py b/daily_jenkins_job_report/daily_report/generate_report.py
new file mode 100644
index 0000000..c26b1ed
--- /dev/null
+++ b/daily_jenkins_job_report/daily_report/generate_report.py
@@ -0,0 +1,206 @@
+"""
+-------------
+Generate report
+-------------
+"""
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# from daily_report
+import config
+import datetime
+import logging
+import re
+
+import jinja2
+from jinja2 import Template
+from jenkinsapi import custom_exceptions
+from jenkinsapi.jenkins import Jenkins
+from jenkinsapi.utils.crumb_requester import CrumbRequester
+
+
+logging.basicConfig(
+ format='[%(asctime)s][%(name)s][%(levelname)s] %(message)s',
+ datefmt='%d-%m-%Y %H:%M:%S',
+ handlers=[logging.FileHandler('{}{}'.format(
+ config.LOG_FOLDER, config.LOG_FILENAME)), logging.StreamHandler()],
+ level=logging.INFO)
+logger = logging.getLogger(config.LOGGER)
+
+
+class GetJobsResults:
+ def __init__(self):
+ self.server = Jenkins(config.JENKINS_URL,
+ username=config.USERNAME,
+ password=config.PASSWORD,
+ requester=CrumbRequester(
+ username=config.USERNAME,
+ password=config.PASSWORD,
+ baseurl=config.JENKINS_URL))
+
+ def get_console_output(self, job_name):
+ logger.info('Getting console output from: {}'.format(job_name))
+
+ job = self.server.get_job(job_name)
+
+ last_build = job.get_last_build()
+ console = last_build.get_console()
+ return console.lower()
+
+ def get_run_jobs_from_console_output(self, job_name):
+ logger.info(
+ 'Getting run jobs from console output: {}'.format(job_name))
+
+ job_console = self.get_console_output(job_name)
+ console_list = job_console.split('\n')
+
+ output = []
+ for i in console_list:
+ if 'starting building: oscore-' in i:
+ output.append(i)
+
+ jobs_ids = ''.join(output)
+ jobs_ids = jobs_ids.replace('starting building:', ' ')
+ jobs_ids = re.findall(r"oscore-[\w-]+ #\d+", jobs_ids)
+
+ res = {}
+ for i in jobs_ids:
+ name_id = i.split(' #')
+ res[name_id[1]] = name_id[0]
+
+ return res
+
+ def get_job_results(self, job_name, get_last_build=False, job_id=None):
+
+ results_multijobs = {}
+ try:
+ logger.info(
+ 'Getting IDs multijobs: {} {}'.format(job_name, job_id))
+
+ job = self.server.get_job(job_name)
+
+ if get_last_build:
+ last_build = job.get_last_build()
+ job_id = last_build.get_number()
+
+ build = job.get_build(int(job_id))
+ build_params = build.get_params()
+ baseurl = build.baseurl
+ build_status = str(build.get_status())
+ timestamp = build.get_timestamp().timestamp()
+
+ try:
+ job_name = build_params['COOKIECUTTER_TEMPLATE_CONTEXT_FILE']
+ except KeyError:
+ try:
+ job_name = build_params['STACK_CLUSTER_NAME']
+ except KeyError:
+ logger.warning(
+ 'KeyError, there are no '
+ 'COOKIECUTTER_TEMPLATE_CONTEXT_FILE '
+ 'or STACK_CLUSTER_NAME')
+ pass
+
+ results_multijobs['build_status'] = build_status
+ results_multijobs['job_name'] = job_name
+ results_multijobs['baseurl'] = baseurl
+ results_multijobs['timestamp'] = timestamp
+ logger.info('build status: {} job name: {} baseurl: {}'.format(
+ build_status, job_name, baseurl))
+ return results_multijobs
+ except custom_exceptions.NotFound:
+ logger.warning('Exception, NotFound: {}'.format(
+ type(custom_exceptions.NotFound)))
+ logger.warning('Job was erased. Exception, NotFound: {}'.format(
+ job_name))
+ results_multijobs['build_status'] = 'No Results'
+ results_multijobs['job_name'] = job_name
+ results_multijobs['baseurl'] = 'No Results'
+ results_multijobs['timestamp'] = '0.0'
+ return results_multijobs
+
+ def get_results_multijobs(self, job_names_to_ids):
+ logger.info('Getting results multijobs: {}'.format(job_names_to_ids))
+
+ list_results = []
+ for job_id, job_name in job_names_to_ids.items():
+ results_multijobs = self.get_job_results(job_id=job_id,
+ job_name=job_name)
+
+ list_results.append(results_multijobs)
+ return list_results
+
+ def get_results_singlejobs(self, job_name):
+ logger.info('Getting results single jobs: {}'.format(job_name))
+
+ results_singlejobs = self.get_job_results(job_name=job_name,
+ get_last_build=True)
+ return results_singlejobs
+
+ def job(self, job_name):
+ return self.server.get_job(job_name)
+
+
+def get_all_jobs_results():
+ logger.info('Getting all jobs results')
+ jr = GetJobsResults()
+
+ m_jobs = {}
+ for job_name in config.MULTIJOBS:
+ logger.info('Getting results multi jobs: {}'.format(job_name))
+ job_names_run_ids = jr.get_run_jobs_from_console_output(job_name)
+
+ logger.info('Jobs names run IDs: {}'.format(job_names_run_ids))
+ m_res = jr.get_results_multijobs(job_names_run_ids)
+ m_jobs[job_name] = m_res
+
+ s_jobs = {}
+ for job_name in config.SINGLEJOBS:
+ s_res = jr.get_results_singlejobs(job_name)
+ s_jobs[job_name] = s_res
+
+ return {'multi_results': m_jobs, 'single_results': s_jobs}
+
+
+def save_results_to_html(all_jobs_results):
+ filename = datetime.datetime.now().strftime("%d-%m-%Y_%H_%M") + '.html'
+ logger.info('Saving results to html file: {}'.format(filename))
+
+ filename = config.GENERATED_REPORT + filename
+ html = open(config.REPORT_TEMPLATE).read()
+ template = Template(html)
+
+ with open(filename, 'w') as fh:
+ fh.write(template.render(results=all_jobs_results))
+ return filename
+
+
+def datetimeformat(format):
+ """
+ Filter for jinja to get date time from timestamp
+
+ :param format: 'timestamp': 1550768955.0
+ :return: 2019-03-01 03:49:35
+ """
+ value = float(format)
+ return datetime.datetime.utcfromtimestamp(
+ value).strftime('%d-%m-%Y %H:%M:%S')
+
+
+jinja2.filters.FILTERS['datetimeformat'] = datetimeformat
+
+
+if __name__ == '__main__':
+ all_jobs_results = get_all_jobs_results()
+ save_results_to_html(all_jobs_results)
diff --git a/daily_jenkins_job_report/daily_report/templates/report_template.html b/daily_jenkins_job_report/daily_report/templates/report_template.html
new file mode 100644
index 0000000..030c2a7
--- /dev/null
+++ b/daily_jenkins_job_report/daily_report/templates/report_template.html
@@ -0,0 +1,92 @@
+<!DOCTYPE html>
+
+ {% set red_color = "#FFA07A" %}
+ {% set green_color = "#98FB98" %}
+ {% set grey_color = "#f2f2f2" %}
+
+ <html>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+ <style>
+ table {
+ width:85%;
+ }
+ table, th, td {
+ border: 1px solid black;
+ border-collapse: collapse;
+ }
+ th, td {
+ padding: 3px;
+ text-align: left;
+ }
+ table#t01 {
+ background-color: #eef;
+ }
+ table#t02 {
+ background-color: #eee;
+ }
+ </style>
+ </head>
+ <body>
+
+ {% for key, value in results['multi_results'].items() %}
+ <h3><a href="https://ci.mcp.mirantis.net/job/{{ key }}">{{ key }}</a></h3>
+ <table>
+
+ <tr>
+ <th>DateTime</th>
+ <th>Build status</th>
+ <th>Job name</th>
+ <th>Base URL</th>
+ </tr>
+
+ {% for item in value %}
+ {% if item.build_status == "SUCCESS" %}
+ <tr bgcolor="{{ green_color }}">
+ {% elif item.build_status == "FAILURE" %}
+ <tr bgcolor="{{ red_color }}">
+ {% else %}
+ <tr bgcolor="{{ grey_color }}">
+ {% endif %}
+ <td>{{ item.timestamp|datetimeformat }}</td>
+ <td>{{ item.build_status }}</td>
+ {% if 'https://ci.mcp.mirantis.net/job/oscore-formula-systest-virtual_mcp11_' in item.baseurl %}
+ <td>{{ item.job_name }}{{ item.baseurl[68:-5] }}</td>
+ {% else %}
+ <td>{{ item.job_name }}</td>
+ {% endif %}
+
+ <td><a href="{{ item.baseurl }}">{{ item.baseurl[32:] }}</a></td>
+ </tr>
+ {% endfor %}
+ </table>
+ {% endfor %}
+
+ <h3>Upgrade Jobs</h3>
+ <table>
+
+ <tr>
+ <th>DateTime</th>
+ <th>Build status</th>
+ <th>Job name</th>
+ <th>Base URL</th>
+ </tr>
+
+ {% for key, value in results['single_results'].items() %}
+ {% if value.build_status == "SUCCESS" %}
+ <tr bgcolor="{{ green_color }}">
+ {% elif value.build_status == "FAILURE" %}
+ <tr bgcolor="{{ red_color }}">
+ {% else %}
+ <tr bgcolor="{{ grey_color }}">
+ {% endif %}
+
+ <td>{{ value.timestamp|datetimeformat }}</td>
+ <td>{{ value.build_status }}</td>
+ <td>{{ value.job_name }}</td>
+ <td><a href="{{ value.baseurl }}">{{ value.baseurl[32:] }}</a></td>
+ </tr>
+ {% endfor %}
+ </table>
+ </body>
+ </html>
diff --git a/daily_jenkins_job_report/index.rst b/daily_jenkins_job_report/index.rst
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/daily_jenkins_job_report/index.rst
diff --git a/daily_jenkins_job_report/requirements.txt b/daily_jenkins_job_report/requirements.txt
new file mode 100644
index 0000000..aebbc58
--- /dev/null
+++ b/daily_jenkins_job_report/requirements.txt
@@ -0,0 +1,3 @@
+jenkinsapi>=0.3.6
+jinja2>=2.10
+setuptools>=40.8.0
\ No newline at end of file
diff --git a/daily_jenkins_job_report/setup.py b/daily_jenkins_job_report/setup.py
new file mode 100644
index 0000000..605b80b
--- /dev/null
+++ b/daily_jenkins_job_report/setup.py
@@ -0,0 +1,11 @@
+from setuptools import setup
+
+setup(
+ name='daily_report',
+ version='1.0',
+ description='Generates daily report from nightly Jenkins jobs',
+ author='Serhii Turivnyi',
+ author_email='sturivnyi@mirantis.com',
+ packages=['daily_report'], #same as name
+ install_requires=['jenkinsapi', 'jinja2', 'setuptools'], #external packages as dependencies
+)
\ No newline at end of file
diff --git a/daily_jenkins_job_report/tests/test_basic.py b/daily_jenkins_job_report/tests/test_basic.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/daily_jenkins_job_report/tests/test_basic.py