PRODX-39509 Implementing auto-triggering the analyzing of today's run
Added time enums for easy controlling of time duration in code
Added clock to the nav bar
Changed default created_after to more proper way of definition through the Django's timedelta,
because standard datetime library can't pass it's value to the model
Change-Id: I8411853dfc3c420a96254a1091f093c24a4217ae
diff --git a/testrail_bot/control/celery_tasks/enums.py b/testrail_bot/control/celery_tasks/enums.py
index 30ccbab..40f4edb 100644
--- a/testrail_bot/control/celery_tasks/enums.py
+++ b/testrail_bot/control/celery_tasks/enums.py
@@ -22,3 +22,10 @@
def __str__(self):
return str(self.value)
+
+
+class TimeEnum(IntEnum):
+ SECONDS = 1
+ MINUTES = 60
+ HOURS = 60 * MINUTES
+ DAYS = 24 * HOURS
diff --git a/testrail_bot/control/celery_tasks/schedules_pipeline.py b/testrail_bot/control/celery_tasks/schedules_pipeline.py
new file mode 100644
index 0000000..ef35323
--- /dev/null
+++ b/testrail_bot/control/celery_tasks/schedules_pipeline.py
@@ -0,0 +1,74 @@
+from datetime import datetime, timedelta, timezone
+import os
+
+from .. import models
+from .test_rail_api import get_planid_by_name
+from . import tasks
+
+
+def task_to_check_today_testplan():
+ """
+ Searching the today testplan
+ Creates BotTestRun with this id
+ Creates Periodic task to analyze created TestRun
+
+ :return:
+ """
+ today = datetime.today().strftime("%Y-%m-%d")
+ plan_name = f"[MCP2.0]OSCORE-{today}"
+ plan_id = get_planid_by_name(
+ name=plan_name,
+ project_name="Mirantis Cloud Platform")
+ if not plan_id:
+ print(f"Can't found {plan_name} TestPlan")
+ return
+
+ testrun_obj, _ = models.TestRailTestRun.objects.get_or_create(
+ run_name=f"[by scheduler] {plan_name}",
+ run_id=plan_id,
+ caching_tests_enabled=True
+ )
+ date = datetime.isoformat(datetime.now())
+ report_name = f"{testrun_obj.run_name}-{date}"
+ path = os.path.join(models.fs.location, report_name)
+ with open(path, "w"):
+ pass
+
+ report_obj = models.TestRailReport.objects.create(
+ report_name=report_name,
+ path=path)
+
+ return tasks.process_run(bot_run_id=testrun_obj.id,
+ report_id=report_obj.id,
+ path=path,
+ is_testplan=True)
+
+
+def task_to_check_testplan(testplan_id: int):
+ """
+ Creates BotTestRun with this id
+ Creates Periodic task to analyze created TestRun
+ :param testplan_id:
+ :return:
+ """
+
+ testrun_obj, _ = models.TestRailTestRun.objects.get_or_create(
+ run_name=f"[by scheduler] plan {testplan_id}",
+ run_id=testplan_id,
+ caching_tests_enabled=True
+ )
+ date = datetime.isoformat(datetime.now())
+ report_name = f"{testrun_obj.run_name}-{date}"
+ path = os.path.join(models.fs.location, report_name)
+ with open(path, "w"):
+ pass
+
+ report_obj = models.TestRailReport.objects.create(
+ report_name=report_name,
+ path=path)
+
+ return tasks.process_run(bot_run_id=testrun_obj.id,
+ report_id=report_obj.id,
+ path=path,
+ is_testplan=True)
+
diff --git a/testrail_bot/control/celery_tasks/tasks.py b/testrail_bot/control/celery_tasks/tasks.py
index 6450c68..d378827 100644
--- a/testrail_bot/control/celery_tasks/tasks.py
+++ b/testrail_bot/control/celery_tasks/tasks.py
@@ -3,8 +3,7 @@
import traceback
from celery import shared_task
-from . import jenkins_pipeline
-from . import testrail_pipeline
+from . import jenkins_pipeline, testrail_pipeline, schedules_pipeline
@shared_task
@@ -38,3 +37,15 @@
r.status = "Unexpected fail"
r.finished = True
r.save()
+
+
+@shared_task
+def check_today_testplan():
+ """
+ Finds today testplan
+ Creates TestRun with this id
+ Creates Periodic task to analyze created TestRun
+
+ :return:
+ """
+ schedules_pipeline.task_to_check_today_testplan()
diff --git a/testrail_bot/control/celery_tasks/test_rail_api.py b/testrail_bot/control/celery_tasks/test_rail_api.py
index 00d4dc6..142f403 100644
--- a/testrail_bot/control/celery_tasks/test_rail_api.py
+++ b/testrail_bot/control/celery_tasks/test_rail_api.py
@@ -4,9 +4,8 @@
from django.utils.html import escape
from typing import Optional, List, Iterator
-from functools import lru_cache
from retry import retry
-from .enums import StatusEnum
+from .enums import StatusEnum, TimeEnum
from ..utils import cached
api = TestRailAPI(
@@ -26,7 +25,7 @@
return None
-@cached(timeout=24*60*60)
+@cached(timeout=1*TimeEnum.DAYS)
def get_suite_by_id(suite_id: int) -> dict:
return api.suites.get_suite(suite_id)
@@ -42,13 +41,31 @@
return suite_name.split(']')[1]
-@cached(timeout=24*60*60)
+@cached(timeout=1*TimeEnum.HOURS)
def get_plans(project_id: int, **kwargs) -> List[dict]:
plans = api.plans.get_plans(project_id=project_id, **kwargs)['plans']
return plans
-@cached(timeout=1*60*60)
+@cached(timeout=2*TimeEnum.HOURS,
+ condition_for_endless_cache=lambda x: x is not None)
+def get_planid_by_name(name: str, project_name: str, **kwargs) \
+ -> Optional[int]:
+ limit_step = 100
+ for offset in range(0, 500, limit_step):
+ plans = get_plans(project_id=get_project_id(project_name),
+ limit=limit_step,
+ offset=offset)
+ if not plans:
+ return
+ for plan in plans:
+ print(f"{plan['name']=}")
+ if plan["name"] == name:
+ return plan["id"]
+ return
+
+
+@cached(timeout=1*TimeEnum.HOURS)
def get_entries(plan_id: int) -> List[dict]:
return api.plans.get_plan(plan_id)["entries"]
@@ -116,7 +133,7 @@
return entries[0]["runs"][0]["id"]
-@cached(timeout=2*60*60,
+@cached(timeout=2*TimeEnum.HOURS,
condition_for_endless_cache=lambda x: x is None)
@retry(ReadTimeout, delay=1, jitter=2, tries=3)
def get_result_for_case(run_id: int,
diff --git a/testrail_bot/control/forms.py b/testrail_bot/control/forms.py
index 7a45622..fc27349 100644
--- a/testrail_bot/control/forms.py
+++ b/testrail_bot/control/forms.py
@@ -1,6 +1,9 @@
from datetime import date
from django import forms
-from .models import TestRailTestRun, DiffOfSuitesPassRates, SuitePassRate
+from .models import TestRailTestRun, \
+ DiffOfSuitesPassRates, \
+ SuitePassRate, \
+ CronPeriodicTask
class TestRunForm(forms.ModelForm):
@@ -56,3 +59,9 @@
labels = {
"suite_id": "Suite ID"
}
+
+
+class PeriodicTaskForm(forms.ModelForm):
+ class Meta:
+ model = CronPeriodicTask
+ fields = ["id", "enabled", "name", "cron"]
diff --git a/testrail_bot/control/migrations/0009_cronperiodictask_alter_testrailtestrun_created_after.py b/testrail_bot/control/migrations/0009_cronperiodictask_alter_testrailtestrun_created_after.py
new file mode 100644
index 0000000..443115b
--- /dev/null
+++ b/testrail_bot/control/migrations/0009_cronperiodictask_alter_testrailtestrun_created_after.py
@@ -0,0 +1,32 @@
+# Generated by Django 4.2.7 on 2024-02-20 14:18
+
+import control.models
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('django_celery_beat', '0018_improve_crontab_helptext'),
+ ('control', '0008_rename_test_pattern_testrailtestrun_testrun_pattern'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='CronPeriodicTask',
+ fields=[
+ ('periodictask_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='django_celery_beat.periodictask')),
+ ('cron', models.CharField(default='', max_length=300)),
+ ],
+ options={
+ 'ordering': ['id'],
+ },
+ bases=('django_celery_beat.periodictask',),
+ ),
+ migrations.AlterField(
+ model_name='testrailtestrun',
+ name='created_after',
+ field=models.DateField(default=control.models.default_created_after),
+ ),
+ ]
diff --git a/testrail_bot/control/models.py b/testrail_bot/control/models.py
index 46c5049..6d7840b 100644
--- a/testrail_bot/control/models.py
+++ b/testrail_bot/control/models.py
@@ -1,6 +1,7 @@
from django.core.files.storage import FileSystemStorage
from django.db import models
-from django.utils.timezone import now
+from django.utils.timezone import now, timedelta
+from django_celery_beat.models import PeriodicTask
class IntegerListField(models.Field):
@@ -29,6 +30,10 @@
return ','.join(str(int(x)) for x in value)
+def default_created_after():
+ return now() + timedelta(days=-3 * 30)
+
+
class TestRailTestRun(models.Model):
project_name = models.CharField(max_length=300,
default="Mirantis Cloud Platform")
@@ -44,7 +49,7 @@
uuid_filter = models.BooleanField(default=True)
filter_last_traceback = models.BooleanField(default=True)
created_before = models.DateField(default=now)
- created_after = models.DateField(default=now)
+ created_after = models.DateField(default=default_created_after)
@property
def text_filters(self):
@@ -121,3 +126,24 @@
blank=True)
started_at = models.DateTimeField(auto_created=True,
auto_now=True)
+
+
+TASK_CHOICES = [
+ ("control.celery_tasks.tasks.check_today_testplan",
+ "Check today testplan",
+ []
+ ),
+ ("control.celery_tasks.tasks.task_to_check_testplan",
+ "Check testplan",
+ ["testplan_id"]
+ ),
+ ]
+
+
+class CronPeriodicTask(PeriodicTask):
+ cron = models.CharField(default="",
+ max_length=300,
+ blank=False)
+
+ class Meta:
+ ordering = ["id"]
diff --git a/testrail_bot/control/templates/base.html b/testrail_bot/control/templates/base.html
index 4bd5e45..fd772d3 100644
--- a/testrail_bot/control/templates/base.html
+++ b/testrail_bot/control/templates/base.html
@@ -27,12 +27,15 @@
<li class="nav-item">
<a class="nav-link" href="{% url 'list_of_comparing_reports' %}">Compare Suites</a></li>
<li class="nav-item">
+ <a class="nav-link" href="{% url 'schedulers' %}">Schedulers</a></li>
+ <li class="nav-item">
<a class="nav-link" href="{% url 'help' %}">Help</a></li>
<li class="nav-item">
- <a class="nav-link" href="{% url 'jenkins_plot' %}">Jenkins
- plot</a></li>
-
+ <a class="nav-link" href="{% url 'jenkins_plot' %}">Jenkins plot</a></li>
</ul>
+ <div class="position-absolute bottom-0 end-0 text-secondary">
+ {% now "jN H:i:s" %}
+ </div>
</div>
</nav>
{% block section %}{% endblock %}
diff --git a/testrail_bot/control/templates/control/scheduler.html b/testrail_bot/control/templates/control/scheduler.html
new file mode 100644
index 0000000..6a33562
--- /dev/null
+++ b/testrail_bot/control/templates/control/scheduler.html
@@ -0,0 +1,73 @@
+{% extends "base.html" %}
+{% load bootstrap5 %}
+{% block section %}
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.7.1/jquery.min.js"></script>
+<script>
+$(document).ready(function(){
+ $("#task-select").change(function(){
+ $("div#kwargs").empty()
+ var t = $("#task-select").val();
+ {% for task, text, args in TASK_CHOICES %}
+ {% if args %}
+ if (t == '{{ task }}') {
+ {% for arg in args %}
+ $("div#kwargs").append("<input type='text' name='{{ arg }}'>");
+ {% endfor %}
+ }
+ {% endif %}
+ {% endfor %}
+ });
+ });
+</script>
+
+<div class="p-2">
+ <input type="button" onclick="location.href={% url 'schedulers' %}" height="8"
+ class="btn btn-primary m-1" value="<- Back"/>
+
+ {% if pk %}
+ <h4>Edit scheduler</h4>
+ <form action="{% url 'save_scheduler' pk %}" method="post" class="form inline ">
+ {% else %}
+ <h4>Create scheduler</h4>
+ <form action="{% url 'create_scheduler' %}" method="post" class="form">
+ {% endif %}
+
+ <div class="btn-toolbar col-xs-7">
+ <button type="submit" class="btn btn-primary m-1">
+ <svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="1em"
+ height="1em" viewBox="0,0,256,256">
+ <g fill="#fffafa" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)">
+ <path d="M6,2c-0.13812,0 -0.27212,0.01438 -0.40234,0.04102c-0.91157,0.18642 -1.59766,0.99211 -1.59766,1.95898v18l8,-3l8,3v-18c0,-0.1375 -0.01426,-0.27246 -0.04102,-0.40234c-0.15978,-0.78135 -0.77529,-1.39686 -1.55664,-1.55664c-0.13022,-0.02663 -0.26422,-0.04102 -0.40234,-0.04102z"></path></g></g>
+ </svg>
+ Save
+ </button>
+ {% if pk %}
+ <button type="submit" formaction="{% url 'delete_scheduler' pk %}"
+ class="btn btn-danger m-1">
+ <svg xmlns="http://www.w3.org/2000/svg" x="0px" y="0px" width="1em"
+ height="1em" viewBox="0,0,256,256">
+ <g fill="#ffffff" fill-rule="nonzero" stroke="none" stroke-width="1" stroke-linecap="butt" stroke-linejoin="miter" stroke-miterlimit="10" stroke-dasharray="" stroke-dashoffset="0" font-family="none" font-weight="none" font-size="none" text-anchor="none" style="mix-blend-mode: normal"><g transform="scale(10.66667,10.66667)">
+ <path d="M10,2l-1,1h-4c-0.6,0 -1,0.4 -1,1c0,0.6 0.4,1 1,1h2h10h2c0.6,0 1,-0.4 1,-1c0,-0.6 -0.4,-1 -1,-1h-4l-1,-1zM5,7v13c0,1.1 0.9,2 2,2h10c1.1,0 2,-0.9 2,-2v-13zM9,9c0.6,0 1,0.4 1,1v9c0,0.6 -0.4,1 -1,1c-0.6,0 -1,-0.4 -1,-1v-9c0,-0.6 0.4,-1 1,-1zM15,9c0.6,0 1,0.4 1,1v9c0,0.6 -0.4,1 -1,1c-0.6,0 -1,-0.4 -1,-1v-9c0,-0.6 0.4,-1 1,-1z"></path></g></g>
+ </svg>
+ Delete
+ </button>
+ {% endif %}
+ </div>
+ {% csrf_token %}
+ {% bootstrap_form_errors form type='non_fields' %}
+
+ <div class="form-check form-switch col-xs-8 py-3">
+ {% bootstrap_field form.enabled size='sm' %}</div>
+ <div class="col-xs-8">
+ {% bootstrap_field form.name size='sm' %}</div>
+ <div class="col-xs-8">
+ {% bootstrap_field form.cron size='sm' %}</div>
+
+ <select name="task" id="task-select">
+ <option value="control.celery_tasks.tasks.check_today_testplan">
+ Check today testplan</option>
+ </select>
+ <div id="kwargs"></div>
+ </form>
+</div>
+{% endblock %}
diff --git a/testrail_bot/control/templates/control/schedulers.html b/testrail_bot/control/templates/control/schedulers.html
new file mode 100644
index 0000000..b0944cb
--- /dev/null
+++ b/testrail_bot/control/templates/control/schedulers.html
@@ -0,0 +1,19 @@
+{% extends "base.html" %}
+{% block section %}
+<input type="button" onclick="location.href='{% url 'create_scheduler' %}'"
+ class="btn btn-primary m-1" value="Create New" />
+<p><b>Schedulers</b> (total {{ schedulers|length }} schedulers):</p>
+<div class="list-group">
+ {% for scheduler in schedulers %}
+ <a href="{% url 'scheduler' scheduler.id %}"
+ class="list-group-item list-group-item-success">
+ <div>
+ {{ scheduler.id }}. {{ scheduler.name }}
+ total_run_count: {{ scheduler.total_run_count }}
+ start_time: {{ scheduler.start_time }}
+ enabled {{ scheduler.enabled }}
+ </div>
+ </a>
+ {% endfor %}
+</div>
+{% endblock %}
diff --git a/testrail_bot/control/urls.py b/testrail_bot/control/urls.py
index 6592132..d385891 100644
--- a/testrail_bot/control/urls.py
+++ b/testrail_bot/control/urls.py
@@ -2,6 +2,7 @@
from . import views
+
urlpatterns = [
path("", views.redirect_to_index, name="redirect"),
path("runs/", views.create_run, name="create_run"),
@@ -18,6 +19,14 @@
views.update_jenkins_plot,
name="update_jenkins"),
path("jenkins_plot", views.jenkins_plot, name="jenkins_plot"),
+ path("schedulers/", views.schedulers, name="schedulers"),
+
+ path("scheduler/new/", views.scheduler, name="create_scheduler"),
+ path("scheduler/<int:pk>/", views.scheduler, name="scheduler"),
+ path("scheduler/<int:pk>/save", views.save_scheduler, name="save_scheduler"),
+ path("scheduler/<int:pk>/delete/", views.delete_scheduler,
+ name="delete_scheduler"),
+
path("compare_suites/new/", views.compare_suites, name="compare_suites"),
path("compare_suites/",
views.list_of_comparing_reports,
diff --git a/testrail_bot/control/views.py b/testrail_bot/control/views.py
index bff73e0..8d5f692 100644
--- a/testrail_bot/control/views.py
+++ b/testrail_bot/control/views.py
@@ -4,6 +4,7 @@
from .celery_tasks import test_rail_api
from django.shortcuts import render, redirect, HttpResponse
+from django_celery_beat.models import CrontabSchedule, PeriodicTasks
from . import models
from . import forms
@@ -240,3 +241,79 @@
"report2": report.report2,
"is_finished": report.report1.finished and report.report2.finished,
"diff_table": diff_table})
+
+
+def schedulers(request):
+
+ return render(
+ request, "control/schedulers.html",
+ {"schedulers": models.CronPeriodicTask.objects.all()})
+
+
+def scheduler(request, pk=None):
+ if request.method == "POST":
+ return save_scheduler(request)
+
+ if pk:
+ task_pk = models.CronPeriodicTask.objects.get(pk=pk)
+ form = forms.PeriodicTaskForm(instance=task_pk)
+ else:
+ form = forms.PeriodicTaskForm()
+
+ return render(
+ request, "control/scheduler.html",
+ {
+ "form": form,
+ "pk": pk,
+ "TASK_CHOICES": models.TASK_CHOICES
+ }
+ )
+
+
+def save_scheduler(request, pk=None):
+ minute, hour, day_of_month, month_of_year, day_of_week = \
+ request.POST.get("cron", "* * * * *").split(" ")
+ if pk is None:
+ sch = CrontabSchedule.objects.create(
+ minute=minute,
+ hour=hour,
+ day_of_month=day_of_month,
+ month_of_year=month_of_year,
+ day_of_week=day_of_week
+ )
+ task = models.CronPeriodicTask.objects.create(
+ crontab=sch,
+ cron=request.POST.get("cron"),
+ name=request.POST.get("name"),
+ task="control.celery_tasks.tasks.check_today_testplan",
+ enabled=request.POST.get("enabled") == 'on',
+ # TODO(harhipova): uncomment when implemented tasks with arguments
+ # args=args,
+ # kwargs=kwargs,
+ )
+ else:
+ task = models.CronPeriodicTask.objects.get(pk=pk)
+ form = forms.PeriodicTaskForm(request.POST, instance=task)
+ CrontabSchedule.objects.filter(id=task.crontab.id).update(
+ minute=minute,
+ hour=hour,
+ day_of_month=day_of_month,
+ month_of_year=month_of_year,
+ day_of_week=day_of_week
+ )
+ form.save()
+ PeriodicTasks.update_changed()
+ return render(
+ request, "control/scheduler.html",
+ {
+ "form": form,
+ "pk": task.id,
+ "TASK_CHOICES": models.TASK_CHOICES
+ }
+ )
+
+
+def delete_scheduler(request, pk):
+ task = models.CronPeriodicTask.objects.get(pk=pk)
+ task.delete()
+ return redirect("schedulers")
diff --git a/testrail_bot/docker-compose.yml b/testrail_bot/docker-compose.yml
index 27a12a8..559b9a3 100644
--- a/testrail_bot/docker-compose.yml
+++ b/testrail_bot/docker-compose.yml
@@ -18,7 +18,11 @@
- tr_bot
worker:
build: .
- command: celery -A testrail_bot worker --concurrency=4 --autoscale=4,12 --loglevel=INFO
+ command: celery -A testrail_bot worker
+ --concurrency=4
+ --autoscale=4,12
+ --beat --scheduler django
+ --loglevel=INFO
volumes:
- .:/testrail_bot
- media_volume:/mediafiles
diff --git a/testrail_bot/requirements.txt b/testrail_bot/requirements.txt
index 3518d14..6a657a5 100644
--- a/testrail_bot/requirements.txt
+++ b/testrail_bot/requirements.txt
@@ -13,3 +13,4 @@
python-jenkins==1.7.0
matplotlib==3.3.2
jira[cli]==3.5.2
+django-celery-beat==2.5.0
diff --git a/testrail_bot/testrail_bot/settings.py b/testrail_bot/testrail_bot/settings.py
index bc49975..1b7f1ef 100644
--- a/testrail_bot/testrail_bot/settings.py
+++ b/testrail_bot/testrail_bot/settings.py
@@ -39,6 +39,7 @@
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
+ 'django_celery_beat',
]
MIDDLEWARE = [