Add sleep after drivetrain update to wait salt completed all jobs
PROD-36385
Change-Id: I9b28e8a35abb666fe5180274e5c3136b4e8bfb1a
diff --git a/tcp_tests/managers/saltmanager.py b/tcp_tests/managers/saltmanager.py
index 1f63ae9..fbbb9b8 100644
--- a/tcp_tests/managers/saltmanager.py
+++ b/tcp_tests/managers/saltmanager.py
@@ -16,6 +16,7 @@
import pkg_resources
from collections import defaultdict
+from polling import poll
from datetime import datetime
from pepper import libpepper
@@ -116,6 +117,10 @@
def local_async(self, tgt, fun, args=None, kwargs=None, timeout=None):
return self.api.local_async(tgt, fun, args, kwargs, timeout=timeout)
+ def runner(self, fun, args=None, **kwargs):
+ response = self.api.runner(fun, arg=args, **kwargs)
+ return response.get("return")
+
def lookup_result(self, jid):
return self.api.lookup_jid(jid)
@@ -194,6 +199,20 @@
result = self.local(tgt=tgt, fun='grains.get', args=grains)
return result['return']
+ def wait_jobs_completed(self, timeout=60, interval=5):
+ """
+ :param timeout: int, time seconds to wait
+ :param interval: int, time in second between attempts
+ :return: None
+ """
+ # TODO(harhipova) PROD-36434 : need to check that last job completed
+ # with successful result
+ poll(
+ target=lambda: self.runner('jobs.active')[0] == {},
+ timeout=timeout,
+ step=interval
+ )
+
def get_ssh_data(self):
"""Generate ssh config for Underlay
diff --git a/tcp_tests/requirements.txt b/tcp_tests/requirements.txt
index 462fa77..194b275 100644
--- a/tcp_tests/requirements.txt
+++ b/tcp_tests/requirements.txt
@@ -23,6 +23,8 @@
python-jenkins
cmd2<0.9
PyYAML!=5.1
+polling==0.3.2
+
# For Queens: https://github.com/openstack/requirements/blob/stable/queens/global-requirements.txt
python-heatclient>=1.10.0
diff --git a/tcp_tests/tests/system/test_mcp_update.py b/tcp_tests/tests/system/test_mcp_update.py
index c1456be..45e3cdd 100644
--- a/tcp_tests/tests/system/test_mcp_update.py
+++ b/tcp_tests/tests/system/test_mcp_update.py
@@ -188,6 +188,7 @@
2. Run job git-mirror-downstream-mk-pipelines
3. Run job git-mirror-downstream-pipeline-library
4. If jobs are passed then start 'Deploy - upgrade MCP Drivetrain'
+ 5. Wait till salt-master completes all async jobs
Duration: ~70 min
"""
@@ -248,6 +249,9 @@
build_timeout=4 * 60 * 60)
assert job_result == 'SUCCESS', job_description
+ # ############### Wait till salt-master completes all async jobs #####
+ show_step(5)
+ salt.wait_jobs_completed(timeout=20*60)
@pytest.mark.grab_versions
@pytest.mark.parametrize("_", [settings.ENV_NAME])