Merge "PROD-36999 Remove redundant jobs"
diff --git a/jobs/global.yaml b/jobs/global.yaml
index 32baf23..0cecf04 100755
--- a/jobs/global.yaml
+++ b/jobs/global.yaml
@@ -12,6 +12,6 @@
       </font></h1>
     current-version: 2019.2.21
     previous-version: 2019.2.20
-    disabled-proposed: false
-    disabled-2019-2-0: true
-    upgrade-to-tag: false
+    disabled-proposed: true
+    disabled-2019-2-0: false
+    upgrade-to-tag: true
diff --git a/tcp_tests/requirements.txt b/tcp_tests/requirements.txt
index 43c9f0c..6686d70 100644
--- a/tcp_tests/requirements.txt
+++ b/tcp_tests/requirements.txt
@@ -23,6 +23,7 @@
 cmd2<0.9
 PyYAML!=5.1
 polling==0.3.2
+retrying
 
 
 # For Queens: https://github.com/openstack/requirements/blob/stable/queens/global-requirements.txt
diff --git a/tcp_tests/tests/system/test_mcp_update.py b/tcp_tests/tests/system/test_mcp_update.py
index 15a0586..747ceda 100644
--- a/tcp_tests/tests/system/test_mcp_update.py
+++ b/tcp_tests/tests/system/test_mcp_update.py
@@ -4,6 +4,7 @@
 
 from tcp_tests import logger
 from tcp_tests import settings
+from retrying import retry
 
 sys.path.append(os.getcwd())
 try:
@@ -735,6 +736,7 @@
         r, errors = salt.enforce_state("I@jenkins:client", "jenkins.client")
         assert errors is None
 
+    @retry(AssertionError, stop_max_attempt_number=2, wait_fixed=5000)
     @pytest.mark.grab_versions
     @pytest.mark.parametrize('target', get_control_plane_targets())
     @pytest.mark.run_mcp_update
@@ -753,6 +755,7 @@
 
         assert job_result == 'SUCCESS', job_description
 
+    @retry(AssertionError, stop_max_attempt_number=2, wait_fixed=5000)
     @pytest.mark.grab_versions
     @pytest.mark.run_mcp_update
     def test__update__data_plane(self, drivetrain_actions, salt_actions):
@@ -770,6 +773,7 @@
 
         assert job_result == 'SUCCESS', job_description
 
+    @retry(AssertionError, stop_max_attempt_number=2, wait_fixed=5000)
     @pytest.mark.grab_versions
     @pytest.mark.run_mcp_update
     def test__update__computes(self, drivetrain_actions):
diff --git a/tcp_tests/tests/system/test_upgrade_pike_queens.py b/tcp_tests/tests/system/test_upgrade_pike_queens.py
index 87616eb..7734c3c 100644
--- a/tcp_tests/tests/system/test_upgrade_pike_queens.py
+++ b/tcp_tests/tests/system/test_upgrade_pike_queens.py
@@ -13,6 +13,7 @@
 #    under the License.
 import pytest
 import json
+from retrying import retry
 
 from tcp_tests import logger
 
@@ -198,12 +199,15 @@
             'UPGRADE_SALTSTACK': False,
             'OS_UPGRADE': True
         }
-        # ####### Run job for ctl* ###
-        job_parameters["TARGET_SERVERS"] = "ctl*"
-        job_result, job_description = dt.start_job_on_jenkins(
-            job_name=job_name,
-            job_parameters=job_parameters)
-        assert job_result == 'SUCCESS', job_description
+
+        @retry(AssertionError, stop_max_attempt_number=2, wait_fixed=5000)
+        def run_deploy_upgrade_control():
+            # ####### Run job for ctl* ###
+            job_parameters["TARGET_SERVERS"] = "ctl*"
+            job_result, job_description = dt.start_job_on_jenkins(
+                job_name=job_name,
+                job_parameters=job_parameters)
+            assert job_result == 'SUCCESS', job_description
 
         if salt_actions.cmd_run("mdb*", "test.ping")[0].keys():
             # ####### Run job for mdb* ###
@@ -231,21 +235,24 @@
         # ########## Upgrade gateway nodes  ###########
         show_step(3)
         LOG.info('Upgrade gateway')
-        if salt_actions.cmd_run("gtw*", "test.ping")[0].keys():
-            job_name = 'deploy-upgrade-ovs-gateway'
-            job_parameters = {
-                'INTERACTIVE': False,
-                'OS_DIST_UPGRADE': True,
-                'OS_UPGRADE': True,
-                'TARGET_SERVERS': "gtw*"
-            }
-            job_result, job_description = dt.start_job_on_jenkins(
-                job_name=job_name,
-                job_parameters=job_parameters)
-            assert job_result == 'SUCCESS', job_description
-        else:
-            LOG.info("This deployment doesn't have gtw* nodes, \
-            so skip this step")
+        job_name = 'deploy-upgrade-ovs-gateway'
+        job_parameters = {
+            'INTERACTIVE': False,
+            'OS_DIST_UPGRADE': True,
+            'OS_UPGRADE': True,
+            'TARGET_SERVERS': "gtw*"
+        }
+
+        @retry(AssertionError, stop_max_attempt_number=2, wait_fixed=5000)
+        def run_deploy_upgrade_ovs_gateway():
+            if salt_actions.cmd_run("gtw*", "test.ping")[0].keys():
+                job_result, job_description = dt.start_job_on_jenkins(
+                    job_name=job_name,
+                    job_parameters=job_parameters)
+                assert job_result == 'SUCCESS', job_description
+            else:
+                LOG.info("This deployment doesn't have gtw* nodes, \
+                so skip this step")
         # ############ Upgrade compute nodes  ############
         show_step(4)
         LOG.info('Upgrade compute nodes')
@@ -256,10 +263,13 @@
             'OS_UPGRADE': True,
             'TARGET_SERVERS': "cmp*"
         }
-        job_result, job_description = dt.start_job_on_jenkins(
-            job_name=job_name,
-            job_parameters=job_parameters)
-        assert job_result == 'SUCCESS', job_description
+
+        @retry(AssertionError, stop_max_attempt_number=2, wait_fixed=5000)
+        def run_deploy_upgrade_compute():
+            job_result, job_description = dt.start_job_on_jenkins(
+                job_name=job_name,
+                job_parameters=job_parameters)
+            assert job_result == 'SUCCESS', job_description
 
         # ############ Perform the post-upgrade activities ##########
         show_step(5)
@@ -279,7 +289,7 @@
                                     verbose, 'post')
         LOG.info('Perform refresh_pillar')
         salt_actions.run_state("*", "saltutil.refresh_pillar")
-        # ######################## Run CPV ##########################
+        # ######################## Run CVP ##########################
         show_step(6)
         job_name = 'cvp-sanity'
         job_parameters = {