Extend the process of updgrading from Pike to Queens

Related-Prod: #PROD-00000

Change-Id: I8e9e5dc943a0206f351830f89777e080afd513d0
diff --git a/tcp_tests/tests/system/test_mcp_update.py b/tcp_tests/tests/system/test_mcp_update.py
index 01fa7bb..ab38698 100644
--- a/tcp_tests/tests/system/test_mcp_update.py
+++ b/tcp_tests/tests/system/test_mcp_update.py
@@ -91,6 +91,7 @@
     reclass.add_key(repo_param, proposed_repo, "cluster/*/ceph/init.yml")
 
     salt_actions.run_state("*", "saltutil.refresh_pillar")
+    salt_actions.enforce_state("*", "salt.minion")
     salt_actions.enforce_state("I@jenkins:client", "jenkins.client")
 
 
diff --git a/tcp_tests/tests/system/test_upgrade_pike_queens.py b/tcp_tests/tests/system/test_upgrade_pike_queens.py
index 4080646..9d3288e 100644
--- a/tcp_tests/tests/system/test_upgrade_pike_queens.py
+++ b/tcp_tests/tests/system/test_upgrade_pike_queens.py
@@ -25,18 +25,21 @@
     Created by https://mirantis.jira.com/browse/PROD-32683
     """
     def execute_pre_post_steps(self, underlay_actions,
-                               cfg_node, verbose, type):
+                               cfg_node, verbose, type, with_verify=False):
 
         # ### Get the list of all upgradable OpenStack components ############
         ret = underlay_actions.check_call(
             node_name=cfg_node, verbose=verbose,
             cmd="salt 'cfg01*' config.get"
-                " orchestration:upgrade:applications --out=json")
-        cfg_nodes_list = json.loads(ret['stdout_str'])
-        services_for_upgrade = []
-        for i in cfg_nodes_list:
-            for j in cfg_nodes_list[i]:
-                services_for_upgrade.append(j)
+                " orchestration:upgrade:applications --out=json")['stdout_str']
+        # It returns json like {"local": {"galera":{"priority":910},"heat":{..
+        all_formulas = json.loads(ret).get("local")
+        sorted_formulas = sorted(all_formulas.items(),
+                                 key=lambda x: x[1].get("priority"),
+                                 reverse=True)
+        # Because 'sorted' function returns list of tuples like
+        # [('aodh', {'priority': 2000}), ('ceilometer', {'priority': 1950}) ...
+        services_for_upgrade = [item[0] for item in sorted_formulas]
         LOG.info(services_for_upgrade)
 
         # ###### Get the list of all target node #############################
@@ -60,11 +63,22 @@
                 node_applications = node_app_output[node][need_output]
                 LOG.info(node_applications)
                 for service in services_for_upgrade:
-                    if service in node_applications:
-                        underlay_actions.check_call(
-                            node_name=cfg_node, verbose=verbose,
-                            cmd="salt {} state.apply "
-                                "{}.upgrade.{}".format(node, service, type))
+                    if service not in node_applications:
+                        continue
+                    cmd = "salt {} state.apply {}.upgrade.{}".\
+                        format(node, service, type)
+                    LOG.info("Apply: {}".format(cmd))
+                    underlay_actions.check_call(
+                        node_name=cfg_node, verbose=verbose, cmd=cmd)
+
+                    # Run upgrade.verify if needed
+                    if not with_verify:
+                        continue
+                    cmd = "salt {} state.apply {}.upgrade.verify".\
+                        format(node, service)
+                    LOG.info("Apply: {}".format(cmd))
+                    underlay_actions.check_call(
+                        node_name=cfg_node, verbose=verbose, cmd=cmd)
 
     @pytest.mark.day1_underlay
     def test_upgrade_pike_queens(self,
@@ -91,8 +105,18 @@
         infra_init_yaml = "cluster/*/infra/init.yml"
         # ########## Perform the pre-upgrade activities ##########
         show_step(1)
+        salt_actions.enforce_state("I@keystone:client:os_client_config",
+                                   "keystone.client.os_client_config")
+        # #### guarantee that the KeystoneRC metadata is exported to mine ####
+        underlay_actions.check_call(
+            node_name=cfg_node, verbose=verbose,
+            cmd="salt -C 'I@keystone:client:enabled' state.sls"
+                " keystone.upgrade.pre")
+        # ### Run upgrade.pre and upgrade.verify
+        self.execute_pre_post_steps(underlay_actions, cfg_node,
+                                    verbose, 'pre', with_verify=True)
         LOG.info('Add parameters to {}'.format(infra_init_yaml))
-        # ### Edit Infra INIT
+        # ### Edit Infra INIT #####
         reclass_actions.add_bool_key(
             'parameters._param.openstack_upgrade_enabled',
             'true',
@@ -107,16 +131,18 @@
             'parameters._param.openstack_old_version',
             'pike',
             infra_init_yaml)
-        # ### Edit Openstack INIT
-        reclass_actions.add_key(
-            'parameters._param.gnocchi_version',
-            4.2,
-            infra_init_yaml)
-        reclass_actions.add_key(
-            'parameters._param.gnocchi_old_version',
-            4.0,
-            infra_init_yaml)
-        # ### Edit Openstack control
+
+        # ### Edit Openstack INIT #####
+        openstack_init_yaml = "cluster/*/openstack/init.yml"
+        LOG.info('Add parameters to {}'.format(openstack_init_yaml))
+        reclass_actions.add_key('parameters._param.gnocchi_version',
+                                4.2,
+                                openstack_init_yaml)
+        reclass_actions.add_key('parameters._param.gnocchi_old_version',
+                                4.0,
+                                openstack_init_yaml)
+
+        # ### Edit Openstack control #####
         reclass_actions.add_class(
             'system.keystone.client.v3',
             'cluster/*/openstack/control_init.yml'
@@ -125,15 +151,10 @@
             node_name=cfg_node, verbose=verbose,
             cmd="cd /srv/salt/reclass; git add -u && "
                 "git commit --allow-empty -m 'Cluster model update'")
+
+        # ### Apply state to enable changes in reclass
         LOG.info('Perform refresh_pillar')
         salt_actions.run_state("*", "saltutil.refresh_pillar")
-        salt_actions.enforce_state("I@keystone:client:os_client_config",
-                                   "keystone.client.os_client_config")
-        # #### guarantee that the KeystoneRC metadata is exported to mine ####
-        underlay_actions.check_call(
-            node_name=cfg_node, verbose=verbose,
-            cmd="salt -C 'I@keystone:client:enabled' state.sls"
-                " keystone.upgrade.pre")
 
         self.execute_pre_post_steps(underlay_actions, cfg_node,
                                     verbose, 'pre')
@@ -159,6 +180,7 @@
                                    "jenkins.client")
         # #### Add QUEENS's repos
         salt_actions.enforce_state("*", "linux.system.repo")
+        salt_actions.enforce_state("*", "salt.minion")
 
         # ########## Upgrade control nodes  ###########
         LOG.info('Upgrade control VMs')
@@ -196,7 +218,7 @@
             job_parameters=job_parameters)
         assert update_control_vms == 'SUCCESS'
 
-        # ########## Upgrade gatewey nodes  ###########
+        # ########## Upgrade gateway nodes  ###########
         show_step(3)
         LOG.info('Upgrade gateway')
         job_name = 'deploy-upgrade-ovs-gateway'