[tcp-qa] add option to update salt to create-mcp-env

The "Deploy Openstack" pipeline now supports updating saltstack before
deploying components. This patch adds this option to the "tcp-qa" jobs
stack.

Closes-Bug: PROD-35376
Change-Id: I4b45d6f4a1997d05629f931eee5bb4e77780de41
diff --git a/jobs/pipelines/deploy-cicd-and-run-tests.groovy b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
index 56a2d9b..c10f76f 100644
--- a/jobs/pipelines/deploy-cicd-and-run-tests.groovy
+++ b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
@@ -6,6 +6,7 @@
 def env_manager = env.ENV_MANAGER ?: 'devops'
 def batch_size = env.BATCH_SIZE ?: ''
 def dist_upgrade_nodes = "${env.DIST_UPGRADE_NODES}" != "false" ? true : false
+def upgrade_saltstack = "${env.UPGRADE_SALTSTACK}" != "false" ? true : false
 
 if (env_manager == 'devops') {
     jenkins_slave_node_name = "${NODE_NAME}"
@@ -19,7 +20,7 @@
 
 currentBuild.description = "${NODE_NAME}:${ENV_NAME}<br>"
 
-def deploy(shared, common, steps, env_manager, batch_size, dist_upgrade_nodes) {
+def deploy(shared, common, steps, env_manager, batch_size, dist_upgrade_nodes, upgrade_saltstack) {
     def report_text = ''
     try {
 
@@ -50,7 +51,7 @@
         stage("Install core infrastructure and deploy CICD nodes") {
         if (env.DRIVETRAIN_STACK_INSTALL) {
                 // steps: env.DRIVETRAIN_STACK_INSTALL
-                shared.swarm_deploy_cicd(env.DRIVETRAIN_STACK_INSTALL, env.DRIVETRAIN_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size, dist_upgrade_nodes)
+                shared.swarm_deploy_cicd(env.DRIVETRAIN_STACK_INSTALL, env.DRIVETRAIN_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size, dist_upgrade_nodes, upgrade_satlstack)
             } else {
                 common.printMsg("DRIVETRAIN_STACK_INSTALL is empty, skipping 'swarm-deploy-cicd' job", "green")
             }
@@ -59,7 +60,7 @@
         stage("Deploy platform components") {
             if (env.PLATFORM_STACK_INSTALL) {
                 // steps: env.PLATFORM_STACK_INSTALL
-                shared.swarm_deploy_platform(env.PLATFORM_STACK_INSTALL, env.PLATFORM_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size, dist_upgrade_nodes)
+                shared.swarm_deploy_platform(env.PLATFORM_STACK_INSTALL, env.PLATFORM_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size, dist_upgrade_nodes, upgrade_saltstack)
             } else {
                 common.printMsg("PLATFORM_STACK_INSTALL is empty, skipping 'swarm-deploy-platform' job", "green")
             }
@@ -128,7 +129,7 @@
     try {
 
         // run deploy stages
-        deploy(shared, common, steps, env_manager, batch_size, dist_upgrade_nodes)
+        deploy(shared, common, steps, env_manager, batch_size, dist_upgrade_nodes, upgrade_saltstack)
         // run test stages
         test(shared, common, steps, env_manager)
     } catch (e) {
diff --git a/jobs/templates/2019.2.0-heat-cicd-pike-dvr-sl.yml b/jobs/templates/2019.2.0-heat-cicd-pike-dvr-sl.yml
index d32fc93..18d9d25 100644
--- a/jobs/templates/2019.2.0-heat-cicd-pike-dvr-sl.yml
+++ b/jobs/templates/2019.2.0-heat-cicd-pike-dvr-sl.yml
@@ -197,6 +197,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/2019.2.0-heat-cicd-queens-contrail41-sl.yml b/jobs/templates/2019.2.0-heat-cicd-queens-contrail41-sl.yml
index 8e04a6b..145feb7 100644
--- a/jobs/templates/2019.2.0-heat-cicd-queens-contrail41-sl.yml
+++ b/jobs/templates/2019.2.0-heat-cicd-queens-contrail41-sl.yml
@@ -202,6 +202,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/2019.2.0-heat-cicd-queens-dvr-sl.yml b/jobs/templates/2019.2.0-heat-cicd-queens-dvr-sl.yml
index e092203..2374cb4 100644
--- a/jobs/templates/2019.2.0-heat-cicd-queens-dvr-sl.yml
+++ b/jobs/templates/2019.2.0-heat-cicd-queens-dvr-sl.yml
@@ -202,6 +202,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/bm-cicd-pike-ovs-maas.yml b/jobs/templates/bm-cicd-pike-ovs-maas.yml
index 6b7c1db..d3bf097 100644
--- a/jobs/templates/bm-cicd-pike-ovs-maas.yml
+++ b/jobs/templates/bm-cicd-pike-ovs-maas.yml
@@ -196,6 +196,10 @@
         default: true
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: true
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/bm-cicd-queens-ovs-maas.yml b/jobs/templates/bm-cicd-queens-ovs-maas.yml
index 6166a66..45f379d 100644
--- a/jobs/templates/bm-cicd-queens-ovs-maas.yml
+++ b/jobs/templates/bm-cicd-queens-ovs-maas.yml
@@ -208,6 +208,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml b/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
index af9f64b..e63862a 100644
--- a/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
+++ b/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
@@ -221,6 +221,10 @@
         default: true
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: true
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml b/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
index 87562d8..6c14a8f 100644
--- a/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
+++ b/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
@@ -222,6 +222,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-cicd-pike-dvr-sl.yml b/jobs/templates/heat-cicd-pike-dvr-sl.yml
index 5d02cdf..3afa221 100644
--- a/jobs/templates/heat-cicd-pike-dvr-sl.yml
+++ b/jobs/templates/heat-cicd-pike-dvr-sl.yml
@@ -202,6 +202,10 @@
         default: true
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: true
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-cicd-queens-contrail41-sl.yml b/jobs/templates/heat-cicd-queens-contrail41-sl.yml
index 9aa9c40..7c2f959 100644
--- a/jobs/templates/heat-cicd-queens-contrail41-sl.yml
+++ b/jobs/templates/heat-cicd-queens-contrail41-sl.yml
@@ -200,6 +200,10 @@
         default: true
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: true
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-cicd-queens-dvr-sl.yml b/jobs/templates/heat-cicd-queens-dvr-sl.yml
index 44c352e..b0afc13 100644
--- a/jobs/templates/heat-cicd-queens-dvr-sl.yml
+++ b/jobs/templates/heat-cicd-queens-dvr-sl.yml
@@ -201,6 +201,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml b/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml
index 7e947a6..7fd3f3b 100644
--- a/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml
+++ b/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml
@@ -196,6 +196,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-bm-pike-ovs.yml b/jobs/templates/released-bm-pike-ovs.yml
index feab620..51e7ecd 100644
--- a/jobs/templates/released-bm-pike-ovs.yml
+++ b/jobs/templates/released-bm-pike-ovs.yml
@@ -185,6 +185,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml b/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
index f0a342a..e3caf3b 100644
--- a/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
+++ b/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
@@ -212,6 +212,10 @@
         default: true
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: true
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-heat-cicd-pike-dvr-sl.yml b/jobs/templates/released-heat-cicd-pike-dvr-sl.yml
index 9c5f6aa..a7db0c8 100644
--- a/jobs/templates/released-heat-cicd-pike-dvr-sl.yml
+++ b/jobs/templates/released-heat-cicd-pike-dvr-sl.yml
@@ -211,6 +211,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-heat-cicd-queens-contrail41-sl.yml b/jobs/templates/released-heat-cicd-queens-contrail41-sl.yml
index 4e394a9..41e57d3 100644
--- a/jobs/templates/released-heat-cicd-queens-contrail41-sl.yml
+++ b/jobs/templates/released-heat-cicd-queens-contrail41-sl.yml
@@ -214,6 +214,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-heat-cicd-queens-dvr-sl.yml b/jobs/templates/released-heat-cicd-queens-dvr-sl.yml
index b39a716..eab85d8 100644
--- a/jobs/templates/released-heat-cicd-queens-dvr-sl.yml
+++ b/jobs/templates/released-heat-cicd-queens-dvr-sl.yml
@@ -211,6 +211,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/swarm-deploy-cicd.yml b/jobs/templates/swarm-deploy-cicd.yml
index 65e77fe..e999b96 100644
--- a/jobs/templates/swarm-deploy-cicd.yml
+++ b/jobs/templates/swarm-deploy-cicd.yml
@@ -56,6 +56,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/swarm-deploy-platform.yml b/jobs/templates/swarm-deploy-platform.yml
index 1405ac4..7971c41 100644
--- a/jobs/templates/swarm-deploy-platform.yml
+++ b/jobs/templates/swarm-deploy-platform.yml
@@ -57,6 +57,10 @@
         default: false
         description: Whether to perform dist-upgrade on virtual nodes during deployment
         name: DIST_UPGRADE_NODES
+    - bool:
+        default: false
+        description: 'Works starting from MCP 2019.2.10 or master. Whether to apply saltstack updates on all nodes in cluster before deployment'
+        name: UPGRADE_SALTSTACK
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index d6f4ed4..020952b 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -314,7 +314,7 @@
         build_pipeline_job('swarm-bootstrap-salt-cluster-heat', parameters)
 }
 
-def swarm_deploy_cicd(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, String batch_size, Boolean dist_upgrade_nodes) {
+def swarm_deploy_cicd(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, String batch_size, Boolean dist_upgrade_nodes, Boolean upgrade_saltstack) {
         // Run openstack_deploy job on cfg01 Jenkins for specified stacks
         def common = new com.mirantis.mk.Common()
         def tcp_qa_refs = env.TCP_QA_REFS ?: ''
@@ -327,6 +327,7 @@
                 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
                 string(name: 'BATCH_SIZE', value: batch_size),
                 booleanParam(name: 'DIST_UPGRADE_NODES', value: dist_upgrade_nodes),
+                booleanParam(name: 'UPGRADE_SALTSTACK', value: upgrade_saltstack),
                 booleanParam(name: 'MAKE_SNAPSHOT_STAGES', value: make_snapshot_stages),
                 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
             ]
@@ -346,6 +347,7 @@
                 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
                 string(name: 'BATCH_SIZE', value: batch_size),
                 booleanParam(name: 'DIST_UPGRADE_NODES', value: dist_upgrade_nodes),
+                booleanParam(name: 'UPGRADE_SALTSTACK', value: upgrade_saltstack),
                 booleanParam(name: 'MAKE_SNAPSHOT_STAGES', value: make_snapshot_stages),
                 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
             ]
@@ -521,7 +523,8 @@
                 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
                 \\\"BATCH_SIZE\\\": \\\"\${BATCH_SIZE}\\\",
                 \\\"STACK_INSTALL\\\": \\\"${stack}\\\",
-                \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\"
+                \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\",
+                \\\"UPGRADE_SALTSTACK\\\": \\\"\${UPGRADE_SALTSTACK}\\\"
             }\"
             JOB_PREFIX="[ ${ENV_NAME}/{build_number}:drivetrain {time} ] "
             python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
@@ -556,7 +559,8 @@
                 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
                 \\\"BATCH_SIZE\\\": \\\"\${BATCH_SIZE}\\\",
                 \\\"STACK_INSTALL\\\": \\\"${stack}\\\",
-                \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\"
+                \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\",
+                \\\"UPGRADE_SALTSTACK\\\": \\\"\${UPGRADE_SALTSTACK}\\\"
             }\"
             JOB_PREFIX="[ ${ENV_NAME}/{build_number}:platform {time} ] "
             python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
diff --git a/tcp_tests/settings.py b/tcp_tests/settings.py
index d3a0aa3..5e60b86 100644
--- a/tcp_tests/settings.py
+++ b/tcp_tests/settings.py
@@ -101,6 +101,7 @@
 STACK_INSTALL = os.environ.get('STACK_INSTALL', None)
 BATCH_SIZE = os.environ.get('BATCH_SIZE', None)
 DIST_UPGRADE_NODES = os.environ.get('DIST_UPGRADE_NODES', False)
+UPGRADE_SALTSTACK = os.environ.get('UPGRADE_SALTSTACK', False)
 SKIP_SYNC_TIME = get_var_as_bool("SKIP_SYNC_TIME", False)
 
 # OpenStack parameters to work with Heat stacks
diff --git a/tcp_tests/tests/system/test_mcp_update.py b/tcp_tests/tests/system/test_mcp_update.py
index 2fa1bfa..c906eb1 100644
--- a/tcp_tests/tests/system/test_mcp_update.py
+++ b/tcp_tests/tests/system/test_mcp_update.py
@@ -542,6 +542,7 @@
         job_parameters = {
             "TARGET_SERVERS": target,
             "OS_DIST_UPGRADE": True,
+            "UPGRADE_SALTSTACK": False,
             "OS_UPGRADE": True,
             "INTERACTIVE": False}
         upgrade_control_pipeline = drivetrain_actions.start_job_on_jenkins(
diff --git a/tcp_tests/tests/system/test_upgrade_pike_queens.py b/tcp_tests/tests/system/test_upgrade_pike_queens.py
index c4c9119..3ea2d23 100644
--- a/tcp_tests/tests/system/test_upgrade_pike_queens.py
+++ b/tcp_tests/tests/system/test_upgrade_pike_queens.py
@@ -194,7 +194,8 @@
         job_name = 'deploy-upgrade-control'
         job_parameters = {
             'INTERACTIVE': False,
-            'OS_DIST_UPGRADE': True,
+            'OS_DIST_UPGRADE': False,
+            'UPGRADE_SALTSTACK': False,
             'OS_UPGRADE': True
         }
         # ####### Run job for ctl* ###