Add support of DIST_UPGRADE_NODES parameter

Add support of DIST_UPGRADE_NODES option for
deploy_openstack job, so that virtual nodes get
latest upgrades during environment deployment
(if the option is enabled).

Change-Id: Idc0b6a12005a31e676e3380ca23ee486ab5adbdd
Related-Prod:PROD-34252
diff --git a/jobs/pipelines/deploy-cicd-and-run-tests.groovy b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
index 27764a7..8240884 100644
--- a/jobs/pipelines/deploy-cicd-and-run-tests.groovy
+++ b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
@@ -5,6 +5,7 @@
 def steps = "hardware,create_model,salt," + env.DRIVETRAIN_STACK_INSTALL + "," + env.PLATFORM_STACK_INSTALL
 def env_manager = env.ENV_MANAGER ?: 'devops'
 def batch_size = env.BATCH_SIZE ?: ''
+def dist_upgrade_nodes = env.DIST_UPGRADE_NODES ?: false
 
 if (env_manager == 'devops') {
     jenkins_slave_node_name = "${NODE_NAME}"
@@ -18,7 +19,7 @@
 
 currentBuild.description = "${NODE_NAME}:${ENV_NAME}<br>"
 
-def deploy(shared, common, steps, env_manager, batch_size) {
+def deploy(shared, common, steps, env_manager, batch_size, dist_upgrade_nodes) {
     def report_text = ''
     try {
 
@@ -49,7 +50,7 @@
         stage("Install core infrastructure and deploy CICD nodes") {
         if (env.DRIVETRAIN_STACK_INSTALL) {
                 // steps: env.DRIVETRAIN_STACK_INSTALL
-                shared.swarm_deploy_cicd(env.DRIVETRAIN_STACK_INSTALL, env.DRIVETRAIN_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size)
+                shared.swarm_deploy_cicd(env.DRIVETRAIN_STACK_INSTALL, env.DRIVETRAIN_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size, dist_upgrade_nodes)
             } else {
                 common.printMsg("DRIVETRAIN_STACK_INSTALL is empty, skipping 'swarm-deploy-cicd' job", "green")
             }
@@ -58,7 +59,7 @@
         stage("Deploy platform components") {
             if (env.PLATFORM_STACK_INSTALL) {
                 // steps: env.PLATFORM_STACK_INSTALL
-                shared.swarm_deploy_platform(env.PLATFORM_STACK_INSTALL, env.PLATFORM_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size)
+                shared.swarm_deploy_platform(env.PLATFORM_STACK_INSTALL, env.PLATFORM_STACK_INSTALL_TIMEOUT, jenkins_slave_node_name, make_snapshot_stages, batch_size, dist_upgrade_nodes)
             } else {
                 common.printMsg("PLATFORM_STACK_INSTALL is empty, skipping 'swarm-deploy-platform' job", "green")
             }
@@ -125,7 +126,7 @@
   node ("${NODE_NAME}") {
     try {
         // run deploy stages
-        deploy(shared, common, steps, env_manager, batch_size)
+        deploy(shared, common, steps, env_manager, batch_size, dist_upgrade_nodes)
         // run test stages
         test(shared, common, steps, env_manager)
     } catch (e) {
diff --git a/jobs/templates/bm-cicd-pike-ovs-maas.yml b/jobs/templates/bm-cicd-pike-ovs-maas.yml
index 13a0a20..4f95a72 100644
--- a/jobs/templates/bm-cicd-pike-ovs-maas.yml
+++ b/jobs/templates/bm-cicd-pike-ovs-maas.yml
@@ -192,6 +192,10 @@
         description: ''
         name: BATCH_SIZE
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/bm-cicd-queens-ovs-maas.yml b/jobs/templates/bm-cicd-queens-ovs-maas.yml
index 9f35ee7..6166a66 100644
--- a/jobs/templates/bm-cicd-queens-ovs-maas.yml
+++ b/jobs/templates/bm-cicd-queens-ovs-maas.yml
@@ -204,6 +204,10 @@
         description: ''
         name: BATCH_SIZE
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml b/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
index 69097ca..d4332b2 100644
--- a/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
+++ b/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
@@ -217,6 +217,10 @@
         description: ''
         name: BATCH_SIZE
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml b/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
index 912707e..87562d8 100644
--- a/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
+++ b/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
@@ -218,6 +218,10 @@
         description: ''
         name: BATCH_SIZE
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-cicd-pike-dvr-sl.yml b/jobs/templates/heat-cicd-pike-dvr-sl.yml
index d5edd30..d86493e 100644
--- a/jobs/templates/heat-cicd-pike-dvr-sl.yml
+++ b/jobs/templates/heat-cicd-pike-dvr-sl.yml
@@ -197,6 +197,10 @@
         description: ''
         name: UPDATE_VERSION
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-cicd-queens-contrail41-sl.yml b/jobs/templates/heat-cicd-queens-contrail41-sl.yml
index 5f6e6cc..b09c639 100644
--- a/jobs/templates/heat-cicd-queens-contrail41-sl.yml
+++ b/jobs/templates/heat-cicd-queens-contrail41-sl.yml
@@ -195,6 +195,10 @@
         description: Exclude tungsten tests by default
         name: TEMPEST_PATTERN
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/heat-cicd-queens-dvr-sl.yml b/jobs/templates/heat-cicd-queens-dvr-sl.yml
index c515cf6..5d2df7c 100644
--- a/jobs/templates/heat-cicd-queens-dvr-sl.yml
+++ b/jobs/templates/heat-cicd-queens-dvr-sl.yml
@@ -196,6 +196,10 @@
         description: ''
         name: TEMPEST_EXTRA_ARGS
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml b/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml
index b0cfe1b..eb15717 100644
--- a/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml
+++ b/jobs/templates/maintenance-heat-cicd-pike-dvr-sl.yml
@@ -192,6 +192,10 @@
         description: ''
         name: UPDATE_VERSION
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-bm-pike-ovs.yml b/jobs/templates/released-bm-pike-ovs.yml
index a88b8ef..dbb8f1d 100644
--- a/jobs/templates/released-bm-pike-ovs.yml
+++ b/jobs/templates/released-bm-pike-ovs.yml
@@ -176,6 +176,15 @@
     - password:
         name: IPMI_PASS
         default: 'KBwdcRqwed3w2'
+    - string:
+        default: ''
+        description: ''
+        name: BATCH_SIZE
+        trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml b/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
index 37ba618..04de084 100644
--- a/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
+++ b/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
@@ -208,6 +208,10 @@
         description: Exclude tungsten tests by default
         name: TEMPEST_PATTERN
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-heat-cicd-pike-dvr-sl.yml b/jobs/templates/released-heat-cicd-pike-dvr-sl.yml
index a690f3d..1d20f18 100644
--- a/jobs/templates/released-heat-cicd-pike-dvr-sl.yml
+++ b/jobs/templates/released-heat-cicd-pike-dvr-sl.yml
@@ -207,6 +207,10 @@
         description: 'for formula repo http://mirror.mirantis.com/update/UPDATE_VERSION/salt-formulas/xenial'
         name: UPDATE_VERSION
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/jobs/templates/released-heat-cicd-queens-dvr-sl.yml b/jobs/templates/released-heat-cicd-queens-dvr-sl.yml
index 61bba24..e54db88 100644
--- a/jobs/templates/released-heat-cicd-queens-dvr-sl.yml
+++ b/jobs/templates/released-heat-cicd-queens-dvr-sl.yml
@@ -207,6 +207,10 @@
         description: 'for formula repo http://mirror.mirantis.com/update/UPDATE_VERSION/salt-formulas/xenial'
         name: UPDATE_VERSION
         trim: 'false'
+    - bool:
+        default: false
+        description: Whether to perform dist-upgrade on virtual nodes during deployment
+        name: DIST_UPGRADE_NODES
     pipeline-scm:
       lightweight-checkout: false
       scm:
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index ce5e7f2..b568702 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -314,7 +314,7 @@
         build_pipeline_job('swarm-bootstrap-salt-cluster-heat', parameters)
 }
 
-def swarm_deploy_cicd(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, batch_size) {
+def swarm_deploy_cicd(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, String batch_size, Boolean dist_upgrade_nodes) {
         // Run openstack_deploy job on cfg01 Jenkins for specified stacks
         def common = new com.mirantis.mk.Common()
         def tcp_qa_refs = env.TCP_QA_REFS ?: ''
@@ -326,13 +326,14 @@
                 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
                 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
                 string(name: 'BATCH_SIZE', value: batch_size),
+                booleanParam(name: 'DIST_UPGRADE_NODES', value: dist_upgrade_nodes),
                 booleanParam(name: 'MAKE_SNAPSHOT_STAGES', value: make_snapshot_stages),
                 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
             ]
         build_pipeline_job('swarm-deploy-cicd', parameters)
 }
 
-def swarm_deploy_platform(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, batch_size) {
+def swarm_deploy_platform(String stack_to_install, String install_timeout, String jenkins_slave_node_name, Boolean make_snapshot_stages, String batch_size, Boolean dist_upgrade_nodes) {
         // Run openstack_deploy job on CICD Jenkins for specified stacks
         def common = new com.mirantis.mk.Common()
         def tcp_qa_refs = env.TCP_QA_REFS ?: ''
@@ -344,6 +345,7 @@
                 string(name: 'STACK_INSTALL_TIMEOUT', value: install_timeout),
                 string(name: 'TCP_QA_REFS', value: "${tcp_qa_refs}"),
                 string(name: 'BATCH_SIZE', value: batch_size),
+                booleanParam(name: 'DIST_UPGRADE_NODES', value: dist_upgrade_nodes),
                 booleanParam(name: 'MAKE_SNAPSHOT_STAGES', value: make_snapshot_stages),
                 booleanParam(name: 'SHUTDOWN_ENV_ON_TEARDOWN', value: false),
             ]
@@ -518,7 +520,8 @@
             JOB_PARAMETERS=\"{
                 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
                 \\\"BATCH_SIZE\\\": \\\"\${BATCH_SIZE}\\\",
-                \\\"STACK_INSTALL\\\": \\\"${stack}\\\"
+                \\\"STACK_INSTALL\\\": \\\"${stack}\\\",
+                \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\"
             }\"
             JOB_PREFIX="[ ${ENV_NAME}/{build_number}:drivetrain {time} ] "
             python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
@@ -552,7 +555,8 @@
             JOB_PARAMETERS=\"{
                 \\\"SALT_MASTER_URL\\\": \\\"\${SALTAPI_URL}\\\",
                 \\\"BATCH_SIZE\\\": \\\"\${BATCH_SIZE}\\\",
-                \\\"STACK_INSTALL\\\": \\\"${stack}\\\"
+                \\\"STACK_INSTALL\\\": \\\"${stack}\\\",
+                \\\"DIST_UPGRADE_NODES\\\": \\\"\${DIST_UPGRADE_NODES}\\\"
             }\"
             JOB_PREFIX="[ ${ENV_NAME}/{build_number}:platform {time} ] "
             python ./tcp_tests/utils/run_jenkins_job.py --verbose --job-name=deploy_openstack --job-parameters="\$JOB_PARAMETERS" --job-output-prefix="\$JOB_PREFIX"
diff --git a/tcp_tests/settings.py b/tcp_tests/settings.py
index 59b2bd0..d3a0aa3 100644
--- a/tcp_tests/settings.py
+++ b/tcp_tests/settings.py
@@ -100,6 +100,7 @@
 
 STACK_INSTALL = os.environ.get('STACK_INSTALL', None)
 BATCH_SIZE = os.environ.get('BATCH_SIZE', None)
+DIST_UPGRADE_NODES = os.environ.get('DIST_UPGRADE_NODES', False)
 SKIP_SYNC_TIME = get_var_as_bool("SKIP_SYNC_TIME", False)
 
 # OpenStack parameters to work with Heat stacks