Merge "Add pipeline to deploy random released MCP by schedule Set 2019.2.7 versions to deploy in the released-* jobs"
diff --git a/jobs/pipelines/rotation_bm_deployments.groovy b/jobs/pipelines/rotation_bm_deployments.groovy
new file mode 100644
index 0000000..49103a9
--- /dev/null
+++ b/jobs/pipelines/rotation_bm_deployments.groovy
@@ -0,0 +1,113 @@
+/**
+ *
+ * Deploy the product cluster using Jenkins master on CICD cluster
+ *
+ * Expected parameters:
+ *   STACKS                        List of deployments from 'all physical' view in the Jenkins
+                                   They should be separated by newline
+ *   PARENT_NODE_NAME
+ *   OS_AUTH_URL                   OpenStack keystone catalog URL
+ *   OS_PROJECT_NAME               OpenStack project (tenant) name
+ *   OS_CREDENTIALS                OpenStack username and password credentials ID in Jenkins
+
+ *   TCP_QA_REFS                   Reference to the tcp-qa change on Gerrit, like refs/changes/46/418546/41
+ */
+@Library('tcp-qa')_
+
+common = new com.mirantis.mk.Common()
+shared = new com.mirantis.system_qa.SharedPipeline()
+import jenkins.model.*
+
+def stacks = env.STACKS.readLines()
+
+def get_last_build_time(job_name) {
+  def job = Hudson.instance.getJob(job_name)
+  def last_build = Jenkins.instance.getItemByFullName(job.fullName).getLastBuild()
+  if (last_build == null) {
+  	return null
+  }
+  def upStreamBuild = Jenkins.getInstance().getItemByFullName(job.fullName).getBuildByNumber(last_build.getNumber())
+  return upStreamBuild.getTime()
+}
+
+@NonCPS
+def oldest_job(map) {
+  println map.sort({a, b -> a.value <=> b.value})*.value
+  return map.sort({a, b -> a.value <=> b.value})*.key[0]
+}
+
+// ============================================================================
+timeout(time: 15, unit: 'HOURS') {
+  node (env.PARENT_NODE_NAME) {
+    stage("Remove extra stacks") {
+        withCredentials([
+           [$class          : 'UsernamePasswordMultiBinding',
+           credentialsId   : env.OS_CREDENTIALS,
+           passwordVariable: 'OS_PASSWORD',
+           usernameVariable: 'OS_USERNAME']
+        ]) {
+            env.OS_IDENTITY_API_VERSION = 3
+            for (stack_name in stacks) {
+              shared.run_cmd("""
+                # export OS_IDENTITY_API_VERSION=3
+                # export OS_AUTH_URL=${OS_AUTH_URL}
+                # export OS_USERNAME=${OS_USERNAME}
+                # export OS_PASSWORD=${OS_PASSWORD}
+                # export OS_PROJECT_NAME=${OS_PROJECT_NAME}
+                openstack --insecure stack delete -y ${stack_name} || true
+                timeout 20m /bin/bash -c "while openstack --insecure stack show ${stack_name} -f value -c stack_status; do sleep 10; done";
+              """)
+            }
+        }
+    }
+
+    stage("Reboot HW nodes ") {
+        bm_ips = [
+        "185.8.59.227",
+        "185.8.59.229",
+        "5.43.225.88",
+        "5.43.225.112",
+        "5.43.225.208",
+        "5.43.227.118",
+        "185.8.58.248",
+        "185.8.59.222",
+        "5.43.225.228",
+        "5.43.229.28",
+        "5.43.225.23",
+        "185.8.58.9",
+        "185.8.58.246",
+        "185.8.58.243",
+        "185.8.58.244"
+        ]
+
+        withCredentials([
+           [$class          : 'UsernamePasswordMultiBinding',
+           credentialsId   : 'lab_engineer',
+           passwordVariable: 'lab_pass',
+           usernameVariable: 'lab_user']
+        ]) {
+            for (ip in bm_ips) { sh ("ipmitool -H ${ip} -U ${lab_user} -P ${lab_pass} chassis power off")}
+            for (ip in bm_ips) { sh ("ipmitool -H ${ip} -U ${lab_user} -P ${lab_pass} chassis bootdev pxe")}
+        }
+    }
+
+    stage("Start deployment") {
+      def jobs_map = [:]
+      for (stack_name in stacks) {
+        println stack_name
+        println get_last_build_time(stack_name)
+        jobs_map.put ("${stack_name}", get_last_build_time(stack_name))
+      }
+      stack_to_deploy = oldest_job(jobs_map)
+
+      println "Build ${stack_to_deploy}"
+      currentBuild.description = "${stack_to_deploy}"
+      // Trigger job
+      def deploy = build job: "${stack_to_deploy}",
+      parameters: [
+            string(name: 'PARENT_NODE_NAME', value: env.PARENT_NODE_NAME),
+            string(name: 'OS_CREDENTIALS', value: env.OS_CREDENTIALS)
+        ]
+    }
+  }
+}
diff --git a/jobs/project.yaml b/jobs/project.yaml
index bca767c..a0c1bd3 100644
--- a/jobs/project.yaml
+++ b/jobs/project.yaml
@@ -7,22 +7,22 @@
       - heat-cicd-queens-contrail41-sl
       - heat-cicd-queens-dvr-sl
       - self-deploy-jobs
-      # - bm-cicd-pike-ovs-maas
-      # - bm-cicd-queens-ovs-maas
+      - bm-cicd-pike-ovs-maas
+      - bm-cicd-queens-ovs-maas
       - deploy_bm
-      # - heat-bm-cicd-pike-contrail-sl
-      # - heat-bm-cicd-queens-contrail-sl
-      # - released-bm-pike-ovs
-      # - show_networks_used_by_libvirt
+      - heat-bm-cicd-pike-contrail-sl
+      - heat-bm-cicd-queens-contrail-sl
+      - released-bm-pike-ovs
+      # - show_networks_used_by_libvirt - can't moved to JJB, node parameter is not supported
       - maintenance-heat-cicd-pike-dvr-sl
       - download-config-drive
-      # - generate-report
-      # - mark-reports
+      # - generate-report - can't moved to JJB, a lot of parameters is not supported like Inject environment variables
+      # - mark-reports - can't moved to JJB, a lot of parameters is not supported like Inject environment variables
       - cookied-cicd-queens-dvr-sl-heat
-      # - diff_params_reclass_system
+      # - diff_params_reclass_system - can't moved to JJB, unsupported parameters
       - environment-template-ci
       - tcp-qa-ci
       - test_cleanup
-      # - testrail-reporter-ci
-      # - testrail-reporter-gerrit-mcp-ci
+      # - testrail-reporter-ci - can't moved to JJB, unsupported parameters
+      # - testrail-reporter-gerrit-mcp-ci - can't moved to JJB, unsupported parameters
 ...
\ No newline at end of file
diff --git a/jobs/templates/bm-cicd-pike-ovs-maas.yml b/jobs/templates/bm-cicd-pike-ovs-maas.yml
new file mode 100644
index 0000000..0c59df9
--- /dev/null
+++ b/jobs/templates/bm-cicd-pike-ovs-maas.yml
@@ -0,0 +1,204 @@
+- job-template:
+    project-type: pipeline
+    description: '{job-description}'
+    concurrent: true
+    description: runs at H(5-15) 00 * * 1
+    disabled: false
+    name: bm-cicd-pike-ovs-maas
+    parameters:
+    - string:
+        default: bm-cicd-pike-ovs-maas
+        description: ''
+        name: LAB_CONFIG_NAME
+        trim: 'false'
+    - string:
+        default: core,kvm,cicd
+        description: Comma-separated list of stacks to deploy the drivetrain (salt
+          cluster and cicd nodes)
+        name: DRIVETRAIN_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: DRIVETRAIN_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: openstack,ovs,ceph,stacklight
+        description: Comma-separated list of stacks to deploy the target platform
+          (openstack and additional components)
+        name: PLATFORM_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: PLATFORM_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: 2019.2.0
+        description: ''
+        name: MCP_VERSION
+        trim: 'false'
+    - string:
+        default: sre-team-infra
+        description: ''
+        name: NODE_NAME
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/ubuntu-16-04-x64-mcp${{MCP_VERSION}}.qcow2
+        description: ''
+        name: MCP_IMAGE_PATH1604
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/cfg01-day01.qcow2
+        description: ''
+        name: IMAGE_PATH_CFG01_DAY01
+        trim: 'false'
+    - string:
+        default: cfg01.${{LAB_CONFIG_NAME}}-config-drive.iso
+        description: ISO name that will be generated and downloaded to the /home/jenkins/images/
+        name: CFG01_CONFIG_IMAGE_NAME
+        trim: 'false'
+    - string:
+        default: bm-cicd-pike-ovs-maas
+        description: ''
+        name: ENV_NAME
+        trim: 'false'
+    - string:
+        default: ''
+        description: |-
+          Example: refs/changes/89/411189/36
+          (for now - only one reference allowed)
+        name: TCP_QA_REFS
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in pipeline-library
+        name: PIPELINE_LIBRARY_REF
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in mk-pipelines
+        name: MK_PIPELINES_REF
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: COOKIECUTTER_TEMPLATE_COMMIT
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: SALT_MODELS_SYSTEM_COMMIT
+        trim: 'false'
+    - string:
+        default: -m "run_cvp_func_sanity|run_tempest|run_stacklight"
+        description: |-
+          Pytest option -k or -m, with expression to select necessary tests.
+          Additional pytest options are allowed.
+        name: RUN_TEST_OPTS
+        trim: 'false'
+    - bool:
+        default: true
+        description: ''
+        name: SHUTDOWN_ENV_ON_TEARDOWN
+    - string:
+        default: ''
+        description: ''
+        name: COOKIECUTTER_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: ENVIRONMENT_TEMPLATE_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: '[MCP1.1_PIKE]Tempest'
+        description: ''
+        name: TEMPEST_TEST_SUITE_NAME
+        trim: 'false'
+    - string:
+        default: pike
+        description: ''
+        name: TEMPEST_IMAGE_VERSION
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_REPO_CUSTOM_TAG
+        trim: 'false'
+    - bool:
+        default: true
+        description: If set, reports will be created in TestRail for this build
+        name: REPORT_TO_TESTRAIL
+    - choice:
+        choices:
+        - heat
+        - devops
+        description: ''
+        name: ENV_MANAGER
+    - string:
+        default: https://ic-eu.ssl.mirantis.net:5000/v3
+        description: 'Openstack keystone catalog endpoint, for example https://10.90.0.80:5000/v3'
+        name: OS_AUTH_URL
+        trim: 'false'
+    - string:
+        default: sre-team
+        description: OpenStack project (tenant) name
+        name: OS_PROJECT_NAME
+        trim: 'false'
+    - string:
+        default: default
+        description: OpenStack user domain name
+        name: OS_USER_DOMAIN_NAME
+        trim: 'false'
+    - string:
+        default: sre-qa-ci-eu
+        description: Jenkins credentials ID with username and password to create a
+          heat stack in OpenStack
+        name: OS_CREDENTIALS
+        trim: 'false'
+    - string:
+        default: eu-cloud.env
+        description: |-
+          Heat template environment filename with 'parameter_defaults' dict, placed in tcp_tests/templates/_heat_environments/
+          , for example: microcloud-8116.env
+        name: LAB_PARAM_DEFAULTS
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: ''
+        name: JENKINS_PIPELINE_BRANCH
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: ''
+        name: MCP_COMMON_SCRIPTS_REFS
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_VERSION
+        trim: 'false'
+    - password:
+        name: IPMI_USER
+        default: 'engineer'
+    - password:
+        name: IPMI_PASS
+        default: 'KBwdcRqwed3w2'
+    - string:
+        default: ''
+        description: ''
+        name: BATCH_SIZE
+        trim: 'false'
+    pipeline-scm:
+      lightweight-checkout: false
+      scm:
+      - git:
+          branches:
+          - FETCH_HEAD
+          refspec: ${{TCP_QA_REFS}}
+          url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa
+      script-path: jobs/pipelines/deploy-cicd-and-run-tests.groovy
+
diff --git a/jobs/templates/bm-cicd-queens-ovs-maas.yml b/jobs/templates/bm-cicd-queens-ovs-maas.yml
new file mode 100644
index 0000000..61dd249
--- /dev/null
+++ b/jobs/templates/bm-cicd-queens-ovs-maas.yml
@@ -0,0 +1,215 @@
+- job-template:
+    project-type: pipeline
+    description: '{job-description}'
+    concurrent: true
+    disabled: false
+    name: bm-cicd-queens-ovs-maas
+    parameters:
+    - string:
+        default: bm-cicd-queens-ovs-maas
+        description: ''
+        name: LAB_CONFIG_NAME
+        trim: 'false'
+    - string:
+        default: core,kvm,cicd
+        description: Comma-separated list of stacks to deploy the drivetrain (salt
+          cluster and cicd nodes)
+        name: DRIVETRAIN_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: DRIVETRAIN_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: openstack,ovs,ceph,stacklight
+        description: Comma-separated list of stacks to deploy the target platform
+          (openstack and additional components)
+        name: PLATFORM_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: PLATFORM_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: 2019.2.0
+        description: ''
+        name: MCP_VERSION
+        trim: 'false'
+    - string:
+        default: sre-team-infra
+        description: ''
+        name: NODE_NAME
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/ubuntu-16-04-x64-mcp${{MCP_VERSION}}.qcow2
+        description: ''
+        name: MCP_IMAGE_PATH1604
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/cfg01-day01.qcow2
+        description: ''
+        name: IMAGE_PATH_CFG01_DAY01
+        trim: 'false'
+    - string:
+        default: cfg01.${{LAB_CONFIG_NAME}}-config-drive.iso
+        description: ISO name that will be generated and downloaded to the /home/jenkins/images/
+        name: CFG01_CONFIG_IMAGE_NAME
+        trim: 'false'
+    - string:
+        default: bm-cicd-queens-ovs-maas
+        description: ''
+        name: ENV_NAME
+        trim: 'false'
+    - string:
+        default: ''
+        description: |-
+          Example: refs/changes/89/411189/36
+          (for now - only one reference allowed)
+        name: TCP_QA_REFS
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in pipeline-library
+        name: PIPELINE_LIBRARY_REF
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in mk-pipelines
+        name: MK_PIPELINES_REF
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: COOKIECUTTER_TEMPLATE_COMMIT
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: SALT_MODELS_SYSTEM_COMMIT
+        trim: 'false'
+    - string:
+        default: -m "run_cvp_func_sanity|run_tempest|run_stacklight"
+        description: |-
+          Pytest option -k or -m, with expression to select necessary tests.
+          Additional pytest options are allowed.
+        name: RUN_TEST_OPTS
+        trim: 'false'
+    - bool:
+        default: true
+        description: ''
+        name: SHUTDOWN_ENV_ON_TEARDOWN
+    - string:
+        default: ''
+        description: ''
+        name: COOKIECUTTER_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: ENVIRONMENT_TEMPLATE_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: '[MCP1.1_QUEENS]Tempest'
+        description: ''
+        name: TEMPEST_TEST_SUITE_NAME
+        trim: 'false'
+    - string:
+        default: queens
+        description: ''
+        name: TEMPEST_IMAGE_VERSION
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_REPO_CUSTOM_TAG
+        trim: 'false'
+    - bool:
+        default: true
+        description: If set, reports will be created in TestRail for this build
+        name: REPORT_TO_TESTRAIL
+    - choice:
+        choices:
+        - heat
+        - devops
+        description: ''
+        name: ENV_MANAGER
+    - string:
+        default: https://ic-eu.ssl.mirantis.net:5000/v3
+        description: 'Openstack keystone catalog endpoint, for example https://10.90.0.80:5000/v3'
+        name: OS_AUTH_URL
+        trim: 'false'
+    - string:
+        default: sre-team
+        description: OpenStack project (tenant) name
+        name: OS_PROJECT_NAME
+        trim: 'false'
+    - string:
+        default: default
+        description: OpenStack user domain name
+        name: OS_USER_DOMAIN_NAME
+        trim: 'false'
+    - string:
+        default: sre-qa-ci-eu
+        description: Jenkins credentials ID with username and password to create a
+          heat stack in OpenStack
+        name: OS_CREDENTIALS
+        trim: 'false'
+    - string:
+        default: eu-cloud.env
+        description: |-
+          Heat template environment filename with 'parameter_defaults' dict, placed in tcp_tests/templates/_heat_environments/
+          , for example: microcloud-8116.env
+        name: LAB_PARAM_DEFAULTS
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: ''
+        name: JENKINS_PIPELINE_BRANCH
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: ''
+        name: MCP_COMMON_SCRIPTS_REFS
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_VERSION
+        trim: 'false'
+    - password:
+        name: IPMI_USER
+        default: 'engineer'
+    - password:
+        name: IPMI_PASS
+        default: 'KBwdcRqwed3w2'
+    - string:
+        default: ''
+        description: ''
+        name: TEMPEST_EXTRA_ARGS
+        trim: 'false'
+    - password:
+        name: CISCO_PASS
+        default: '1fac0DlhILBo'
+    - string:
+        default: ''
+        description: ''
+        name: SALT_MODELS_SYSTEM_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: BATCH_SIZE
+        trim: 'false'
+    pipeline-scm:
+      lightweight-checkout: false
+      scm:
+      - git:
+          branches:
+          - FETCH_HEAD
+          refspec: ${{TCP_QA_REFS}}
+          url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa
+      script-path: jobs/pipelines/deploy-cicd-and-run-tests.groovy
diff --git a/jobs/templates/deploy_bm.yml b/jobs/templates/deploy_bm.yml
index 3f1d02a..3507e54 100644
--- a/jobs/templates/deploy_bm.yml
+++ b/jobs/templates/deploy_bm.yml
@@ -1,121 +1,11 @@
 - job-template:
     project-type: pipeline
     description: '{job-description}'
-    concurrent: true
+    concurrent: false
     disabled: false
     sandbox: true
     triggers:
     - timed: H(40-59) 0 * * 1-5
-    dsl: |-
-      /**
-      *
-      * Deploy the product cluster using Jenkins master on CICD cluster
-      *
-      * Expected parameters:
-      *   STACKS                        List of deployments from 'all physical' view in the Jenkins
-      They should be separated by newline
-      *   PARENT_NODE_NAME
-      *   OS_AUTH_URL                   OpenStack keystone catalog URL
-      *   OS_PROJECT_NAME               OpenStack project (tenant) name
-      *   OS_CREDENTIALS                OpenStack username and password credentials ID in Jenkins
-
-      *   TCP_QA_REFS                   Reference to the tcp-qa change on Gerrit, like refs/changes/46/418546/41
-      */
-      @Library('tcp-qa')_
-
-      common = new com.mirantis.mk.Common()
-      shared = new com.mirantis.system_qa.SharedPipeline()
-      import jenkins.model.*
-
-      def stacks = env.STACKS.readLines()
-
-      def get_last_build_time(job_name) {{
-      def job = Hudson.instance.getJob(job_name)
-      def last_build = Jenkins.instance.getItemByFullName(job.fullName).getLastBuild()
-      if (last_build == null) {{
-      return null
-      }}
-      def upStreamBuild = Jenkins.getInstance().getItemByFullName(job.fullName).getBuildByNumber(last_build.getNumber())
-      return upStreamBuild.getTime()
-      }}
-
-      @NonCPS
-      def oldest_job(map) {{
-      println map.sort({{a, b -> a.value <=> b.value}})*.value
-      return map.sort({{a, b -> a.value <=> b.value}})*.key[0]
-      }}
-
-      // ============================================================================
-      timeout(time: 15, unit: 'HOURS') {{
-      node (env.PARENT_NODE_NAME) {{
-      stage("Remove extra stacks") {{
-      withCredentials([
-      [$class          : 'UsernamePasswordMultiBinding',
-      credentialsId   : env.OS_CREDENTIALS,
-      passwordVariable: 'OS_PASSWORD',
-      usernameVariable: 'OS_USERNAME']
-      ]) {{
-      env.OS_IDENTITY_API_VERSION = 3
-      for (stack_name in stacks) {{
-      shared.run_cmd("""
-      # export OS_IDENTITY_API_VERSION=3
-      # export OS_AUTH_URL=${{OS_AUTH_URL}}
-      # export OS_USERNAME=${{OS_USERNAME}}
-      # export OS_PASSWORD=${{OS_PASSWORD}}
-      # export OS_PROJECT_NAME=${{OS_PROJECT_NAME}}
-      openstack --insecure stack delete -y ${{stack_name}} || true
-      timeout 20m /bin/bash -c "while openstack --insecure stack show ${{stack_name}} -f value -c stack_status; do sleep 10; done";
-      """)
-      }}
-      }}
-      }}
-
-      stage("Reboot HW nodes ") {{
-      bm_ips = [
-      "185.8.59.227",
-      "185.8.59.229",
-      "5.43.225.88",
-      "5.43.225.112",
-      "5.43.225.208",
-      "5.43.227.118",
-      "185.8.58.248",
-      "185.8.59.222",
-      "5.43.225.228",
-      "5.43.229.28",
-      "5.43.225.23",
-      "185.8.58.9",
-      "185.8.58.246",
-      "185.8.58.243",
-      "185.8.58.244"
-      ]
-
-      lab_user = "engineer"
-      lab_pass = "KBwdcRqwed3w2"
-      for (ip in bm_ips) {{ sh ("ipmitool -H ${{ip}} -U ${{lab_user}} -P ${{lab_pass}} chassis power off")}}
-      for (ip in bm_ips) {{ sh ("ipmitool -H ${{ip}} -U ${{lab_user}} -P ${{lab_pass}} chassis bootdev pxe")}}
-      }}
-
-      stage("Start deployment") {{
-      def jobs_map = [:]
-      for (stack_name in stacks) {{
-      println stack_name
-      println get_last_build_time(stack_name)
-      jobs_map.put ("${{stack_name}}", get_last_build_time(stack_name))
-      }}
-      stack_to_deploy = oldest_job(jobs_map)
-
-      println "Build ${{stack_to_deploy}}"
-      currentBuild.description = "${{stack_to_deploy}}"
-      // Trigger job
-      def deploy = build job: "${{stack_to_deploy}}",
-      parameters: [
-      string(name: 'PARENT_NODE_NAME', value: env.PARENT_NODE_NAME),
-      string(name: 'OS_CREDENTIALS', value: env.OS_CREDENTIALS)
-      ]
-      // currentBuild.status = deploy.status
-      }}
-      }}
-      }}
     name: deploy_bm
     parameters:
     - string:
@@ -152,4 +42,13 @@
         description: ''
         name: OS_PROJECT_NAME
         trim: 'false'
+    pipeline-scm:
+      lightweight-checkout: false
+      scm:
+        - git:
+            branches:
+              - FETCH_HEAD
+            refspec: ${{TCP_QA_REFS}}
+            url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa.git
+      script-path: jobs/pipelines/rotation-released-deployment.groovy
 
diff --git a/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml b/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
new file mode 100644
index 0000000..2ea8bc5
--- /dev/null
+++ b/jobs/templates/heat-bm-cicd-pike-contrail-sl.yml
@@ -0,0 +1,228 @@
+- job-template:
+    project-type: pipeline
+    description: '{job-description}'
+    concurrent: true
+    disabled: false
+    name: heat-bm-cicd-pike-contrail-sl
+    parameters:
+    - string:
+        default: heat-bm-cicd-pike-contrail-sl
+        description: ''
+        name: LAB_CONFIG_NAME
+        trim: 'false'
+    - string:
+        default: core,kvm,cicd
+        description: Comma-separated list of stacks to deploy the drivetrain (salt
+          cluster and cicd nodes)
+        name: DRIVETRAIN_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: DRIVETRAIN_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: openstack,contrail,ceph,stacklight
+        description: Comma-separated list of stacks to deploy the target platform
+          (openstack and additional components)
+        name: PLATFORM_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '26000'
+        description: |-
+          26000 - Default for enabled encryption;
+          12000 - without
+        name: PLATFORM_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: 2019.2.0
+        description: ''
+        name: MCP_VERSION
+        trim: 'false'
+    - string:
+        default: sre-team-infra
+        description: ''
+        name: NODE_NAME
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/ubuntu-16-04-x64-mcp${{MCP_VERSION}}.qcow2
+        description: ''
+        name: MCP_IMAGE_PATH1604
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/cfg01-day01.qcow2
+        description: ''
+        name: IMAGE_PATH_CFG01_DAY01
+        trim: 'false'
+    - string:
+        default: cfg01.${{LAB_CONFIG_NAME}}-config-drive.iso
+        description: ISO name that will be generated and downloaded to the /home/jenkins/images/
+        name: CFG01_CONFIG_IMAGE_NAME
+        trim: 'false'
+    - string:
+        default: heat-bm-cicd-pike-contrail-sl
+        description: ''
+        name: ENV_NAME
+        trim: 'false'
+    - string:
+        default: ''
+        description: |-
+          Example: refs/changes/89/411189/36
+          (for now - only one reference allowed)
+        name: TCP_QA_REFS
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in pipeline-library
+        name: PIPELINE_LIBRARY_REF
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in mk-pipelines
+        name: MK_PIPELINES_REF
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: COOKIECUTTER_TEMPLATE_COMMIT
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: SALT_MODELS_SYSTEM_COMMIT
+        trim: 'false'
+    - string:
+        default: -m "run_cvp_func_sanity|run_tempest|run_stacklight"
+        description: |-
+          Pytest option -k or -m, with expression to select necessary tests.
+          Additional pytest options are allowed.
+        name: RUN_TEST_OPTS
+        trim: 'false'
+    - bool:
+        default: true
+        description: ''
+        name: SHUTDOWN_ENV_ON_TEARDOWN
+    - string:
+        default: ''
+        description: ''
+        name: COOKIECUTTER_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: ENVIRONMENT_TEMPLATE_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: '[MCP1.1_PIKE]Tempest'
+        description: ''
+        name: TEMPEST_TEST_SUITE_NAME
+        trim: 'false'
+    - string:
+        default: pike
+        description: ''
+        name: TEMPEST_IMAGE_VERSION
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: TEMPEST_TARGET
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_REPO_CUSTOM_TAG
+        trim: 'false'
+    - bool:
+        default: true
+        description: If set, reports will be created in TestRail for this build
+        name: REPORT_TO_TESTRAIL
+    - choice:
+        choices:
+        - heat
+        - devops
+        description: ''
+        name: ENV_MANAGER
+    - string:
+        default: https://ic-eu.ssl.mirantis.net:5000/v3
+        description: 'Openstack keystone catalog endpoint, for example https://10.90.0.80:5000/v3'
+        name: OS_AUTH_URL
+        trim: 'false'
+    - string:
+        default: sre-team
+        description: OpenStack project (tenant) name
+        name: OS_PROJECT_NAME
+        trim: 'false'
+    - string:
+        default: default
+        description: OpenStack user domain name
+        name: OS_USER_DOMAIN_NAME
+        trim: 'false'
+    - string:
+        default: sre-qa-ci-eu
+        description: Jenkins credentials ID with username and password to create a
+          heat stack in OpenStack
+        name: OS_CREDENTIALS
+        trim: 'false'
+    - string:
+        default: eu-cloud.env
+        description: |-
+          Heat template environment filename with 'parameter_defaults' dict, placed in tcp_tests/templates/_heat_environments/
+          , for example: microcloud-8116.env
+        name: LAB_PARAM_DEFAULTS
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: ''
+        name: JENKINS_PIPELINE_BRANCH
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: ''
+        name: MCP_COMMON_SCRIPTS_REFS
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_VERSION
+        trim: 'false'
+    - password:
+        name: IPMI_USER
+        default: 'engineer'
+    - password:
+        name: IPMI_PASS
+        default: 'KBwdcRqwed3w2'
+    - string:
+        default: ''
+        description: ''
+        name: TEMPEST_EXTRA_ARGS
+        trim: 'false'
+    - bool:
+        default: true
+        description: ''
+        name: SKIP_SYNC_TIME
+    - string:
+        default: ^heat_tempest_plugin.tests*|^tempest.api.image*|^tempest_horizon*|^tempest.api.identity*|^tempest.api.network*|^tempest.api.compute*|^tempest.api.volume*|^tempest.scenario*|^tempest.api.object_storage*
+        description: Exclude tungsten tests by default
+        name: TEMPEST_PATTERN
+        trim: 'false'
+    - string:
+        default: ''
+        description: Comma separated list of additional stacks to delete
+        name: ADDITIONAL_STACKS
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: BATCH_SIZE
+        trim: 'false'
+    pipeline-scm:
+      lightweight-checkout: false
+      scm:
+      - git:
+          branches:
+          - FETCH_HEAD
+          refspec: ${{TCP_QA_REFS}}
+          url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa
+      script-path: jobs/pipelines/deploy-cicd-and-run-tests.groovy
diff --git a/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml b/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
new file mode 100644
index 0000000..b2c53bc
--- /dev/null
+++ b/jobs/templates/heat-bm-cicd-queens-contrail-sl.yml
@@ -0,0 +1,231 @@
+- job-template:
+    project-type: pipeline
+    concurrent: true
+    description: |-
+      {job-description}
+      Runs by
+      H(5-15) 00 * * 1,4
+    disabled: false
+    name: heat-bm-cicd-queens-contrail-sl
+    parameters:
+    - string:
+        default: heat-bm-cicd-queens-contrail-sl
+        description: ''
+        name: LAB_CONFIG_NAME
+        trim: 'false'
+    - string:
+        default: core,kvm,cicd
+        description: Comma-separated list of stacks to deploy the drivetrain (salt
+          cluster and cicd nodes)
+        name: DRIVETRAIN_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: DRIVETRAIN_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: openstack,contrail,ceph,stacklight
+        description: Comma-separated list of stacks to deploy the target platform
+          (openstack and additional components)
+        name: PLATFORM_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: PLATFORM_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: 2019.2.0
+        description: ''
+        name: MCP_VERSION
+        trim: 'false'
+    - string:
+        default: sre-team-infra
+        description: ''
+        name: NODE_NAME
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/ubuntu-16-04-x64-mcp${{MCP_VERSION}}.qcow2
+        description: ''
+        name: MCP_IMAGE_PATH1604
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/cfg01-day01.qcow2
+        description: ''
+        name: IMAGE_PATH_CFG01_DAY01
+        trim: 'false'
+    - string:
+        default: cfg01.${{LAB_CONFIG_NAME}}-config-drive.iso
+        description: ISO name that will be generated and downloaded to the /home/jenkins/images/
+        name: CFG01_CONFIG_IMAGE_NAME
+        trim: 'false'
+    - string:
+        default: heat-bm-cicd-queens-contrail-sl
+        description: ''
+        name: ENV_NAME
+        trim: 'false'
+    - string:
+        default: ''
+        description: |-
+          Example: refs/changes/89/411189/36
+          (for now - only one reference allowed)
+        name: TCP_QA_REFS
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in pipeline-library
+        name: PIPELINE_LIBRARY_REF
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: reference to patchset in mk-pipelines
+        name: MK_PIPELINES_REF
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: COOKIECUTTER_TEMPLATE_COMMIT
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: SALT_MODELS_SYSTEM_COMMIT
+        trim: 'false'
+    - string:
+        default: -m "run_cvp_func_sanity|run_tempest|run_stacklight"
+        description: |-
+          Pytest option -k or -m, with expression to select necessary tests.
+          Additional pytest options are allowed.
+        name: RUN_TEST_OPTS
+        trim: 'false'
+    - bool:
+        default: true
+        description: ''
+        name: SHUTDOWN_ENV_ON_TEARDOWN
+    - string:
+        default: ''
+        description: ''
+        name: COOKIECUTTER_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: ENVIRONMENT_TEMPLATE_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: '[MCP1.1_QUEENS]Tempest'
+        description: ''
+        name: TEMPEST_TEST_SUITE_NAME
+        trim: 'false'
+    - string:
+        default: queens
+        description: ''
+        name: TEMPEST_IMAGE_VERSION
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: TEMPEST_TARGET
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_REPO_CUSTOM_TAG
+        trim: 'false'
+    - bool:
+        default: true
+        description: If set, reports will be created in TestRail for this build
+        name: REPORT_TO_TESTRAIL
+    - choice:
+        choices:
+        - heat
+        - devops
+        description: ''
+        name: ENV_MANAGER
+    - string:
+        default: https://ic-eu.ssl.mirantis.net:5000/v3
+        description: 'Openstack keystone catalog endpoint, for example https://10.90.0.80:5000/v3
+
+          '
+        name: OS_AUTH_URL
+        trim: 'false'
+    - string:
+        default: sre-team
+        description: OpenStack project (tenant) name
+        name: OS_PROJECT_NAME
+        trim: 'false'
+    - string:
+        default: default
+        description: OpenStack user domain name
+        name: OS_USER_DOMAIN_NAME
+        trim: 'false'
+    - string:
+        default: sre-qa-ci-eu
+        description: Jenkins credentials ID with username and password to create a
+          heat stack in OpenStack
+        name: OS_CREDENTIALS
+        trim: 'false'
+    - string:
+        default: eu-cloud.env
+        description: |-
+          Heat template environment filename with 'parameter_defaults' dict, placed in tcp_tests/templates/_heat_environments/
+          , for example: microcloud-8116.env
+        name: LAB_PARAM_DEFAULTS
+        trim: 'false'
+    - string:
+        default: release/proposed/2019.2.0
+        description: ''
+        name: JENKINS_PIPELINE_BRANCH
+        trim: 'false'
+    - string:
+        default: refs/heads/release/proposed/2019.2.0
+        description: ''
+        name: MCP_COMMON_SCRIPTS_REFS
+        trim: 'false'
+    - string:
+        default: proposed
+        description: ''
+        name: UPDATE_VERSION
+        trim: 'false'
+    - password:
+        name: IPMI_USER
+        default: 'engineer'
+    - password:
+        name: IPMI_PASS
+        default: 'KBwdcRqwed3w2'
+    - string:
+        default: ''
+        description: ''
+        name: TEMPEST_EXTRA_ARGS
+        trim: 'false'
+    - bool:
+        default: true
+        description: ''
+        name: SKIP_SYNC_TIME
+    - string:
+        default: ^heat_tempest_plugin.tests*|^tempest.api.image*|^tempest_horizon*|^tempest.api.identity*|^tempest.api.network*|^tempest.api.compute*|^tempest.api.volume*|^tempest.scenario*|^tempest.api.object_storage*
+        description: Exclude tungsten tests by default
+        name: TEMPEST_PATTERN
+        trim: 'false'
+    - string:
+        default: ''
+        description: Comma separated list of additional stacks to delete
+        name: ADDITIONAL_STACKS
+        trim: 'false'
+    - string:
+        default: ''
+        description: ''
+        name: BATCH_SIZE
+        trim: 'false'
+    pipeline-scm:
+      lightweight-checkout: false
+      scm:
+      - git:
+          branches:
+          - FETCH_HEAD
+          refspec: ${{TCP_QA_REFS}}
+          url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa
+      script-path: jobs/pipelines/deploy-cicd-and-run-tests.groovy
diff --git a/jobs/templates/released-bm-pike-ovs.yml b/jobs/templates/released-bm-pike-ovs.yml
new file mode 100644
index 0000000..6828da1
--- /dev/null
+++ b/jobs/templates/released-bm-pike-ovs.yml
@@ -0,0 +1,187 @@
+- job-template:
+    project-type: pipeline
+    description: '{job-description}'
+    concurrent: true
+    disabled: false
+    name: released-bm-pike-ovs
+    parameters:
+    - string:
+        default: released-bm-pike-ovs
+        description: ''
+        name: LAB_CONFIG_NAME
+        trim: 'false'
+    - string:
+        default: core,kvm,cicd
+        description: Comma-separated list of stacks to deploy the drivetrain (salt
+          cluster and cicd nodes)
+        name: DRIVETRAIN_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: DRIVETRAIN_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: openstack,ovs,ceph,stacklight
+        description: Comma-separated list of stacks to deploy the target platform
+          (openstack and additional components)
+        name: PLATFORM_STACK_INSTALL
+        trim: 'false'
+    - string:
+        default: '24000'
+        description: ''
+        name: PLATFORM_STACK_INSTALL_TIMEOUT
+        trim: 'false'
+    - string:
+        default: 2019.2.0
+        description: ''
+        name: MCP_VERSION
+        trim: 'false'
+    - string:
+        default: sre-team-infra
+        description: ''
+        name: NODE_NAME
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/ubuntu-16-04-x64-mcp${{MCP_VERSION}}.qcow2
+        description: ''
+        name: MCP_IMAGE_PATH1604
+        trim: 'false'
+    - string:
+        default: /home/jenkins/images/cfg01-day01-2018.11.0.qcow2
+        description: ''
+        name: IMAGE_PATH_CFG01_DAY01
+        trim: 'false'
+    - string:
+        default: cfg01.${{LAB_CONFIG_NAME}}-config-drive.iso
+        description: ISO name that will be generated and downloaded to the /home/jenkins/images/
+        name: CFG01_CONFIG_IMAGE_NAME
+        trim: 'false'
+    - string:
+        default: released-bm-pike-ovs
+        description: ''
+        name: ENV_NAME
+        trim: 'false'
+    - string:
+        default: refs/heads/2019.2.0
+        description: |-
+          Example: refs/changes/89/411189/36
+          (for now - only one reference allowed)
+        name: TCP_QA_REFS
+        trim: 'false'
+    - string:
+        default: refs/heads/release/2019.2.0
+        description: reference to patchset in pipeline-library
+        name: PIPELINE_LIBRARY_REF
+        trim: 'false'
+    - string:
+        default: refs/heads/release/2019.2.0
+        description: reference to patchset in mk-pipelines
+        name: MK_PIPELINES_REF
+        trim: 'false'
+    - string:
+        default: 2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: COOKIECUTTER_TEMPLATE_COMMIT
+        trim: 'false'
+    - string:
+        default: 2019.2.0
+        description: Can be 'master' or 'proposed'. If empty, then takes ${{MCP_VERSION}}
+          value
+        name: SALT_MODELS_SYSTEM_COMMIT
+        trim: 'false'
+    - string:
+        default: -m "run_cvp_func_sanity|run_tempest|run_stacklight"
+        description: |-
+          Pytest option -k or -m, with expression to select necessary tests.
+          Additional pytest options are allowed.
+        name: RUN_TEST_OPTS
+        trim: 'false'
+    - bool:
+        default: true
+        description: ''
+        name: SHUTDOWN_ENV_ON_TEARDOWN
+    - string:
+        default: ''
+        description: ''
+        name: COOKIECUTTER_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: refs/heads/2019.2.0
+        description: ''
+        name: ENVIRONMENT_TEMPLATE_REF_CHANGE
+        trim: 'false'
+    - string:
+        default: '[MCP1.1_PIKE]Tempest'
+        description: ''
+        name: TEMPEST_TEST_SUITE_NAME
+        trim: 'false'
+    - string:
+        default: pike
+        description: ''
+        name: TEMPEST_IMAGE_VERSION
+        trim: 'false'
+    - bool:
+        default: true
+        description: If set, reports will be created in TestRail for this build
+        name: REPORT_TO_TESTRAIL
+    - choice:
+        choices:
+        - heat
+        - devops
+        description: ''
+        name: ENV_MANAGER
+    - string:
+        default: https://ic-eu.ssl.mirantis.net:5000/v3
+        description: 'Openstack keystone catalog endpoint, for example https://10.90.0.80:5000/v3'
+        name: OS_AUTH_URL
+        trim: 'false'
+    - string:
+        default: sre-team
+        description: OpenStack project (tenant) name
+        name: OS_PROJECT_NAME
+        trim: 'false'
+    - string:
+        default: ldap_mirantis
+        description: OpenStack user domain name
+        name: OS_USER_DOMAIN_NAME
+        trim: 'false'
+    - string:
+        default: openstack_ic-eu.ssl.mirantis.net
+        description: Jenkins credentials ID with username and password to create a
+          heat stack in OpenStack
+        name: OS_CREDENTIALS
+        trim: 'false'
+    - string:
+        default: eu-cloud.env
+        description: |-
+          Heat template environment filename with 'parameter_defaults' dict, placed in tcp_tests/templates/_heat_environments/
+          , for example: microcloud-8116.env
+        name: LAB_PARAM_DEFAULTS
+        trim: 'false'
+    - string:
+        default: refs/heads/release/2019.2.0
+        description: ''
+        name: JENKINS_PIPELINE_BRANCH
+        trim: 'false'
+    - string:
+        default: refs/heads/release/2019.2.0
+        description: ''
+        name: MCP_COMMON_SCRIPTS_REFS
+        trim: 'false'
+    - password:
+        name: IPMI_USER
+        default: 'engineer'
+    - password:
+        name: IPMI_PASS
+        default: 'KBwdcRqwed3w2'
+    pipeline-scm:
+      lightweight-checkout: false
+      scm:
+      - git:
+          branches:
+          - FETCH_HEAD
+          refspec: ${{TCP_QA_REFS}}
+          url: https://gerrit.mcp.mirantis.com/mcp/tcp-qa
+      script-path: jobs/pipelines/deploy-cicd-and-run-tests.groovy
diff --git a/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml b/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
index 7e03a30..ad0c20b 100644
--- a/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
+++ b/jobs/templates/released-heat-cicd-pike-contrail41-sl.yml
@@ -162,9 +162,7 @@
         name: ENV_MANAGER
     - string:
         default: https://ic-eu.ssl.mirantis.net:5000/v3
-        description: 'Openstack keystone catalog endpoint, for example https://10.90.0.80:5000/v3
-
-          '
+        description: 'Openstack keystone catalog endpoint, for example https://10.90.0.80:5000/v3'
         name: OS_AUTH_URL
         trim: 'false'
     - string: