Mark testresults automatically for each deployment

PROD-35718

Change-Id: I0289d9db8316d4ffcfd984b42be22e02b1abab1d
diff --git a/checklist.yaml b/checklist.yaml
index 570b16f..d962ec4 100644
--- a/checklist.yaml
+++ b/checklist.yaml
@@ -1,13 +1,4 @@
 tests:
-#  - title: 'tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_list_servers_by_admin[id-51717b38-bdc1-458b-b636-1cf82d99f62f]'
-#    status: 'ProdFailed'
-#    errors:
-#      - 'tempest.exceptions.BuildErrorException: Server'
-#      - 'Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance'
-#    defects: 'PROD-25148'
-#
-#  - title: 'tempest.api.compute.admin.test_servers.ServersAdminTestJSON.test_create_server_with_scheduling_hint[id-fdcd9b33-0903-4e00-a1f7-b5f6543068d6]'
-#    status: 'InfraFailed'
 
   - title: 'tempest.api.network.admin.test_routers.RoutersAdminTest.test_update_router_set_gateway[id-6cc285d8-46bf-4f36-9b1a-783e3008ba79]'
     errors:
@@ -21,17 +12,6 @@
     status: "ProdFailed"
     defects: 'PROD-25134'
 
-  - title: 'test_prometheus_alert_count[ReclassRemoteDesync]'
-    errors:
-      - "local files under /srv/salt/reclass have diverged from expected remote state"
-    status: "Won't Fix"
-
-  - title: "test_prometheus_alert_count[ContrailBGPSessionsDown]"
-    errors:
-      - "1 OpenContrail BGP sessions on the ntw01 node are down for 2 minutes."
-    status: "ProdFailed"
-    defects: 'PROD-34379'
-
 ## ==================================================
   - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_delete_image_metadata_item"
     errors:
@@ -39,25 +19,25 @@
     status: "ProdFailed"
     defects: "PROD-35212"
 
-  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_paused_server"
+  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_paused_server[id-71bcb732-0261-11e7-9086-fa163e4fa634]"
     errors:
       - "502 GET https://10.6.0.80:8774/v2.1/images/"
     status: "ProdFailed"
     defects: "PROD-35212"
 
-  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_stopped_server"
+  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_stopped_server[id-aaacd1d0-55a2-4ce8-818a-b5439df8adc9]"
     errors:
       - "502 GET https://10.6.0.80:8774/v2.1/images/"
     status: "ProdFailed"
     defects: "PROD-35212"
 
-  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_suspended_server"
+  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_image_from_suspended_server[id-8ca07fec-0262-11e7-907e-fa163e4fa634]"
     errors:
       - "502 GET https://10.6.0.80:8774/v2.1/images/"
     status: "ProdFailed"
     defects: "PROD-35212"
 
-  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image"
+  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image[id-3731d080-d4c5-4872-b41a-64d0d0021314]"
     errors:
       - "502 GET https://10.6.0.80:8774/v2.1/images/"
     status: "ProdFailed"
@@ -66,19 +46,26 @@
 
   - title: ".setUpClass (tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON)"
     errors:
-      - "cls.server2['id'], wait_until='ACTIVE')"
+      - 'resp, body = self.get("images/%s" % image_id)'
       - "tempest.lib.exceptions.UnexpectedResponseCode: Unexpected response code received"
       - "Details: 502"
     status: "ProdFailed"
     defects: "PROD-35212"
 
-  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image"
+  - title: "tempest.api.compute.images.test_images.ImagesTestJSON.test_create_delete_image[id-3731d080-d4c5-4872-b41a-64d0d0021314]"
     errors:
       - "message': u\"Cannot 'createImage' instance"
       - "while it is in task_state image_uploading\", u'code': 409"
     status: "ProdFailed"
     defects: "PROD-35223"
 
+  - title: ".setUpClass (tempest.api.compute.images.test_list_image_filters.ListImageFiltersTestJSON)"
+    errors:
+      - 'resp, body = self.get("images/%s" % image_id)'
+      - "Details: 502"
+    status: "ProdFailed"
+    defects: "PROD-35212"
+
 
 # ---------------------------------------------------
   - title: "tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_get_console_output_server_id_in_shutoff_status"
@@ -117,13 +104,19 @@
       - "local files under /srv/salt/reclass have diverged from expected remote state"
     status: "WontFix"
 
+  - title: "test_prometheus_alert_count[ContrailBGPSessionsDown]"
+    errors:
+      - "1 OpenContrail BGP sessions on the ntw01 node are down for 2 minutes."
+    status: "ProdFailed"
+    defects: 'PROD-34379'
 
 ## ==================================================
 #              LMA2.0_Automated
 ## ==================================================
-  - title: "test_openstack_api_check_status_metrics "
+  - title: "test_openstack_api_check_status_metrics"
     errors:
-      - "Incorrect value in metric {u'value': [1600846840.578, u'0'], u'metric': {u'service_name': u'opencontrail', u'interface': u'public', u'service_id': u'35662704fa664daf95de90abc5180139', u'region': u'RegionOne', u'name': u'opencontrail', u'job': u'remote_agent_openstack_api', u'environment': u'heat-cicd-queens-contrail41-sl.local', u'instance': u'monitoring_remote_agent:9128', u'endpoint_id': u'721e2ad9e82148c9b5dbde40db83e47e', u'__name__': u'openstack_api_check_status'}}"
+      - "Incorrect value in metric"
+      - "u'metric': {u'service_name': u'opencontrail', u'interface': u'public'"
     status: "WontFix"
     defects: "PROD-35759"
 
diff --git a/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy b/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy
index 10ea257..dd866a3 100644
--- a/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy
+++ b/jobs/pipelines/swarm-bootstrap-salt-cluster-heat.groovy
@@ -263,18 +263,27 @@
 
                 stage("Clean the environment and clone tcp-qa") {
                     deleteDir()
+                    // Install  TestRail reporter to upload test results to TestRail
                     shared.verbose_sh("""\
                         [ -d /home/jenkins/venv_testrail_reporter ] || virtualenv /home/jenkins/venv_testrail_reporter
                     """, true, false, true)
                     shared.run_cmd("""\
                         . /home/jenkins/venv_testrail_reporter/bin/activate; pip install git+https://github.com/dis-xcom/testrail_reporter -U
                     """)
+                    // Install DEVOPS tools
                     shared.verbose_sh("""\
                         [ -d /home/jenkins/fuel-devops30 ] || virtualenv /home/jenkins/fuel-devops30
                     """, true, false, true)
                     shared.run_cmd("""\
                         git clone https://gerrit.mcp.mirantis.com/mcp/tcp-qa ${PARENT_WORKSPACE}
                     """)
+                    // Install  TestRail reporter to mark known failures test results to TestRail
+                    shared.verbose_sh("""\
+                        [ -d /home/jenkins/venv_testrail_analyzer ] || virtualenv --python=python3 /home/jenkins/venv_testrail_analyzer
+                    """, true, false, true)
+                    shared.run_cmd("""\
+                        . /home/jenkins/venv_testrail_analyzer/bin/activate; pip install git+https://github.com/ibumarskov/testrail-reporter
+                    """)
                     shared.update_working_dir()
                 }
 
@@ -289,7 +298,7 @@
                     stage("Run the 'underlay' and 'salt-deployed' fixtures to bootstrap salt cluster") {
                         def xml_report_name = "deploy_salt.xml"
                         try {
-                            // deploy_salt.xml
+                            // deploy_salt.xmli
                             shared.run_sh("""\
                                 export ENV_NAME=${ENV_NAME}
                                 export LAB_CONFIG_NAME=${LAB_CONFIG_NAME}
diff --git a/jobs/pipelines/swarm-testrail-report.groovy b/jobs/pipelines/swarm-testrail-report.groovy
index 17b770f..4af60af 100644
--- a/jobs/pipelines/swarm-testrail-report.groovy
+++ b/jobs/pipelines/swarm-testrail-report.groovy
@@ -19,6 +19,8 @@
 def common = new com.mirantis.mk.Common()
 def shared = new com.mirantis.system_qa.SharedPipeline()
 def stacks = shared.get_steps_list(PASSED_STEPS)
+def testPlanNamePrefix = env.TEST_PLAN_NAME_PREFIX ?: "[2019.2.0-update]System"
+def testPlanName = env.TEST_PLAN_NAME ?: "${testPlanNamePrefix}-${MCP_VERSION}-${new Date().format('yyyy-MM-dd')}"
 
 if (! env.PARENT_NODE_NAME) {
     error "'PARENT_NODE_NAME' must be set from the parent deployment job!"
@@ -85,7 +87,7 @@
                     reporter_extra_options = [
                       "--testrail-add-missing-cases",
                     ]
-                    ret = shared.upload_results_to_testrail(deployment_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    ret = shared.upload_results_to_testrail(deployment_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
                     common.printMsg(ret.stdout, "blue")
                     report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
@@ -106,7 +108,7 @@
                       "--testrail-add-missing-cases",
 //                       "--testrail_configuration_name tcp-qa",
                     ]
-                    ret = shared.upload_results_to_testrail(tcpqa_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    ret = shared.upload_results_to_testrail(tcpqa_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
                     common.printMsg(ret.stdout, "blue")
                     report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
@@ -129,11 +131,15 @@
 //                       "--testrail_configuration_name tcp-qa",
                     ]
                     ret = shared.upload_results_to_testrail(tempest_report_name,
+                                                            testPlanName,
                                                             testSuiteName,
                                                             methodname,
                                                             testrail_name_template,
                                                             reporter_extra_options)
                     common.printMsg(ret.stdout, "blue")
+                    marked = shared.mark_test_results(testPlanName, testSuiteName)
+                    common.printMsg(marked.stdout, "blue")
+
                     report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
@@ -153,8 +159,11 @@
                       "--testrail-add-missing-cases",
                       "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
                     ]
-                    ret = shared.upload_results_to_testrail(stacklight_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    ret = shared.upload_results_to_testrail(stacklight_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
                     common.printMsg(ret.stdout, "blue")
+                    marked = shared.mark_test_results(testPlanName, testSuiteName)
+                    common.printMsg(marked.stdout, "blue")
+
                     report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
@@ -175,9 +184,12 @@
                       "--testrail-add-missing-cases",
 //                       "--testrail_configuration_name tcp-qa",
                     ]
-                    ret = shared.upload_results_to_testrail(cvp_sanity_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
-                    common.printMsg(ret.stdout, "blue")
-                    report_url = ret.stdout.split("\n").each {
+                    uploaded = shared.upload_results_to_testrail(cvp_sanity_report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    common.printMsg(uploaded.stdout, "blue")
+                    marked = shared.mark_test_results(testPlanName, testSuiteName)
+                    common.printMsg(marked.stdout, "blue")
+
+                    report_url = uploaded.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
@@ -187,6 +199,7 @@
                 }
             }
 
+
             // Check if there were any exceptions during reporting
             if (exception_message) {
                 throw new Exception(exception_message)
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index 6b5f82c..ccb0163 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -680,13 +680,11 @@
     writeFile(file: filename, text: script, encoding: "UTF-8")
 }
 
-def upload_results_to_testrail(report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
+def upload_results_to_testrail(report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
   def venvPath = '/home/jenkins/venv_testrail_reporter'
   def testPlanDesc = env.LAB_CONFIG_NAME
   def testrailURL = "https://mirantis.testrail.com"
   def testrailProject = "Mirantis Cloud Platform"
-  def testPlanNamePrefix = env.TEST_PLAN_NAME_PREFIX ?: "[2019.2.0-update]System"
-  def testPlanName = env.TEST_PLAN_NAME ?: "${testPlanNamePrefix}-${MCP_VERSION}-${new Date().format('yyyy-MM-dd')}"
   def testrailMilestone = "MCP1.1"
   def testrailCaseMaxNameLenght = 250
   def jobURL = env.BUILD_URL
@@ -744,3 +742,38 @@
     def status = STATUS_MAP[result ?: 'FAILURE']   // currentBuild.result *must* be set at the finish of the try/catch
     create_xml_report(filename, classname, name, status, "Deploy components: ${deploy_expected_stacks}", text, '', '')
 }
+
+def mark_test_results(testPlanName, testSuiteName) {
+    def venvPath = '/home/jenkins/venv_testrail_analyzer'
+    def testrailURL = "https://mirantis.testrail.com"
+    def testrailProject = "Mirantis Cloud Platform"
+    def configName = env.LAB_CONFIG_NAME
+    def testRunName = "${configName} <${testSuiteName}>"
+
+    def script = """
+    . ${venvPath}/bin/activate
+    set -ex
+    export TESTRAIL_URL=${testrailURL}
+    testrail-reporter analyze -p "${testrailProject}" -t "${testPlanName}" -r "${testRunName}" checklist.yaml
+    """
+
+  def testrail_cred_id = params.TESTRAIL_CRED ?: 'testrail_system_tests'
+
+  withCredentials([
+             [$class          : 'UsernamePasswordMultiBinding',
+             credentialsId   : testrail_cred_id,
+             passwordVariable: 'TESTRAIL_PASSWORD',
+             usernameVariable: 'TESTRAIL_USER']
+  ]) {
+    def ret = [:]
+    ret.stdout = ''
+    ret.exception = ''
+    try {
+        ret.stdout = run_cmd_stdout(script)
+    } catch (Exception ex) {
+        ret.exception = ("""\
+##### Report to '${testRunName}' failed: #####\n""" + ex.message + "\n\n")
+    }
+    return ret
+  }
+}
\ No newline at end of file
diff --git a/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh b/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
index eb83ab4..cbf25d9 100644
--- a/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
+++ b/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
@@ -1,7 +1,9 @@
 #!/bin/bash -xe
 
+LC_ALL=en_US.UTF-8
 DEVOPS_VENV_PATH=/home/jenkins/fuel-devops30
 REPORT_VENV_PATH=/home/jenkins/venv_testrail_reporter
+TESTMARKER_VENV_PATH=/home/jenkins/venv_testrail_analyzer
 
 if [ ! -d ${DEVOPS_VENV_PATH} ]; then
     virtualenv ${DEVOPS_VENV_PATH}
@@ -9,15 +11,25 @@
 if [ ! -d ${REPORT_VENV_PATH} ]; then
     virtualenv ${REPORT_VENV_PATH}
 fi
+if [ ! -d ${TESTMARKER_VENV_PATH} ]; then
+    virtualenv --python=python3 ${TESTMARKER_VENV_PATH}
+fi
 
 # Install tcp-qa requirements
 . ${DEVOPS_VENV_PATH}/bin/activate
 pip install -r https://raw.githubusercontent.com/Mirantis/tcp-qa/master/tcp_tests/requirements.txt
 pip install psycopg2  # workaround for setup with PostgreSQL , to keep requirements.txt for Sqlite3 only
+deactivate
 
 # Install xunit2testrail
 . ${REPORT_VENV_PATH}/bin/activate
 #pip install xunit2testrail -U
 pip install git+https://github.com/dis-xcom/testrail_reporter -U  # Removed accessing to an unexisting pastebin on srv62
+deactivate
+
+# Install testRail analyzer
+. ${TESTMARKER_VENV_PATH}/bin/activate
+pip install git+https://github.com/ibumarskov/testrail-reporter -U
+deactivate
 
 chown -R jenkins:jenkins /home/jenkins/