Merge "Report results in proper order"
diff --git a/jobs/pipelines/deploy-cicd-and-run-tests.groovy b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
index 567ddc6..bf447eb 100644
--- a/jobs/pipelines/deploy-cicd-and-run-tests.groovy
+++ b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
@@ -38,6 +38,11 @@
             } else {
                 throw new Exception("Unknow env_manager: '${env_manager}'")
             }
+
+            if (fileExists("jenkins_agent_description.txt")) {
+                def String jenkins_agent_description = readFile("jenkins_agent_description.txt")
+                currentBuild.description += "${jenkins_agent_description}"
+            }
         }
 
         stage("Install core infrastructure and deploy CICD nodes") {
@@ -135,36 +140,10 @@
             }
         }
 
-        if (fileExists("jenkins_agent_description.txt")) {
-            def String jenkins_agent_description = readFile("jenkins_agent_description.txt")
-            currentBuild.description += "${jenkins_agent_description}"
-
-            // if there is a separated foundation node on $jenkins_slave_node_name,
-            // then archive artifacts also on that node
-            if (jenkins_slave_node_name != env.NODE_NAME) {
-                node ("${jenkins_slave_node_name}") {
-                    stage("Archive all xml reports from node ${jenkins_slave_node_name}") {
-                        archiveArtifacts artifacts: "**/*.xml,**/*.ini,**/*.log,**/*.tar.gz"
-                    }
-                    if ("${env.REPORT_TO_TESTRAIL}" != "false") {
-                        stage("report results to testrail") {
-                            common.printMsg("Running on: " + node_with_reports, "blue")
-                            shared.swarm_testrail_report(steps, node_with_reports)
-                    }
-                        stage("Store TestRail reports to job description from ${jenkins_slave_node_name}") {
-                            if (fileExists("description.txt")) {
-                                def String description  = readFile("description.txt")
-                                currentBuild.description += "${description}"
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
         stage("Archive all xml reports") {
             archiveArtifacts artifacts: "**/*.xml,**/*.ini,**/*.log,**/*.tar.gz"
         }
+
         if ("${env.REPORT_TO_TESTRAIL}" != "false") {
             stage("report results to testrail from jenkins master") {
                 common.printMsg("Running on: " + node_with_reports, "blue")
@@ -182,6 +161,31 @@
                 }
             }
         }
+
+        if (fileExists("jenkins_agent_description.txt")) {
+            // if there is a separated foundation node on $jenkins_slave_node_name,
+            // then archive artifacts also on that node
+            if (jenkins_slave_node_name != env.NODE_NAME) {
+                node ("${jenkins_slave_node_name}") {
+                    stage("Archive all xml reports from node ${jenkins_slave_node_name}") {
+                        archiveArtifacts artifacts: "**/*.xml,**/*.ini,**/*.log,**/*.tar.gz"
+                    }
+                    if ("${env.REPORT_TO_TESTRAIL}" != "false") {
+                        stage("report results to testrail") {
+                            common.printMsg("Running on: " + node_with_reports, "blue")
+                            shared.swarm_testrail_report(steps, node_with_reports)
+                        }
+                        stage("Store TestRail reports to job description from ${jenkins_slave_node_name}") {
+                            if (fileExists("description.txt")) {
+                                def String description  = readFile("description.txt")
+                                currentBuild.description += "${description}"
+                            }
+                        }
+                    }
+                } // node
+            }
+        }
+
     } // try
   } // node
 
diff --git a/jobs/pipelines/swarm-testrail-report.groovy b/jobs/pipelines/swarm-testrail-report.groovy
index 38a7968..e754936 100644
--- a/jobs/pipelines/swarm-testrail-report.groovy
+++ b/jobs/pipelines/swarm-testrail-report.groovy
@@ -33,6 +33,7 @@
     }
     dir("${PARENT_WORKSPACE}") {
         def description = ''
+        def exception_message = ''
         try {
 
             if (env.TCP_QA_REFS) {
@@ -47,7 +48,6 @@
             def testrail_name_template = ''
             def reporter_extra_options = []
 
-            def report_result = ''
             def report_url = ''
 
             //  deployment_report_name = "deployment_${ENV_NAME}.xml"
@@ -83,14 +83,15 @@
                       "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
                       "--testrail-case-section-name \'All\'",
                     ]
-                    report_result = shared.upload_results_to_testrail(deployment_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
-                    common.printMsg(report_result, "blue")
-                    report_url = report_result.split("\n").each {
+                    ret = shared.upload_results_to_testrail(deployment_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    common.printMsg(ret.stdout, "blue")
+                    report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
                         }
                     }
+                    exception_message += ret.exception
                 }
             }
 
@@ -104,14 +105,15 @@
                       "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
                       "--testrail-case-section-name \'All\'",
                     ]
-                    report_result = shared.upload_results_to_testrail(tcpqa_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
-                    common.printMsg(report_result, "blue")
-                    report_url = report_result.split("\n").each {
+                    ret = shared.upload_results_to_testrail(tcpqa_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    common.printMsg(ret.stdout, "blue")
+                    report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
                         }
                     }
+                    exception_message += ret.exception
                 }
             }
 
@@ -120,14 +122,15 @@
                     testSuiteName = env.TEMPEST_TEST_SUITE_NAME
                     methodname = "{classname}.{methodname}"
                     testrail_name_template = "{title}"
-                    report_result = shared.upload_results_to_testrail(tempest_report_name, testSuiteName, methodname, testrail_name_template)
-                    common.printMsg(report_result, "blue")
-                    report_url = report_result.split("\n").each {
+                    ret = shared.upload_results_to_testrail(tempest_report_name, testSuiteName, methodname, testrail_name_template)
+                    common.printMsg(ret.stdout, "blue")
+                    report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
                         }
                     }
+                    exception_message += ret.exception
                 }
             }
 
@@ -146,14 +149,15 @@
                       "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
                       "--testrail-case-section-name \'Conformance\'",
                     ]
-                    report_result = shared.upload_results_to_testrail(k8s_conformance_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
-                    common.printMsg(report_result, "blue")
-                    report_url = report_result.split("\n").each {
+                    ret = shared.upload_results_to_testrail(k8s_conformance_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    common.printMsg(ret.stdout, "blue")
+                    report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
                         }
                     }
+                    exception_message += ret.exception
                 }
             }
 
@@ -168,14 +172,15 @@
                       "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
                       "--testrail-case-section-name \'Conformance\'",
                     ]
-                    report_result = shared.upload_results_to_testrail(k8s_conformance_virtlet_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
-                    common.printMsg(report_result, "blue")
-                    report_url = report_result.split("\n").each {
+                    ret = shared.upload_results_to_testrail(k8s_conformance_virtlet_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    common.printMsg(ret.stdout, "blue")
+                    report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
                         }
                     }
+                    exception_message += ret.exception
                 }
             }
 
@@ -184,14 +189,15 @@
                     testSuiteName = "LMA2.0_Automated"
                     methodname = "{methodname}"
                     testrail_name_template = "{title}"
-                    report_result = shared.upload_results_to_testrail(stacklight_report_name, testSuiteName, methodname, testrail_name_template)
-                    common.printMsg(report_result, "blue")
-                    report_url = report_result.split("\n").each {
+                    ret = shared.upload_results_to_testrail(stacklight_report_name, testSuiteName, methodname, testrail_name_template)
+                    common.printMsg(ret.stdout, "blue")
+                    report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
                         }
                     }
+                    exception_message += ret.exception
                 }
             }
 
@@ -206,17 +212,23 @@
                       "--testrail-case-custom-fields {\\\"custom_qa_team\\\":\\\"9\\\"}",
                       "--testrail-case-section-name \'All\'",
                     ]
-                    report_result = shared.upload_results_to_testrail(cvp_sanity_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
-                    common.printMsg(report_result, "blue")
-                    report_url = report_result.split("\n").each {
+                    ret = shared.upload_results_to_testrail(cvp_sanity_report_name, testSuiteName, methodname, testrail_name_template, reporter_extra_options)
+                    common.printMsg(ret.stdout, "blue")
+                    report_url = ret.stdout.split("\n").each {
                         if (it.contains("[TestRun URL]")) {
                             common.printMsg("Found report URL: " + it.trim().split().last(), "blue")
                             description += "<a href=" + it.trim().split().last() + ">${testSuiteName}</a><br>"
                         }
                     }
+                    exception_message += ret.exception
                 }
             }
 
+            // Check if there were any exceptions during reporting
+            if (exception_message) {
+                throw new Exception(exception_message)
+            }
+
         } catch (e) {
             common.printMsg("Job is failed", "purple")
             throw e
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index 300aad5..1380668 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -705,7 +705,16 @@
              passwordVariable: 'TESTRAIL_PASSWORD',
              usernameVariable: 'TESTRAIL_USER']
   ]) {
-    return run_cmd_stdout(script)
+    def ret = [:]
+    ret.stdout = ''
+    ret.exception = ''
+    try {
+        ret.stdout = run_cmd_stdout(script)
+    } catch (Exception ex) {
+        ret.exception = ("""\
+##### Report to '${testSuiteName}' failed: #####\n""" + ex.message + "\n\n")
+    }
+    return ret
   }
 }