Rewrite managing of virtualenvs
Change-Id: Icd729b1ac66661f27dcaa8423fc2d72c45a027c7
diff --git a/jobs/pipelines/deploy-cicd-and-run-tests.groovy b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
index 8156eed..5106d80 100644
--- a/jobs/pipelines/deploy-cicd-and-run-tests.groovy
+++ b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
@@ -178,12 +178,6 @@
stage("report results to testrail from jenkins master") {
common.printMsg("Running on: " + node_with_reports, "blue")
common.printMsg("Running on: " + env.NODE_NAME, "blue")
- shared.verbose_sh("""\
- [ -d /home/jenkins/venv_testrail_reporter ] || virtualenv --python=python3.7 /home/jenkins/venv_testrail_reporter""", true, false, true)
- sshagent(credentials: ['maintenance-team-ssh']) {
- shared.run_cmd("""\
- . /home/jenkins/venv_testrail_reporter/bin/activate; pip install git+ssh://maintenance-ci-robot@gerrit.mcp.mirantis.com:29418/mcp/testrail-reporter -U""")
- }
shared.swarm_testrail_report(steps, env.NODE_NAME)
}
stage("Store TestRail reports to job description") {
diff --git a/jobs/pipelines/run-test-scenarios.groovy b/jobs/pipelines/run-test-scenarios.groovy
index fc45ad8..b83dcec 100644
--- a/jobs/pipelines/run-test-scenarios.groovy
+++ b/jobs/pipelines/run-test-scenarios.groovy
@@ -65,12 +65,6 @@
stage("report results to testrail from jenkins master") {
if ("${env.REPORT_TO_TESTRAIL}" != "false") {
common.infoMsg("Running on: " + env.PARENT_NODE_NAME, "blue")
- shared.verbose_sh("""\
- [ -d /home/jenkins/venv_testrail_reporter ] || virtualenv --python=python3.7 /home/jenkins/venv_testrail_reporter""", true, false, true)
- sshagent(credentials: ['maintenance-team-ssh']) {
- shared.run_cmd("""\
- . /home/jenkins/venv_testrail_reporter/bin/activate; pip install git+ssh://maintenance-ci-robot@gerrit.mcp.mirantis.com:29418/mcp/testrail-reporter -U""")
- }
shared.swarm_testrail_report(steps, env.PARENT_NODE_NAME)
} else {
common.infoMsg("REPORT_TO_TESTRAIL is set to FALSE. Skipped this step ")
diff --git a/jobs/pipelines/swarm-testrail-report.groovy b/jobs/pipelines/swarm-testrail-report.groovy
index 1167978..f9820f8 100644
--- a/jobs/pipelines/swarm-testrail-report.groovy
+++ b/jobs/pipelines/swarm-testrail-report.groovy
@@ -78,26 +78,6 @@
common.printMsg(stacklight_report_name ? "Found stacklight-pytest report: ${stacklight_report_name}" : "stacklight-pytest report not found", stacklight_report_name ? "blue" : "red")
common.printMsg(cvp_sanity_report_name ? "Found CVP Sanity report: ${cvp_sanity_report_name}" : "CVP Sanity report not found", cvp_sanity_report_name ? "blue" : "red")
- stage("Prepare/update venvs for reporter/analyzer tools"){
- sshagent(credentials: ['maintenance-team-ssh']) {
- shared.run_cmd("""\
- . /home/jenkins/venv_testrail_reporter/bin/activate;
- pip install git+ssh://maintenance-ci-robot@gerrit.mcp.mirantis.com:29418/mcp/testrail-reporter -U;
- pip install --force-reinstall --no-deps git+ssh://maintenance-ci-robot@gerrit.mcp.mirantis.com:29418/mcp/testrail-reporter;""")
- }
- shared.run_cmd("""\
- if [ -f /home/jenkins/venv_testrail_analyzer/bin/python &&
- $(/home/jenkins/venv_testrail_analyzer/bin/python -V | awk -F "." '{print \$2}') -lt 6 ] ; then
- rm -rf /home/jenkins/venv_testrail_analyzer ;
- fi
- """)
- shared.run_cmd("""\
- [ -d /home/jenkins/venv_testrail_analyzer ] || virtualenv --python=python3.7 /home/jenkins/venv_testrail_analyzer ;
- source /home/jenkins/venv_testrail_analyzer/bin/activate;
- pip install git+https://review.gerrithub.io/ibumarskov/testrail-reporter@b9041b241f3364022cd0ad63510f02aab4352a6b
- """)
- }
-
if (deployment_report_name) {
stage("Deployment report") {
testSuiteName = "[MCP] Integration automation"
diff --git a/src/com/mirantis/system_qa/SharedPipeline.groovy b/src/com/mirantis/system_qa/SharedPipeline.groovy
index 8390c1b..efd8d49 100644
--- a/src/com/mirantis/system_qa/SharedPipeline.groovy
+++ b/src/com/mirantis/system_qa/SharedPipeline.groovy
@@ -686,8 +686,34 @@
writeFile(file: filename, text: script, encoding: "UTF-8")
}
+
+def withVirtualEnv(name, pythonVersion, installRepo = "", command = "") {
+ /// name - str, name of virtualenv, name of folder which will be located in /home/jenkins
+ /// pythonVersion - str, version of python defined through the ShiningPanda settings in the Jenkins (Jenkins ->
+ //Manage Jenkins -> Global Tool Configuration -> Python Installations)
+ /// installRepo - repo with tool which should be installed in that virtualenv
+ def homeDir = "/home/jenkins"
+ venvPath = "${homeDir}/${name}"
+ // check if venv exists
+
+ withPythonEnv("${pythonVersion}") {
+ if (!fileExists(venvPath)) {
+ sh """
+ virtualenv --python=\$(which python) ${venvPath}
+ . ${venvPath}/bin/activate
+ pip install -U ${installRepo}
+ """
+ }
+ sh """
+ . ${venvPath}/bin/activate
+ ${command}
+ deactivate
+ """
+ }
+}
+
+
def upload_results_to_testrail(report_name, testPlanName, testSuiteName, methodname, testrail_name_template, reporter_extra_options=[]) {
- def venvPath = '/home/jenkins/venv_testrail_reporter'
def testPlanDesc = env.LAB_CONFIG_NAME
def testrailURL = "https://mirantis.testrail.com"
def testrailProject = "Mirantis Cloud Platform"
@@ -713,8 +739,6 @@
] + reporter_extra_options
def script = """
- . ${venvPath}/bin/activate
- set -ex
report ${reporterOptions.join(' ')} ${report_name}
"""
@@ -726,17 +750,26 @@
passwordVariable: 'TESTRAIL_PASSWORD',
usernameVariable: 'TESTRAIL_USER']
]) {
- def ret = [:]
- ret.stdout = ''
- ret.exception = ''
- try {
- ret.stdout = run_cmd_stdout(script)
- } catch (Exception ex) {
- ret.exception = ("""\
-##### Report to '${testSuiteName}' failed: #####\n""" + ex.message + "\n\n")
- }
- return ret
- }
+
+ def ret = [:]
+ ret.stdout = ''
+ ret.exception = ''
+ try {
+ sshagent(credentials: ['maintenance-team-ssh']) {
+ withVirtualEnv(
+ "venv_testrail_reporter",
+ "python3.7",
+ "git+ssh://maintenance-ci-robot@gerrit.mcp.mirantis.com:29418/mcp/testrail-reporter",
+ script
+ )}
+// ret.stdout = run_cmd_stdout(script)
+ } catch (Exception ex) {
+ ret.exception = ("""\
+ ##### Report to '${testSuiteName}' failed: #####\n""" + ex.message + "\n\n")
+ }
+
+ return ret
+ }
}
@@ -750,15 +783,12 @@
}
def mark_test_results(testPlanName, testSuiteName) {
- def venvPath = '/home/jenkins/venv_testrail_analyzer'
def testrailURL = "https://mirantis.testrail.com"
def testrailProject = "Mirantis Cloud Platform"
def configName = env.LAB_CONFIG_NAME
def testRunName = "${configName} <${testSuiteName}>"
def script = """
- . ${venvPath}/bin/activate
- set -ex
export TESTRAIL_URL=${testrailURL}
testrail-reporter analyze -p "${testrailProject}" -t "${testPlanName}" -r "${testRunName}" checklist.yaml
"""
@@ -775,7 +805,11 @@
ret.stdout = ''
ret.exception = ''
try {
- ret.stdout = run_cmd_stdout(script)
+ withVirtualEnv("venv_testrail_analyzer",
+ "python3.7",
+ "git+https://review.gerrithub.io/ibumarskov/testrail-reporter",
+ script)
+// ret.stdout = run_cmd_stdout(script)
} catch (Exception ex) {
ret.exception = ("""\
##### Report to '${testRunName}' failed: #####\n""" + ex.message + "\n\n")
@@ -783,6 +817,7 @@
return ret
}
}
+
def reboot_hardware_nodes(creds_name) {
bm_ips_file = readFile("tcp_tests/templates/${creds_name}")
bm_ips = bm_ips_file.split("\n")
diff --git a/tcp_tests/templates/_heat_environments/eu-cloud-low-flavors.env b/tcp_tests/templates/_heat_environments/eu-cloud-low-flavors.env
index abbe367..5c65579 100644
--- a/tcp_tests/templates/_heat_environments/eu-cloud-low-flavors.env
+++ b/tcp_tests/templates/_heat_environments/eu-cloud-low-flavors.env
@@ -44,5 +44,5 @@
vm_availability_zone: nova
net_public: public
- foundation_image: system.foundation01
+ foundation_image: system.foundation02
nameservers: 172.18.176.6,172.18.224.6
diff --git a/tcp_tests/templates/_heat_environments/eu-cloud.env b/tcp_tests/templates/_heat_environments/eu-cloud.env
index d5e0e5a..463f000 100644
--- a/tcp_tests/templates/_heat_environments/eu-cloud.env
+++ b/tcp_tests/templates/_heat_environments/eu-cloud.env
@@ -48,5 +48,5 @@
vm_availability_zone: nova
net_public: public
- foundation_image: system.foundation01
+ foundation_image: system.foundation02
nameservers: 172.18.176.6,172.18.224.6
diff --git a/tcp_tests/templates/_heat_environments/us-cloud-low-flavors.env b/tcp_tests/templates/_heat_environments/us-cloud-low-flavors.env
index 1af6209..dd35a4b 100644
--- a/tcp_tests/templates/_heat_environments/us-cloud-low-flavors.env
+++ b/tcp_tests/templates/_heat_environments/us-cloud-low-flavors.env
@@ -44,5 +44,5 @@
vm_availability_zone: nova
net_public: public
- foundation_image: system.foundation01
+ foundation_image: system.foundation02
nameservers: 172.18.224.6,172.18.176.6
diff --git a/tcp_tests/templates/_heat_environments/us-cloud.env b/tcp_tests/templates/_heat_environments/us-cloud.env
index 44c79dd..d216a1f 100644
--- a/tcp_tests/templates/_heat_environments/us-cloud.env
+++ b/tcp_tests/templates/_heat_environments/us-cloud.env
@@ -48,5 +48,5 @@
vm_availability_zone: nova
net_public: public
- foundation_image: system.foundation01
+ foundation_image: system.foundation02
nameservers: 172.18.224.6,172.18.176.6
diff --git a/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh b/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
index 70cd37f..d3b2f4a 100644
--- a/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
+++ b/tcp_tests/templates/_packer/scripts/jenkins_virtualenvs.sh
@@ -2,8 +2,6 @@
LC_ALL=en_US.UTF-8
DEVOPS_VENV_PATH=/home/jenkins/fuel-devops30
-REPORT_VENV_PATH=/home/jenkins/venv_testrail_reporter
-TESTMARKER_VENV_PATH=/home/jenkins/venv_testrail_analyzer
if [ ! "$CHANGE_RIGHTS" ]; then
CHANGE_RIGHTS=true
@@ -25,19 +23,6 @@
pip install psycopg2 # workaround for setup with PostgreSQL , to keep requirements.txt for Sqlite3 only
deactivate
-# Install xunit2testrail
-. ${REPORT_VENV_PATH}/bin/activate
-#pip install xunit2testrail -U
-pip install git+https://github.com/dis-xcom/testrail_reporter -U # Removed accessing to an unexisting pastebin on srv62
-deactivate
-
-# Install testRail analyzer
-. ${TESTMARKER_VENV_PATH}/bin/activate
-#pip install git+https://github.com/ibumarskov/testrail-reporter -U
-# Pull from review to test changes in testrail-reporter before their merging
-pip install git+https://review.gerrithub.io/ibumarskov/testrail-reporter@refs/changes/94/514594/8
-deactivate
-
if [ "$CHANGE_RIGHTS" = true ]; then
chown -R jenkins:jenkins /home/jenkins/
fi
diff --git a/tcp_tests/templates/_packer/scripts/ubuntu_packets.sh b/tcp_tests/templates/_packer/scripts/ubuntu_packets.sh
index 2e59834..cd29a30 100644
--- a/tcp_tests/templates/_packer/scripts/ubuntu_packets.sh
+++ b/tcp_tests/templates/_packer/scripts/ubuntu_packets.sh
@@ -16,6 +16,9 @@
# update kernel
apt-get install -y linux-generic-hwe-16.04
+
+
+
#install python3.7
cd /usr/src
wget https://www.python.org/ftp/python/3.7.9/Python-3.7.9.tgz
@@ -23,3 +26,14 @@
cd Python-3.7.9
./configure --enable-optimizations
make altinstall
+
+# install Pyenv
+apt-get install -y build-essential git libreadline-dev zlib1g-dev libssl-dev libbz2-dev libsqlite3-dev
+export PYENV_ROOT=/usr/local/.pyenv/
+curl https://pyenv.run | bash
+command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
+ln -s $(which pyenv) /usr/local/bin/pyenv
+pyenv install 3.7.12
+pyenv install 3.8.12
+pyenv install 3.9.12
+chown -R jenkins:jenkins $PYENV_ROOT
\ No newline at end of file