Add tools to run jenkins jobs and remote commands
- ./tcp_tests/utils/create_devops_env.py
Creates a fuel-devops enviromnet with VMs in disabled
state, to generate networks and addresses for inventory.
Required parameters:
export ENV_NAME=test
export LAB_CONFIG_NAME=<template directory with underlay.yml>
export MANAGER=devops
Other parameters may be required for the underlay.yml
CLI example:
export PYTHONPATH=$(pwd)
python ./tcp_tests/utils/create_devops_env.py
- ./tcp_tests/utils/run_jenkins_job.py
Run a jenkins job with parameters, wait for completion,
print the console output to stdout while waiting.
Required parameters:
export JENKINS_URL=http://host:port/
export JENKINS_USER=admin
export JENKINS_PASS=admin
CLI example:
JOB_PARAMETERS="{
\"SALT_MASTER_URL\": \"${SALTAPI_URL}\",
\"STACK_INSTALL\": \"core,cicd\"
}"
JOB_PREFIX="[ {job_name} #{build_number}:cicd {time} ] "
python ./tcp_tests/utils/run_jenkins_job.py \
--verbose \
--job-name=deploy_openstack \
--job-parameters="$JOB_PARAMETERS" \
--job-output-prefix="$JOB_PREFIX"
- ./tcp_tests/utils/get_param.py
Get a single parameter from the salt pillar.
Useful to get addresses and other scalar values.
Required parameters are the same as for 'pepper' CLI:
export SALTAPI_URL=http://${SALT_MASTER_IP}:6969/
export SALTAPI_USER='salt'
export SALTAPI_PASS='icecream12345!'
export SALTAPI_EAUTH='pam'
CLI example:
export JENKINS_HOST=$(./tcp_tests/utils/get_param.py \
-C 'I@docker:client:stack:jenkins' \
pillar.get jenkins:client:master:host)
- ./tcp_tests/utils/run_template_commands.py
Run remote commands from the ./tcp_tests/templates/
No environment varialbes are required, but may be
useful to provide the INI config from some completed
deployment.
CLI example:
export TESTS_CONFIGS=$(pwd)/test_salt_deployed.ini
./tcp_tests/utils/run_template_commands.py \
./tcp_tests/templates/<lab_name>/common_services.yaml
- some env files for sourcing to get access to different APIs.
This will simplify using the scripts above.
. ./tcp_tests/utils/env_salt # salt-api access
. ./tcp_tests/utils/env_jenkins_day01 # jenkins on salt-master
. ./tcp_tests/utils/env_jenkins_cicd # jenkins on cicd
. ./tcp_tests/utils/env_k8s # k8s api access
- fixed UnderlayManager.sudo_check_call() to remove
deprecation warning.
Improvements to JenkisClient:
- Add JenkinsWrapper class to workaround the bug
https://bugs.launchpad.net/python-jenkins/+bug/1775047
which is happened to CICD Jenkins behind the haproxy
- improved waiting for start of the job in run_build()
- new argument 'interval' in wait_end_of_build(), to set
the polling interval while waiting the job
- new argument 'job_output_prefix' in wait_end_of_build(),
which allows to set the prefix to each line of the console
output of the job; with some pre-defined template keys.
- improved printing the job output in case of non-unicode characters
Change-Id: Ie7d1324d8247e55ba9c0f0492ca39fc176ff4935
diff --git a/tcp_tests/managers/jenkins/client.py b/tcp_tests/managers/jenkins/client.py
index 0b0d7de..2628e7a 100644
--- a/tcp_tests/managers/jenkins/client.py
+++ b/tcp_tests/managers/jenkins/client.py
@@ -1,4 +1,5 @@
from __future__ import print_function
+import time
import jenkins
import requests
@@ -6,13 +7,40 @@
from devops.helpers import helpers
+class JenkinsWrapper(jenkins.Jenkins):
+ """Workaround for the bug:
+ https://bugs.launchpad.net/python-jenkins/+bug/1775047
+ """
+ def _response_handler(self, response):
+ '''Handle response objects'''
+
+ # raise exceptions if occurred
+ response.raise_for_status()
+
+ headers = response.headers
+ if (headers.get('content-length') is None and
+ headers.get('transfer-encoding') is None and
+ (response.status_code == 201 and
+ headers.get('location') is None) and
+ (response.content is None or len(response.content) <= 0)):
+ # response body should only exist if one of these is provided
+ raise jenkins.EmptyResponseException(
+ "Error communicating with server[%s]: "
+ "empty response" % self.server)
+
+ # Response objects will automatically return unicode encoded
+ # when accessing .text property
+ return response
+
+
class JenkinsClient(object):
def __init__(self, host=None, username=None, password=None):
host = host or 'http://172.16.44.33:8081'
username = username or 'admin'
password = password or 'r00tme'
- self.__client = jenkins.Jenkins(
+ # self.__client = jenkins.Jenkins(
+ self.__client = JenkinsWrapper(
host,
username=username,
password=password)
@@ -50,62 +78,92 @@
def run_build(self, name, params=None, timeout=600, verbose=False):
params = params or self.make_defults_params(name)
num = self.__client.build_job(name, params)
+ time.sleep(2) # wait while job is started
def is_blocked():
queued = self.__client.get_queue_item(num)
status = not queued['blocked']
if not status and verbose:
print("pending the job [{}] : {}".format(name, queued['why']))
- return status
+ return (status and
+ 'executable' in (queued or {}) and
+ 'number' in (queued['executable'] or {}))
helpers.wait(
is_blocked,
timeout=timeout,
interval=30,
timeout_msg='Timeout waiting to run the job [{}]'.format(name))
-
build_id = self.__client.get_queue_item(num)['executable']['number']
return name, build_id
- def wait_end_of_build(self, name, build_id, timeout=600,
- verbose=False):
+ def wait_end_of_build(self, name, build_id, timeout=600, interval=5,
+ verbose=False, job_output_prefix=''):
+ '''Wait until the specified build is finished
+
+ :param name: ``str``, job name
+ :param build_id: ``int``, build id
+ :param timeout: ``int``, timeout waiting the job, sec
+ :param interval: ``int``, interval of polling the job result, sec
+ :param verbose: ``bool``, print the job console updates during waiting
+ :param job_output_prefix: ``str``, print the prefix for each console
+ output line, with the pre-defined
+ substitution keys:
+ - '{name}' : the current job name
+ - '{build_id}' : the current build-id
+ - '{time}' : the current time
+ :returns: requests object with headers and console output, ``obj``
+ '''
start = [0]
+ time_str = time.strftime("%H:%M:%S")
+ prefix = "\n" + job_output_prefix.format(job_name=name,
+ build_number=build_id,
+ time=time_str)
+ if verbose:
+ print(prefix, end='')
def building():
status = not self.build_info(name, build_id)['building']
if verbose:
+ time_str = time.strftime("%H:%M:%S")
+ prefix = "\n" + job_output_prefix.format(
+ job_name=name, build_number=build_id, time=time_str)
res = self.get_progressive_build_output(name,
build_id,
start=start[0])
if 'X-Text-Size' in res.headers:
text_size = int(res.headers['X-Text-Size'])
if start[0] < text_size:
- print(res.content, end='')
+ text = res.content.decode('utf-8',
+ errors='backslashreplace')
+ print(text.replace("\n", prefix), end='')
start[0] = text_size
return status
helpers.wait(
building,
timeout=timeout,
+ interval=interval,
timeout_msg='Timeout waiting, job {0} are not finished "{1}" build'
' still'.format(name, build_id))
def get_build_output(self, name, build_id):
return self.__client.get_build_console_output(name, build_id)
- def get_progressive_build_output(self, name, build_id, start=0,
- raise_on_err=False):
+ def get_progressive_build_output(self, name, build_id, start=0):
'''Get build console text.
:param name: Job name, ``str``
- :param name: Build id, ``int``
- :param name: Start offset, ``int``
+ :param build_id: Build id, ``int``
+ :param start: Start offset, ``int``
:returns: requests object with headers and console output, ``obj``
'''
folder_url, short_name = self.__client._get_job_folder(name)
PROGRESSIVE_CONSOLE_OUTPUT = (
'%(folder_url)sjob/%(short_name)s/%(build_id)d/'
- 'logText/progressiveHtml?start=%(start)d')
- url = self.__client._build_url(PROGRESSIVE_CONSOLE_OUTPUT, locals())
- return(requests.get(url))
+ 'logText/progressiveText?start=%(start)d')
+ req = requests.Request(
+ 'GET',
+ self.__client._build_url(PROGRESSIVE_CONSOLE_OUTPUT, locals()))
+ return(self.__client.jenkins_request(req))
diff --git a/tcp_tests/managers/underlay_ssh_manager.py b/tcp_tests/managers/underlay_ssh_manager.py
index 44c8830..495e51d 100644
--- a/tcp_tests/managers/underlay_ssh_manager.py
+++ b/tcp_tests/managers/underlay_ssh_manager.py
@@ -357,7 +357,7 @@
"""
remote = self.remote(node_name=node_name, host=host,
address_pool=address_pool)
- with remote.get_sudo(remote):
+ with remote.sudo(enforce=True):
return remote.check_call(
command=cmd, verbose=verbose, timeout=timeout,
error_info=error_info, expected=expected,