| package com.mirantis.mk |
| |
| /** |
| * |
| * Python functions |
| * |
| */ |
| |
| /** |
| * Install python virtualenv |
| * |
| * @param path Path to virtualenv |
| * @param python Version of Python (python/python3) |
| * @param reqs Environment requirements in list format |
| * @param reqs_path Environment requirements path in str format |
| */ |
| def setupVirtualenv(path, python = 'python2', reqs=[], reqs_path=null, clean=false, useSystemPackages=false) { |
| def common = new com.mirantis.mk.Common() |
| |
| def offlineDeployment = env.getEnvironment().containsKey("OFFLINE_DEPLOYMENT") && env["OFFLINE_DEPLOYMENT"].toBoolean() |
| def virtualenv_cmd = "virtualenv ${path} --python ${python}" |
| if (useSystemPackages){ |
| virtualenv_cmd += " --system-site-packages" |
| } |
| if (clean) { |
| common.infoMsg("Cleaning venv directory " + path) |
| sh("rm -rf \"${path}\"") |
| } |
| |
| if(offlineDeployment){ |
| virtualenv_cmd+=" --no-download" |
| } |
| common.infoMsg("[Python ${path}] Setup ${python} environment") |
| sh(returnStdout: true, script: virtualenv_cmd) |
| if(!offlineDeployment){ |
| try { |
| runVirtualenvCommand(path, "pip install -U setuptools pip") |
| } catch(Exception e) { |
| common.warningMsg("Setuptools and pip cannot be updated, you might be offline but OFFLINE_DEPLOYMENT global property not initialized!") |
| } |
| } |
| if (reqs_path==null) { |
| def args = "" |
| for (req in reqs) { |
| args = args + "${req}\n" |
| } |
| writeFile file: "${path}/requirements.txt", text: args |
| reqs_path = "${path}/requirements.txt" |
| } |
| runVirtualenvCommand(path, "pip install -r ${reqs_path}", true) |
| } |
| |
| /** |
| * Run command in specific python virtualenv |
| * |
| * @param path Path to virtualenv |
| * @param cmd Command to be executed |
| * @param silent dont print any messages (optional, default false) |
| */ |
| def runVirtualenvCommand(path, cmd, silent=false) { |
| def common = new com.mirantis.mk.Common() |
| |
| virtualenv_cmd = "set +x; . ${path}/bin/activate; ${cmd}" |
| if(!silent){ |
| common.infoMsg("[Python ${path}] Run command ${cmd}") |
| } |
| output = sh( |
| returnStdout: true, |
| script: virtualenv_cmd |
| ).trim() |
| return output |
| } |
| |
| |
| /** |
| * Install docutils in isolated environment |
| * |
| * @param path Path where virtualenv is created |
| */ |
| def setupDocutilsVirtualenv(path) { |
| requirements = [ |
| 'docutils', |
| ] |
| setupVirtualenv(path, 'python2', requirements) |
| } |
| |
| |
| @NonCPS |
| def loadJson(rawData) { |
| return new groovy.json.JsonSlurperClassic().parseText(rawData) |
| } |
| |
| /** |
| * Parse content from markup-text tables to variables |
| * |
| * @param tableStr String representing the table |
| * @param mode Either list (1st row are keys) or item (key, value rows) |
| * @param format Format of the table |
| */ |
| def parseTextTable(tableStr, type = 'item', format = 'rest', path = none) { |
| parserFile = "${env.WORKSPACE}/textTableParser.py" |
| parserScript = """import json |
| import argparse |
| from docutils.parsers.rst import tableparser |
| from docutils import statemachine |
| |
| def parse_item_table(raw_data): |
| i = 1 |
| pretty_raw_data = [] |
| for datum in raw_data: |
| if datum != "": |
| if datum[3] != ' ' and i > 4: |
| pretty_raw_data.append(raw_data[0]) |
| if i == 3: |
| pretty_raw_data.append(datum.replace('-', '=')) |
| else: |
| pretty_raw_data.append(datum) |
| i += 1 |
| parser = tableparser.GridTableParser() |
| block = statemachine.StringList(pretty_raw_data) |
| docutils_data = parser.parse(block) |
| final_data = {} |
| for line in docutils_data[2]: |
| key = ' '.join(line[0][3]).strip() |
| value = ' '.join(line[1][3]).strip() |
| if key != "": |
| try: |
| value = json.loads(value) |
| except: |
| pass |
| final_data[key] = value |
| i+=1 |
| return final_data |
| |
| def parse_list_table(raw_data): |
| i = 1 |
| pretty_raw_data = [] |
| for datum in raw_data: |
| if datum != "": |
| if datum[3] != ' ' and i > 4: |
| pretty_raw_data.append(raw_data[0]) |
| if i == 3: |
| pretty_raw_data.append(datum.replace('-', '=')) |
| else: |
| pretty_raw_data.append(datum) |
| i += 1 |
| parser = tableparser.GridTableParser() |
| block = statemachine.StringList(pretty_raw_data) |
| docutils_data = parser.parse(block) |
| final_data = [] |
| keys = [] |
| for line in docutils_data[1]: |
| for item in line: |
| keys.append(' '.join(item[3]).strip()) |
| for line in docutils_data[2]: |
| final_line = {} |
| key = ' '.join(line[0][3]).strip() |
| value = ' '.join(line[1][3]).strip() |
| if key != "": |
| try: |
| value = json.loads(value) |
| except: |
| pass |
| final_data[key] = value |
| i+=1 |
| return final_data |
| |
| def parse_list_table(raw_data): |
| i = 1 |
| pretty_raw_data = [] |
| for datum in raw_data: |
| if datum != "": |
| if datum[3] != ' ' and i > 4: |
| pretty_raw_data.append(raw_data[0]) |
| if i == 3: |
| pretty_raw_data.append(datum.replace('-', '=')) |
| else: |
| pretty_raw_data.append(datum) |
| i += 1 |
| parser = tableparser.GridTableParser() |
| block = statemachine.StringList(pretty_raw_data) |
| docutils_data = parser.parse(block) |
| final_data = [] |
| keys = [] |
| for line in docutils_data[1]: |
| for item in line: |
| keys.append(' '.join(item[3]).strip()) |
| for line in docutils_data[2]: |
| final_line = {} |
| i = 0 |
| for item in line: |
| value = ' '.join(item[3]).strip() |
| try: |
| value = json.loads(value) |
| except: |
| pass |
| final_line[keys[i]] = value |
| i += 1 |
| final_data.append(final_line) |
| return final_data |
| |
| def read_table_file(file): |
| table_file = open(file, 'r') |
| raw_data = table_file.read().split('\\n') |
| table_file.close() |
| return raw_data |
| |
| parser = argparse.ArgumentParser() |
| parser.add_argument('-f','--file', help='File with table data', required=True) |
| parser.add_argument('-t','--type', help='Type of table (list/item)', required=True) |
| args = vars(parser.parse_args()) |
| |
| raw_data = read_table_file(args['file']) |
| |
| if args['type'] == 'list': |
| final_data = parse_list_table(raw_data) |
| else: |
| final_data = parse_item_table(raw_data) |
| |
| print json.dumps(final_data) |
| """ |
| writeFile file: parserFile, text: parserScript |
| tableFile = "${env.WORKSPACE}/prettytable.txt" |
| writeFile file: tableFile, text: tableStr |
| |
| cmd = "python ${parserFile} --file '${tableFile}' --type ${type}" |
| if (path) { |
| rawData = runVirtualenvCommand(path, cmd) |
| } |
| else { |
| rawData = sh ( |
| script: cmd, |
| returnStdout: true |
| ).trim() |
| } |
| data = loadJson(rawData) |
| echo("[Parsed table] ${data}") |
| return data |
| } |
| |
| /** |
| * Install cookiecutter in isolated environment |
| * |
| * @param path Path where virtualenv is created |
| */ |
| def setupCookiecutterVirtualenv(path) { |
| requirements = [ |
| 'cookiecutter', |
| 'jinja2==2.8.1', |
| 'PyYAML==3.12' |
| ] |
| setupVirtualenv(path, 'python2', requirements) |
| } |
| |
| /** |
| * Generate the cookiecutter templates with given context |
| * |
| * @param template template |
| * @param context template context |
| * @param path Path where virtualenv is created (optional) |
| * @param templatePath path to cookiecutter template repo (optional) |
| */ |
| def buildCookiecutterTemplate(template, context, outputDir = '.', path = null, templatePath = ".") { |
| configFile = "default_config.yaml" |
| configString = "default_context:\n" |
| writeFile file: configFile, text: context |
| command = ". ${path}/bin/activate; if [ -f ${templatePath}/generate.py ]; then python ${templatePath}/generate.py --config-file ${configFile} --template ${template} --output-dir ${outputDir}; else cookiecutter --config-file ${configFile} --output-dir ${outputDir} --overwrite-if-exists --verbose --no-input ${template}; fi" |
| output = sh (returnStdout: true, script: command) |
| echo("[Cookiecutter build] Output: ${output}") |
| } |
| |
| /** |
| * Install jinja rendering in isolated environment |
| * |
| * @param path Path where virtualenv is created |
| */ |
| def setupJinjaVirtualenv(path) { |
| requirements = [ |
| 'jinja2-cli', |
| 'pyyaml', |
| ] |
| setupVirtualenv(path, 'python2', requirements) |
| } |
| |
| /** |
| * Generate the Jinja templates with given context |
| * |
| * @param path Path where virtualenv is created |
| */ |
| def jinjaBuildTemplate (template, context, path = none) { |
| contextFile = "jinja_context.yml" |
| contextString = "" |
| for (parameter in context) { |
| contextString = "${contextString}${parameter.key}: ${parameter.value}\n" |
| } |
| writeFile file: contextFile, text: contextString |
| cmd = "jinja2 ${template} ${contextFile} --format=yaml" |
| data = sh (returnStdout: true, script: cmd) |
| echo(data) |
| return data |
| } |
| |
| /** |
| * Install salt-pepper in isolated environment |
| * |
| * @param path Path where virtualenv is created |
| * @param url SALT_MASTER_URL |
| * @param credentialsId Credentials to salt api |
| */ |
| def setupPepperVirtualenv(path, url, credentialsId) { |
| def common = new com.mirantis.mk.Common() |
| |
| // virtualenv setup |
| // pin pepper till https://mirantis.jira.com/browse/PROD-18188 is fixed |
| requirements = ['salt-pepper>=0.5.2,<0.5.4'] |
| setupVirtualenv(path, 'python2', requirements, null, true, true) |
| |
| // pepperrc creation |
| rcFile = "${path}/pepperrc" |
| creds = common.getPasswordCredentials(credentialsId) |
| rc = """\ |
| [main] |
| SALTAPI_EAUTH=pam |
| SALTAPI_URL=${url} |
| SALTAPI_USER=${creds.username} |
| SALTAPI_PASS=${creds.password.toString()} |
| """ |
| writeFile file: rcFile, text: rc |
| return rcFile |
| } |
| |
| /** |
| * Install devops in isolated environment |
| * |
| * @param path Path where virtualenv is created |
| * @param clean Define to true is the venv have to cleaned up before install a new one |
| */ |
| def setupDevOpsVenv(venv, clean=false) { |
| requirements = ['git+https://github.com/openstack/fuel-devops.git'] |
| setupVirtualenv(venv, 'python2', requirements, null, false, clean) |
| } |