Merge "Extend and document Http.restCall better"
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index b2c8d8a..87b1696 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -619,7 +619,7 @@
**/
def comparePillars(compRoot, b_url, grepOpts) {
- common = new com.mirantis.mk.Common()
+
// Some global constants. Don't change\move them!
keyNew = 'new'
keyRemoved = 'removed'
@@ -643,7 +643,7 @@
returnStatus: true
)
if (grep_status == 1) {
- common.warningMsg("Grep regexp ${grepOpts} removed all diff!")
+ warningMsg("Grep regexp ${grepOpts} removed all diff!")
diff_status = 0
}
}
@@ -654,14 +654,14 @@
// Analyse output file and prepare array with results
String data_ = readFile file: "${compRoot}/pillar.diff"
def diff_list = diffCheckMultidir(data_.split("\\r?\\n"))
- common.infoMsg(diff_list)
+ infoMsg(diff_list)
dir(compRoot) {
if (diff_list[keyDiff].size() > 0) {
if (!fileExists('diff')) {
sh('mkdir -p diff')
}
description += '<b>CHANGED</b><ul>'
- common.infoMsg('Changed items:')
+ infoMsg('Changed items:')
def stepsForParallel = [:]
stepsForParallel.failFast = true
diff_list[keyDiff].each {
@@ -885,3 +885,43 @@
parallel branches
}
}
+
+/**
+ * Ugly processing basic funcs with /etc/apt
+ * @param configYaml
+ * Example :
+ configYaml = '''
+ ---
+ distrib_revision: 'nightly'
+ aprConfD: |-
+ APT::Get::AllowUnauthenticated 'true';
+ repo:
+ mcp_saltstack:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/saltstack-2017.7/xenial xenial main"
+ pinning: |-
+ Package: libsodium18
+ Pin: release o=SaltStack
+ Pin-Priority: 50
+ '''
+ *
+ */
+
+def debianExtraRepos(configYaml) {
+ def config = readYaml text: configYaml
+ def distribRevision = config.get('distrib_revision', 'nightly')
+ if (config.get('repo', false)) {
+ for (String repo in config['repo'].keySet()) {
+ source = config['repo'][repo]['source'].replace('SUB_DISTRIB_REVISION', distribRevision)
+ warningMsg("Write ${source} > /etc/apt/sources.list.d/${repo}.list")
+ sh("echo '${source}' > /etc/apt/sources.list.d/${repo}.list")
+ // TODO implement pining
+ }
+ }
+ if (config.get('aprConfD', false)) {
+ for (String pref in config['aprConfD'].tokenize('\n')) {
+ warningMsg("Adding ${pref} => /etc/apt/apt.conf.d/99setupAndTestNode")
+ sh("echo '${pref}' >> /etc/apt/apt.conf.d/99setupAndTestNode")
+ }
+ sh('cat /etc/apt/apt.conf.d/99setupAndTestNode')
+ }
+}
diff --git a/src/com/mirantis/mk/Orchestrate.groovy b/src/com/mirantis/mk/Orchestrate.groovy
index 4962d50..265cfa0 100644
--- a/src/com/mirantis/mk/Orchestrate.groovy
+++ b/src/com/mirantis/mk/Orchestrate.groovy
@@ -626,8 +626,7 @@
}
// Run k8s master setup
- first_target = salt.getFirstMinion(master, "I@kubernetes:master ${extra_tgt}")
- salt.enforceState(master, "${first_target} ${extra_tgt}", 'kubernetes.master.setup')
+ salt.enforceState(master, "I@kubernetes:master ${extra_tgt}", 'kubernetes.master.setup')
// Restart kubelet
salt.runSaltProcessStep(master, "I@kubernetes:master ${extra_tgt}", 'service.restart', ['kubelet'])
diff --git a/src/com/mirantis/mk/Salt.groovy b/src/com/mirantis/mk/Salt.groovy
index 2273ebd..ca209be 100644
--- a/src/com/mirantis/mk/Salt.groovy
+++ b/src/com/mirantis/mk/Salt.groovy
@@ -1004,8 +1004,8 @@
* @param file File path to read (/etc/hosts for example)
*/
-def getFileContent(saltId, target, file) {
- result = cmdRun(saltId, target, "cat ${file}")
+def getFileContent(saltId, target, file, checkResponse = true, batch=null, output = true, saltArgs = []) {
+ result = cmdRun(saltId, target, "cat ${file}", checkResponse, batch, output, saltArgs)
return result['return'][0].values()[0].replaceAll('Salt command execution success','')
}
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index 78a8d6f..2d1a888 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -5,7 +5,7 @@
were tests successful or not.
* @param config - LinkedHashMap with configuration params:
* dockerHostname - (required) Hostname to use for Docker container.
- * formulasRevision - (optional) Revision of packages to use (default stable).
+ * formulasRevision - (optional) Revision of packages to use (default proposed).
* runCommands - (optional) Dict with closure structure of body required tests. For example:
* [ '001_Test': { sh("./run-some-test") }, '002_Test': { sh("./run-another-test") } ]
* Before execution runCommands will be sorted by key names. Alpabetical order is preferred.
@@ -45,7 +45,36 @@
]
def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ')
+ def defaultExtraReposYaml = '''
+---
+distrib_revision: 'nightly'
+aprConfD: |-
+ APT::Get::AllowUnauthenticated 'true';
+ APT::Get::Install-Suggests 'false';
+ APT::Get::Install-Recommends 'false';
+repo:
+ mcp_saltstack:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/saltstack-2017.7/xenial xenial main"
+ pinning: |-
+ Package: libsodium18
+ Pin: release o=SaltStack
+ Pin-Priority: 50
+
+ Package: *
+ Pin: release o=SaltStack
+ Pin-Priority: 1100
+ mcp_extra:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/extra/xenial xenial main"
+ ubuntu:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial main restricted universe"
+ ubuntu-upd:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial-updates main restricted universe"
+ ubuntu-sec:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial-security main restricted universe"
+'''
def img = docker.image(dockerImageName)
+ def extraReposYaml = config.get('extraReposYaml', defaultExtraReposYaml)
+
img.pull()
try {
@@ -55,17 +84,19 @@
// Currently, we don't have any other point to install
// runtime dependencies for tests.
if (baseRepoPreConfig) {
+ // Warning! POssible point of 'allow-downgrades' issue
+ // Probably, need to add such flag into apt.prefs
sh("""#!/bin/bash -xe
echo "Installing extra-deb dependencies inside docker:"
- echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
- echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
- echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
+ echo > /etc/apt/sources.list
rm -vf /etc/apt/sources.list.d/* || true
- echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list
- echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
- apt-get update
- apt-get install -y python-netaddr
""")
+ common.debianExtraRepos(extraReposYaml)
+ sh('''#!/bin/bash -xe
+ apt-get update
+ apt-get install -y python-netaddr reclass
+ ''')
+
}
runCommands.sort().each { command, body ->
common.warningMsg("Running command: ${command}")
@@ -111,6 +142,83 @@
}
/**
+ * Wrapper around setupDockerAndTest, to run checks against new Reclass version
+ * that current model is compatible with new Reclass.
+ *
+ * @param config - LinkedHashMap with configuration params:
+ * dockerHostname - (required) Hostname to use for Docker container.
+ * distribRevision - (optional) Revision of packages to use (default proposed).
+ * extraRepo - (optional) Extra repo to use to install new Reclass version. Has
+ * high priority on distribRevision
+ * targetNodes - (required) List nodes to check pillar data.
+ */
+def compareReclassVersions(config) {
+ def common = new com.mirantis.mk.Common()
+ def salt = new com.mirantis.mk.Salt()
+ common.infoMsg("Going to test new reclass for CFG node")
+ def distribRevision = config.get('distribRevision', 'proposed')
+ def venv = config.get('venv')
+ def extraRepo = config.get('extraRepo', '')
+ def extraRepoKey = config.get('extraRepoKey', '')
+ def targetNodes = config.get('targetNodes')
+ sh "rm -rf ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
+ sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
+ def configRun = [
+ 'formulasRevision': distribRevision,
+ 'dockerExtraOpts' : [
+ "-v /srv/salt/reclass:/srv/salt/reclass:ro",
+ "-v /etc/salt:/etc/salt:ro",
+ "-v /usr/share/salt-formulas/:/usr/share/salt-formulas/:ro"
+ ],
+ 'envOpts' : [
+ "WORKSPACE=${env.WORKSPACE}",
+ "NODES_LIST=${targetNodes.join(' ')}"
+ ],
+ 'runCommands' : [
+ '001_Update_Reclass_package' : {
+ sh('apt-get update && apt-get install -y reclass')
+ },
+ '002_Test_Reclass_Compatibility': {
+ sh('''
+ reclass-salt -b /srv/salt/reclass -t > ${WORKSPACE}/new/inventory || exit 1
+ for node in $NODES_LIST; do
+ reclass-salt -b /srv/salt/reclass -p $node > ${WORKSPACE}/new/$node || exit 1
+ done
+ ''')
+ }
+ ]
+ ]
+ if (extraRepo) {
+ // FIXME
+ configRun['runCommands']['0001_Additional_Extra_Repo_Passed'] = {
+ sh("""
+ echo "${extraRepo}" > /etc/apt/sources.list.d/mcp_extra.list
+ [ "${extraRepoKey}" ] && wget -O - ${extraRepoKey} | apt-key add -
+ """)
+ }
+ }
+ if (setupDockerAndTest(configRun)) {
+ common.infoMsg("New reclass version is compatible with current model: SUCCESS")
+ def inventoryOld = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -t", true, null, true).get("return")[0].values()[0]
+ // [0..-31] to exclude 'echo Salt command execution success' from output
+ writeFile(file: "${env.WORKSPACE}/old/inventory", text: inventoryOld[0..-31])
+ for (String node in targetNodes) {
+ def nodeOut = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -p ${node}", true, null, true).get("return")[0].values()[0]
+ writeFile(file: "${env.WORKSPACE}/old/${node}", text: nodeOut[0..-31])
+ }
+ def reclassDiff = common.comparePillars(env.WORKSPACE, env.BUILD_URL, '')
+ currentBuild.description = reclassDiff
+ if (reclassDiff != '<b>No job changes</b>') {
+ throw new RuntimeException("Pillars with new reclass version has been changed: FAILED")
+ } else {
+ common.infoMsg("Pillars not changed with new reclass version: SUCCESS")
+ }
+ } else {
+ throw new RuntimeException("New reclass version is not compatible with current model: FAILED")
+ }
+}
+
+/**
* Wrapper over setupDockerAndTest, to test CC model.
*
* @param config - dict with params:
@@ -124,7 +232,7 @@
* aptRepoUrl - (optional) package repository with salt formulas
* aptRepoGPG - (optional) GPG key for apt repository with formulas
* testContext - (optional) Description of test
- Return: true\exception
+ Return: true\exception
*/
def testNode(LinkedHashMap config) {
@@ -152,29 +260,20 @@
config['runCommands'] = [
'001_Clone_salt_formulas_scripts': {
- sh(script: 'git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
+ sh(script: 'git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
},
- '002_Prepare_something': {
+ '002_Prepare_something' : {
sh('''rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
''')
},
- // should be switched on packages later
- '003_Install_reclass': {
- sh('''for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
- sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
- -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${RECLASS_VERSION};
- done
- ''')
- },
-
- '004_Run_tests': {
+ '004_Run_tests' : {
def testTimeout = 40 * 60
timeout(time: testTimeout, unit: 'SECONDS') {
- sh('''#!/bin/bash
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
source_local_envs
@@ -185,13 +284,13 @@
cd /srv/salt/scripts
saltservice_restart''')
- sh('''#!/bin/bash
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
source_local_envs
saltmaster_init''')
- sh('''#!/bin/bash
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
verify_salt_minions''')