Implement and switch docker-test to use bin-reclass
* Add common.debianExtraRepos function
* Misc: drop duplicate import common from comparePillars
Closes-Bug: PROD-22476 (PROD:22476)
Change-Id: Idf6ffc57502fc9d756b3bfd489f666ca0165b145
diff --git a/src/com/mirantis/mk/Common.groovy b/src/com/mirantis/mk/Common.groovy
index b2c8d8a..87b1696 100644
--- a/src/com/mirantis/mk/Common.groovy
+++ b/src/com/mirantis/mk/Common.groovy
@@ -619,7 +619,7 @@
**/
def comparePillars(compRoot, b_url, grepOpts) {
- common = new com.mirantis.mk.Common()
+
// Some global constants. Don't change\move them!
keyNew = 'new'
keyRemoved = 'removed'
@@ -643,7 +643,7 @@
returnStatus: true
)
if (grep_status == 1) {
- common.warningMsg("Grep regexp ${grepOpts} removed all diff!")
+ warningMsg("Grep regexp ${grepOpts} removed all diff!")
diff_status = 0
}
}
@@ -654,14 +654,14 @@
// Analyse output file and prepare array with results
String data_ = readFile file: "${compRoot}/pillar.diff"
def diff_list = diffCheckMultidir(data_.split("\\r?\\n"))
- common.infoMsg(diff_list)
+ infoMsg(diff_list)
dir(compRoot) {
if (diff_list[keyDiff].size() > 0) {
if (!fileExists('diff')) {
sh('mkdir -p diff')
}
description += '<b>CHANGED</b><ul>'
- common.infoMsg('Changed items:')
+ infoMsg('Changed items:')
def stepsForParallel = [:]
stepsForParallel.failFast = true
diff_list[keyDiff].each {
@@ -885,3 +885,43 @@
parallel branches
}
}
+
+/**
+ * Ugly processing basic funcs with /etc/apt
+ * @param configYaml
+ * Example :
+ configYaml = '''
+ ---
+ distrib_revision: 'nightly'
+ aprConfD: |-
+ APT::Get::AllowUnauthenticated 'true';
+ repo:
+ mcp_saltstack:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/saltstack-2017.7/xenial xenial main"
+ pinning: |-
+ Package: libsodium18
+ Pin: release o=SaltStack
+ Pin-Priority: 50
+ '''
+ *
+ */
+
+def debianExtraRepos(configYaml) {
+ def config = readYaml text: configYaml
+ def distribRevision = config.get('distrib_revision', 'nightly')
+ if (config.get('repo', false)) {
+ for (String repo in config['repo'].keySet()) {
+ source = config['repo'][repo]['source'].replace('SUB_DISTRIB_REVISION', distribRevision)
+ warningMsg("Write ${source} > /etc/apt/sources.list.d/${repo}.list")
+ sh("echo '${source}' > /etc/apt/sources.list.d/${repo}.list")
+ // TODO implement pining
+ }
+ }
+ if (config.get('aprConfD', false)) {
+ for (String pref in config['aprConfD'].tokenize('\n')) {
+ warningMsg("Adding ${pref} => /etc/apt/apt.conf.d/99setupAndTestNode")
+ sh("echo '${pref}' >> /etc/apt/apt.conf.d/99setupAndTestNode")
+ }
+ sh('cat /etc/apt/apt.conf.d/99setupAndTestNode')
+ }
+}
diff --git a/src/com/mirantis/mk/SaltModelTesting.groovy b/src/com/mirantis/mk/SaltModelTesting.groovy
index d6a1b82..2d1a888 100644
--- a/src/com/mirantis/mk/SaltModelTesting.groovy
+++ b/src/com/mirantis/mk/SaltModelTesting.groovy
@@ -45,7 +45,36 @@
]
def dockerOptsFinal = (dockerBaseOpts + dockerExtraOpts).join(' ')
+ def defaultExtraReposYaml = '''
+---
+distrib_revision: 'nightly'
+aprConfD: |-
+ APT::Get::AllowUnauthenticated 'true';
+ APT::Get::Install-Suggests 'false';
+ APT::Get::Install-Recommends 'false';
+repo:
+ mcp_saltstack:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/saltstack-2017.7/xenial xenial main"
+ pinning: |-
+ Package: libsodium18
+ Pin: release o=SaltStack
+ Pin-Priority: 50
+
+ Package: *
+ Pin: release o=SaltStack
+ Pin-Priority: 1100
+ mcp_extra:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/extra/xenial xenial main"
+ ubuntu:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial main restricted universe"
+ ubuntu-upd:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial-updates main restricted universe"
+ ubuntu-sec:
+ source: "deb [arch=amd64] http://mirror.mirantis.com/SUB_DISTRIB_REVISION/ubuntu xenial-security main restricted universe"
+'''
def img = docker.image(dockerImageName)
+ def extraReposYaml = config.get('extraReposYaml', defaultExtraReposYaml)
+
img.pull()
try {
@@ -55,17 +84,19 @@
// Currently, we don't have any other point to install
// runtime dependencies for tests.
if (baseRepoPreConfig) {
+ // Warning! POssible point of 'allow-downgrades' issue
+ // Probably, need to add such flag into apt.prefs
sh("""#!/bin/bash -xe
echo "Installing extra-deb dependencies inside docker:"
- echo "APT::Get::AllowUnauthenticated 'true';" > /etc/apt/apt.conf.d/99setupAndTestNode
- echo "APT::Get::Install-Suggests 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
- echo "APT::Get::Install-Recommends 'false';" >> /etc/apt/apt.conf.d/99setupAndTestNode
+ echo > /etc/apt/sources.list
rm -vf /etc/apt/sources.list.d/* || true
- echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial main restricted universe' > /etc/apt/sources.list
- echo 'deb [arch=amd64] http://mirror.mirantis.com/$DISTRIB_REVISION/ubuntu xenial-updates main restricted universe' >> /etc/apt/sources.list
- apt-get update
- apt-get install -y python-netaddr
""")
+ common.debianExtraRepos(extraReposYaml)
+ sh('''#!/bin/bash -xe
+ apt-get update
+ apt-get install -y python-netaddr reclass
+ ''')
+
}
runCommands.sort().each { command, body ->
common.warningMsg("Running command: ${command}")
@@ -111,15 +142,15 @@
}
/**
- * Wrapper around setupDockerAndTest, to run checks against new Reclass version
- * that current model is compatible with new Reclass.
- *
- * @param config - LinkedHashMap with configuration params:
- * dockerHostname - (required) Hostname to use for Docker container.
- * distribRevision - (optional) Revision of packages to use (default proposed).
- * extraRepo - (optional) Extra repo to use to install new Reclass version. Has
- * high priority on distribRevision
- * targetNodes - (required) List nodes to check pillar data.
+ * Wrapper around setupDockerAndTest, to run checks against new Reclass version
+ * that current model is compatible with new Reclass.
+ *
+ * @param config - LinkedHashMap with configuration params:
+ * dockerHostname - (required) Hostname to use for Docker container.
+ * distribRevision - (optional) Revision of packages to use (default proposed).
+ * extraRepo - (optional) Extra repo to use to install new Reclass version. Has
+ * high priority on distribRevision
+ * targetNodes - (required) List nodes to check pillar data.
*/
def compareReclassVersions(config) {
def common = new com.mirantis.mk.Common()
@@ -134,21 +165,21 @@
sh "mkdir -p ${env.WORKSPACE}/old ${env.WORKSPACE}/new"
def configRun = [
'formulasRevision': distribRevision,
- 'dockerExtraOpts': [
+ 'dockerExtraOpts' : [
"-v /srv/salt/reclass:/srv/salt/reclass:ro",
"-v /etc/salt:/etc/salt:ro",
"-v /usr/share/salt-formulas/:/usr/share/salt-formulas/:ro"
],
- 'envOpts': [
+ 'envOpts' : [
"WORKSPACE=${env.WORKSPACE}",
"NODES_LIST=${targetNodes.join(' ')}"
],
- 'runCommands': [
- '001_Update_Reclass_package': {
- sh('apt-get update && apt-get install -y reclass')
+ 'runCommands' : [
+ '001_Update_Reclass_package' : {
+ sh('apt-get update && apt-get install -y reclass')
},
'002_Test_Reclass_Compatibility': {
- sh('''
+ sh('''
reclass-salt -b /srv/salt/reclass -t > ${WORKSPACE}/new/inventory || exit 1
for node in $NODES_LIST; do
reclass-salt -b /srv/salt/reclass -p $node > ${WORKSPACE}/new/$node || exit 1
@@ -158,26 +189,20 @@
]
]
if (extraRepo) {
+ // FIXME
configRun['runCommands']['0001_Additional_Extra_Repo_Passed'] = {
sh("""
echo "${extraRepo}" > /etc/apt/sources.list.d/mcp_extra.list
[ "${extraRepoKey}" ] && wget -O - ${extraRepoKey} | apt-key add -
""")
}
- } else {
- configRun['runCommands']['0001_Additional_Extra_Repo_Default'] = {
- sh("""
- echo "deb [arch=amd64] http://apt.mirantis.com/xenial ${distribRevision} extra" > /etc/apt/sources.list.d/mcp_extra.list
- wget -O - http://apt.mirantis.com/public.gpg | apt-key add -
- """)
- }
}
if (setupDockerAndTest(configRun)) {
common.infoMsg("New reclass version is compatible with current model: SUCCESS")
def inventoryOld = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -t", true, null, true).get("return")[0].values()[0]
// [0..-31] to exclude 'echo Salt command execution success' from output
writeFile(file: "${env.WORKSPACE}/old/inventory", text: inventoryOld[0..-31])
- for(String node in targetNodes) {
+ for (String node in targetNodes) {
def nodeOut = salt.cmdRun(venv, "I@salt:master", "reclass-salt -b /srv/salt/reclass -p ${node}", true, null, true).get("return")[0].values()[0]
writeFile(file: "${env.WORKSPACE}/old/${node}", text: nodeOut[0..-31])
}
@@ -207,7 +232,7 @@
* aptRepoUrl - (optional) package repository with salt formulas
* aptRepoGPG - (optional) GPG key for apt repository with formulas
* testContext - (optional) Description of test
- Return: true\exception
+ Return: true\exception
*/
def testNode(LinkedHashMap config) {
@@ -235,29 +260,20 @@
config['runCommands'] = [
'001_Clone_salt_formulas_scripts': {
- sh(script: 'git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
+ sh(script: 'git clone https://github.com/salt-formulas/salt-formulas-scripts /srv/salt/scripts', returnStdout: true)
},
- '002_Prepare_something': {
+ '002_Prepare_something' : {
sh('''rsync -ah ${RECLASS_ENV}/* /srv/salt/reclass && echo '127.0.1.2 salt' >> /etc/hosts
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt-mk.mirantis.com/apt.mirantis.net:8085/g' {} \\;
cd /srv/salt && find . -type f \\( -name '*.yml' -or -name '*.sh' \\) -exec sed -i 's/apt.mirantis.com/apt.mirantis.net:8085/g' {} \\;
''')
},
- // should be switched on packages later
- '003_Install_reclass': {
- sh('''for s in \$(python -c \"import site; print(' '.join(site.getsitepackages()))\"); do
- sudo -H pip install --install-option=\"--prefix=\" --upgrade --force-reinstall -I \
- -t \"\$s\" git+https://github.com/salt-formulas/reclass.git@${RECLASS_VERSION};
- done
- ''')
- },
-
- '004_Run_tests': {
+ '004_Run_tests' : {
def testTimeout = 40 * 60
timeout(time: testTimeout, unit: 'SECONDS') {
- sh('''#!/bin/bash
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
source_local_envs
@@ -268,13 +284,13 @@
cd /srv/salt/scripts
saltservice_restart''')
- sh('''#!/bin/bash
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
source_local_envs
saltmaster_init''')
- sh('''#!/bin/bash
+ sh('''#!/bin/bash
source /srv/salt/scripts/bootstrap.sh
cd /srv/salt/scripts
verify_salt_minions''')