Generate model in docker
* Install all requirments into docker one-shot env, to
not depends on slave env.
* Misc: decrese git log size
Change-Id: I692e2d2abadf7b28499afb0da2c34ad857382c69
Prod-related: PROD-28044 (PROD:28044)
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 54e9853..010b0ba 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -6,6 +6,7 @@
* CREDENTIALS_ID Credentials id for git
* TEST_MODEL Run syntax tests for model
**/
+
import static groovy.json.JsonOutput.toJson
import static groovy.json.JsonOutput.prettyPrint
import org.apache.commons.net.util.SubnetUtils
@@ -22,6 +23,47 @@
distribRevision = 'proposed'
gitGuessedVersion = false
+def GenerateModelToxDocker(Map params) {
+ def ccRoot = params['ccRoot']
+ def context = params['context']
+ def outDir = params['outDir']
+ def envOpts = params['envOpts']
+ if (fileExists(new File(ccRoot, 'tox.ini').toString())) {
+ def tempContextFile = new File(ccRoot, 'tempContext.yaml_' + UUID.randomUUID().toString()).toString()
+ writeFile file: tempContextFile, text: context
+ // Get Jenkins user UID and GID
+ def jenkinsUID = sh(script: 'id -u', returnStdout: true).trim()
+ def jenkinsGID = sh(script: 'id -g', returnStdout: true).trim()
+ /// by default, process in image operates via root user
+ /// Otherwise, gpg key for model and all files managed by jenkins user
+ /// To make it compatible, install rrequirementfrom user, but generate model via jenkins
+ def configRun = ['distribRevision': 'nightly',
+ 'envOpts' : envOpts + ["CONFIG_FILE=$tempContextFile",
+ "OUTPUT_DIR=${outDir}",
+ "jenkinsUID=${jenkinsUID}",
+ "jenkinsGID=${jenkinsGID}"
+ ],
+ 'runCommands' : [
+ '001_prepare_generate_auto_reqs': {
+ sh('''
+ groupadd -g ${jenkinsGID} jenkins
+ useradd -u ${jenkinsUID} -g ${jenkinsGID} -m jenkins
+ # Install required packages
+ apt-get install -y python3-dev python-tox
+ ''')
+ },
+ '002_run_generate_auto' : {
+ print('[Cookiecutter build] Result:\n' +
+ sh(returnStdout: true, script: 'cd ' + ccRoot + '; su jenkins -c "tox -ve generate_auto" '))
+ }
+ ]
+ ]
+
+ print(saltModelTesting.setupDockerAndTest(configRun))
+ }
+
+}
+
def globalVariatorsUpdate() {
def templateContext = readYaml text: env.COOKIECUTTER_TEMPLATE_CONTEXT
@@ -83,7 +125,7 @@
}
common.warningMsg("Fetching:\n" +
"DISTRIB_REVISION from ${distribRevision}")
- common.infoMsg("Using context:\n" + context)
+ common.infoMsg("Using context:\n")
print prettyPrint(toJson(context))
return context
@@ -136,11 +178,10 @@
}
stage('Generate model') {
- // GNUPGHOME environment variable is required for all gpg commands
- // and for python.generateModel execution
- withEnv(["GNUPGHOME=${env.WORKSPACE}/gpghome"]) {
+ def envOpts = ["GNUPGHOME=${env.WORKSPACE}/gpghome"]
+ withEnv(envOpts) {
if (context['secrets_encryption_enabled'] == 'True') {
- sh "mkdir gpghome; chmod 700 gpghome"
+ sh 'mkdir gpghome; chmod 700 gpghome'
def secretKeyID = RequesterEmail ?: "salt@${context['cluster_domain']}".toString()
if (!context.get('secrets_encryption_private_key')) {
def batchData = """
@@ -176,7 +217,10 @@
// still expect only lower lvl of project, aka model/classes/cluster/XXX/. So,lets dump result into
// temp dir, and then copy it over initial structure.
reclassTempRootDir = sh(script: "mktemp -d -p ${env.WORKSPACE}", returnStdout: true).trim()
- python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, reclassTempRootDir, templateEnv, false)
+ GenerateModelToxDocker(['context': common2.dumpYAML(['default_context': context]),
+ 'ccRoot' : templateEnv,
+ 'outDir' : reclassTempRootDir,
+ 'envOpts': envOpts])
dir(modelEnv) {
common.warningMsg('Forming reclass-root structure...')
sh("cp -ra ${reclassTempRootDir}/reclass/* .")
@@ -186,7 +230,7 @@
}
}
- stage("Test") {
+ stage('Test') {
if (runTestModel) {
sh("cp -r ${modelEnv} ${testEnv}")
if (fileExists('gpgkey.asc')) {
@@ -235,8 +279,8 @@
}
sh '[ -f mcp-common-scripts/config-drive/master_config.sh ] && cp mcp-common-scripts/config-drive/master_config.sh user_data || cp mcp-common-scripts/config-drive/master_config.yaml user_data'
- sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
- sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
+ sh "git clone --depth 10 --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
+ sh "git clone --depth 10 --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
args = [
"--user-data user_data", "--model ${modelEnv}",
"--mk-pipelines ${pipelineEnv}/mk-pipelines/", "--pipeline-library ${pipelineEnv}/pipeline-library/"