Merge "Add 'Galera DB backup' pipeline"
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 9878a49..fe21725 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -331,7 +331,7 @@
// Set up override params
- if (common.validInputParam('SALT_OVERRIDES')) {
+ if (common.validInputParam('SALT_OVERRIDES') && common.validInputParam('STACK_INSTALL')) {
stage('Set Salt overrides') {
salt.setSaltOverrides(venvPepper, SALT_OVERRIDES, '/srv/salt/reclass', extra_tgt)
}
@@ -366,8 +366,10 @@
}
}
- stage('Install Orchestrated Apps'){
- orchestrate.OrchestrateApplications(venvPepper, "I@salt:master ${extra_tgt}", "orchestration.deploy.applications")
+ if (common.validInputParam('STACK_INSTALL')) {
+ stage('Install Orchestrated Apps'){
+ orchestrate.OrchestrateApplications(venvPepper, "I@salt:master ${extra_tgt}", "orchestration.deploy.applications")
+ }
}
// install k8s
@@ -458,7 +460,9 @@
}
// install docker swarm
- orchestrate.installDockerSwarm(venvPepper, extra_tgt)
+ if (common.validInputParam('STACK_INSTALL')) {
+ orchestrate.installDockerSwarm(venvPepper, extra_tgt)
+ }
// install openstack
if (common.checkContains('STACK_INSTALL', 'openstack')) {
diff --git a/cloud-update.groovy b/cloud-update.groovy
index 28c5005..0ea5fea 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -60,6 +60,17 @@
wait = "${MINIONS_TEST_TIMEOUT}".toInteger()
}
+def updateSaltPackage(pepperEnv, target, pkgs, masterUpdate = false) {
+ def salt = new com.mirantis.mk.Salt()
+ salt.cmdRun(pepperEnv, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
+ def minions_reachable = target
+ if (masterUpdate) {
+ // in case of update Salt Master packages - check all minions are good
+ minions_reachable = '*'
+ }
+ salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': target, 'target_reachable': minions_reachable])
+}
+
def updatePkgs(pepperEnv, target, targetType="", targetPackages="") {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
@@ -155,12 +166,10 @@
// salt master pkg
if (targetType == 'cfg') {
common.warningMsg('salt-master pkg upgrade, rerun the pipeline if disconnected')
- salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-master'], null, true, 5)
- salt.minionsReachable(pepperEnv, 'I@salt:master', '*', null, wait)
+ updateSaltPackage(pepperEnv, target, '["salt-master"]', true)
}
// salt minion pkg
- salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ['salt-minion'], null, true, 5)
- salt.minionsReachable(pepperEnv, 'I@salt:master', target, null, wait)
+ updateSaltPackage(pepperEnv, target, '["salt-minion"]')
common.infoMsg('Performing pkg upgrades ... ')
common.retry(3){
out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], command, true, packages, commandKwargs)
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 1fc38b3..b672c4e 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -69,7 +69,7 @@
}
// Check if we are going to test bleeding-edge release, which doesn't have binary release yet
// After 2018q4 releases, need to also check 'static' repo, for example ubuntu.
- binTest = common.checkRemoteBinary(['mcp_version': distribRevision])
+ def binTest = common.checkRemoteBinary(['mcp_version': distribRevision])
if (!binTest.linux_system_repo_url || !binTest.linux_system_repo_ubuntu_url) {
common.errorMsg("Binary release: ${distribRevision} not exist or not full. Fallback to 'proposed'! ")
distribRevision = 'proposed'
@@ -93,6 +93,7 @@
def context = globalVariatorsUpdate()
def RequesterEmail = context.get('email_address', '')
def templateEnv = "${env.WORKSPACE}/template"
+ // modelEnv - this is reclass root, aka /srv/salt/reclass
def modelEnv = "${env.WORKSPACE}/model"
def testEnv = "${env.WORKSPACE}/test"
def pipelineEnv = "${env.WORKSPACE}/pipelines"
@@ -119,7 +120,7 @@
}
stage('Create empty reclass model') {
dir(path: modelEnv) {
- sh "rm -rfv .git; git init"
+ sh 'rm -rfv .git; git init'
sshagent(credentials: [gerritCredentials]) {
sh "git submodule add ${context['shared_reclass_url']} 'classes/system'"
}
@@ -130,7 +131,7 @@
extensions : [[$class: 'RelativeTargetDirectory', relativeTargetDir: systemEnv]],
userRemoteConfigs: [[url: context['shared_reclass_url'], refspec: context['shared_reclass_branch'], credentialsId: gerritCredentials],],
])
- git.commitGitChanges(modelEnv, "Added new shared reclass submodule", "${user}@localhost", "${user}")
+ git.commitGitChanges(modelEnv, 'Added new shared reclass submodule', "${user}@localhost", "${user}")
}
stage('Generate model') {
@@ -161,9 +162,20 @@
if (context.get('cfg_failsafe_ssh_public_key')) {
writeFile file: 'failsafe-ssh-key.pub', text: context['cfg_failsafe_ssh_public_key']
}
- python.setupCookiecutterVirtualenv(cutterEnv)
- // FIXME refactor generateModel
- python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, modelEnv, templateEnv, false)
+ if (!fileExists(new File(templateEnv, 'tox.ini').toString())) {
+ python.setupCookiecutterVirtualenv(cutterEnv)
+ python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, modelEnv, templateEnv, false)
+ } else {
+ // tox-based CC generated structure of reclass,from the root. Otherwise for bw compat, modelEnv
+ // still expect only lower lvl of project, aka model/classes/cluster/XXX/. So,lets dump result into
+ // temp dir, and then copy it over initial structure.
+ reclassTempRootDir = sh(script: "mktemp -d -p ${env.WORKSPACE}", returnStdout: true).trim()
+ python.generateModel(common2.dumpYAML(['default_context': context]), 'default_context', context['salt_master_hostname'], cutterEnv, reclassTempRootDir, templateEnv, false)
+ dir(modelEnv) {
+ common.warningMsg('Forming reclass-root structure...')
+ sh("cp -ra ${reclassTempRootDir}/reclass/* .")
+ }
+ }
git.commitGitChanges(modelEnv, "Create model ${context['cluster_name']}", "${user}@localhost", "${user}")
}
}
@@ -184,7 +196,8 @@
'reclassEnv' : testEnv,
'distribRevision' : distribRevision,
'dockerContainerName': DockerCName,
- 'testContext' : 'salt-model-node'
+ 'testContext' : 'salt-model-node',
+ 'dockerExtraOpts' : ['--memory=3g']
]
testResult = saltModelTesting.testNode(config)
common.infoMsg("Test finished: SUCCESS")
@@ -207,29 +220,43 @@
userRemoteConfigs: [[url: commonScriptsRepoUrl, refspec: context['mcp_common_scripts_branch'], credentialsId: gerritCredentials],],
])
- sh 'cp mcp-common-scripts/config-drive/create_config_drive.sh create-config-drive && chmod +x create-config-drive'
+ def outdateGeneration = false
+ if (fileExists('mcp-common-scripts/config-drive/create_config_drive.py')) {
+ sh 'cp mcp-common-scripts/config-drive/create_config_drive.py create-config-drive.py'
+ } else {
+ outdateGeneration = true
+ sh 'cp mcp-common-scripts/config-drive/create_config_drive.sh create-config-drive && chmod +x create-config-drive'
+ }
sh '[ -f mcp-common-scripts/config-drive/master_config.sh ] && cp mcp-common-scripts/config-drive/master_config.sh user_data || cp mcp-common-scripts/config-drive/master_config.yaml user_data'
sh "git clone --mirror https://github.com/Mirantis/mk-pipelines.git ${pipelineEnv}/mk-pipelines"
sh "git clone --mirror https://github.com/Mirantis/pipeline-library.git ${pipelineEnv}/pipeline-library"
- args = "--user-data user_data --hostname ${context['salt_master_hostname']} --model ${modelEnv} --mk-pipelines ${pipelineEnv}/mk-pipelines/ --pipeline-library ${pipelineEnv}/pipeline-library/ ${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso"
+ args = [
+ "--user-data user_data", , "--model ${modelEnv}",
+ "--mk-pipelines ${pipelineEnv}/mk-pipelines/", "--pipeline-library ${pipelineEnv}/pipeline-library/"
+ ]
if (context['secrets_encryption_enabled'] == 'True') {
- args = "--gpg-key gpgkey.asc " + args
+ args.add('--gpg-key gpgkey.asc')
}
if (context.get('cfg_failsafe_ssh_public_key')) {
- args = "--ssh-key failsafe-ssh-key.pub " + args
+ if (outdateGeneration) {
+ args.add('--ssh-key failsafe-ssh-key.pub')
+ } else {
+ args.add('--ssh-keys failsafe-ssh-key.pub')
+ }
}
-
// load data from model
def smc = [:]
smc['SALT_MASTER_MINION_ID'] = "${context['salt_master_hostname']}.${context['cluster_domain']}"
smc['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
- smc['DEPLOY_NETWORK_GW'] = context['deploy_network_gateway']
- smc['DEPLOY_NETWORK_NETMASK'] = context['deploy_network_netmask']
- if (context.get('deploy_network_mtu')) {
- smc['DEPLOY_NETWORK_MTU'] = context['deploy_network_mtu']
+ if (outdateGeneration) {
+ smc['DEPLOY_NETWORK_GW'] = context['deploy_network_gateway']
+ smc['DEPLOY_NETWORK_NETMASK'] = context['deploy_network_netmask']
+ if (context.get('deploy_network_mtu')) {
+ smc['DEPLOY_NETWORK_MTU'] = context['deploy_network_mtu']
+ }
+ smc['DNS_SERVERS'] = context['dns_server01']
}
- smc['DNS_SERVERS'] = context['dns_server01']
smc['MCP_VERSION'] = "${context['mcp_version']}"
if (context['local_repositories'] == 'True') {
def localRepoIP = ''
@@ -260,7 +287,17 @@
}
// create cfg config-drive
- sh "./create-config-drive ${args}"
+ if (outdateGeneration) {
+ args += [ "--hostname ${context['salt_master_hostname']}", "${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso" ]
+ sh "./create-config-drive ${args.join(' ')}"
+ } else {
+ args += [
+ "--name ${context['salt_master_hostname']}", "--hostname ${context['salt_master_hostname']}.${context['cluster_domain']}", "--clean-up",
+ "--ip ${context['salt_master_management_address']}", "--netmask ${context['deploy_network_netmask']}", "--gateway ${context['deploy_network_gateway']}",
+ "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
+ ]
+ sh "python ./create-config-drive.py ${args.join(' ')}"
+ }
sh("mkdir output-${context['cluster_name']} && mv ${context['salt_master_hostname']}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
// save cfg iso to artifacts
@@ -272,8 +309,10 @@
def smc_apt = [:]
smc_apt['SALT_MASTER_DEPLOY_IP'] = context['salt_master_management_address']
- smc_apt['APTLY_DEPLOY_IP'] = context['aptly_server_deploy_address']
- smc_apt['APTLY_DEPLOY_NETMASK'] = context['deploy_network_netmask']
+ if (outdateGeneration) {
+ smc_apt['APTLY_DEPLOY_IP'] = context['aptly_server_deploy_address']
+ smc_apt['APTLY_DEPLOY_NETMASK'] = context['deploy_network_netmask']
+ }
smc_apt['APTLY_MINION_ID'] = "${aptlyServerHostname}.${context['cluster_domain']}"
for (i in common.entries(smc_apt)) {
@@ -281,7 +320,16 @@
}
// create apt config-drive
- sh "./create-config-drive --user-data mirror_config --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
+ if (outdateGeneration) {
+ sh "./create-config-drive --user-data mirror_config --hostname ${aptlyServerHostname} ${aptlyServerHostname}.${context['cluster_domain']}-config.iso"
+ } else {
+ args = [
+ "--ip ${context['aptly_server_deploy_address']}", "--netmask ${context['deploy_network_netmask']}", "--gateway ${context['deploy_network_gateway']}",
+ "--user-data mirror_config", "--hostname ${aptlyServerHostname}.${context['cluster_domain']}", "--name ${aptlyServerHostname}", "--clean-up",
+ "--dns-nameservers ${context['dns_server01']},${context['dns_server02']}"
+ ]
+ sh "python ./create-config-drive.py ${args.join(' ')}"
+ }
sh("mv ${aptlyServerHostname}.${context['cluster_domain']}-config.iso output-${context['cluster_name']}/")
// save apt iso to artifacts
diff --git a/restore-zookeeper.groovy b/restore-zookeeper.groovy
index 185f097..8afc5a5 100644
--- a/restore-zookeeper.groovy
+++ b/restore-zookeeper.groovy
@@ -12,6 +12,14 @@
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
+
+def oc3SupervisorServices = ["supervisor-config", "supervisor-control"]
+def oc4ConfigServices = ["contrail-api", "contrail-schema", "contrail-svc-monitor", "contrail-device-manager", "contrail-config-nodemgr"]
+def oc4ControlServices = ["contrail-control", "contrail-named", "contrail-dns", "contrail-control-nodemgr"]
+def zkService = "zookeeper"
+def contrailStatusCheckCmd = "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup"
+def zkDbPath = "/var/lib/zookeeper/version-2"
+
timeout(time: 12, unit: 'HOURS') {
node() {
@@ -20,65 +28,86 @@
}
stage('Restore') {
- try {
- salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ['supervisor-config'], null, true)
- } catch (Exception er) {
- common.warningMsg('Supervisor-config service already stopped')
+
+ def ocVersionPillarKey = salt.getReturnValues(salt.getPillar(pepperEnv, "I@opencontrail:control:role:primary", "_param:opencontrail_version"))
+
+ if (ocVersionPillarKey == '') {
+ throw new Exception("Cannot get value for _param:opencontrail_version key on I@opencontrail:control:role:primary target")
}
- try {
- salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ['supervisor-control'], null, true)
- } catch (Exception er) {
- common.warningMsg('Supervisor-control service already stopped')
+
+ def ocVersion = ocVersionPillarKey.toString()
+
+ if (ocVersion >= "4.0") {
+
+ contrailStatusCheckCmd = "doctrail controller ${contrailStatusCheckCmd}"
+ zkDbPath = "/var/lib/config_zookeeper_data/version-2"
+
+ for (service in (oc4ConfigServices + oc4ControlServices + [zkService])) {
+ try {
+ salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'cmd.run', ["doctrail controller systemctl stop ${service}"])
+ } catch (Exception er) {
+ common.warningMsg("${service} cannot be stopped inside controller container")
+ }
+ }
+ // wait until zookeeper service is down
+ salt.commandStatus(pepperEnv, 'I@opencontrail:control', "doctrail controller service ${zkService} status", 'Active: inactive')
+ } else {
+ for (service in (oc3SupervisorServices + [zkService])) {
+ try {
+ salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ["${service}"])
+ } catch (Exception er) {
+ common.warningMsg("${service} service cannot be stopped. It may be already stopped before.")
+ }
+ }
+ // wait until zookeeper service is down
+ salt.commandStatus(pepperEnv, 'I@opencontrail:control', "service ${zkService} status", "stop")
}
- try {
- salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.stop', ['zookeeper'], null, true)
- } catch (Exception er) {
- common.warningMsg('Zookeeper service already stopped')
- }
- //sleep(5)
- // wait until zookeeper service is down
- salt.commandStatus(pepperEnv, 'I@opencontrail:control', 'service zookeeper status', 'stop')
try {
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "mkdir -p /root/zookeeper/zookeeper.bak")
} catch (Exception er) {
- common.warningMsg('Directory already exists')
+ common.warningMsg('/root/zookeeper/zookeeper.bak directory already exists')
}
try {
- salt.cmdRun(pepperEnv, 'I@opencontrail:control', "mv /var/lib/zookeeper/version-2/* /root/zookeeper/zookeeper.bak")
+ salt.cmdRun(pepperEnv, 'I@opencontrail:control', "mv ${zkDbPath}/* /root/zookeeper/zookeeper.bak")
} catch (Exception er) {
common.warningMsg('Files were already moved')
}
try {
- salt.cmdRun(pepperEnv, 'I@opencontrail:control', "rm -rf /var/lib/zookeeper/version-2/*")
+ salt.cmdRun(pepperEnv, 'I@opencontrail:control', "rm -rf ${zkDbPath}/*")
} catch (Exception er) {
common.warningMsg('Directory already empty')
}
- _pillar = salt.getPillar(pepperEnv, "I@opencontrail:control", 'zookeeper:backup:backup_dir')
- backup_dir = _pillar['return'][0].values()[0]
- if(backup_dir == null || backup_dir.isEmpty()) { backup_dir='/var/backups/zookeeper' }
- print(backup_dir)
- salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'file.remove', ["${backup_dir}/dbrestored"], null, true)
+ backupDirPillarKey = salt.getPillar(pepperEnv, "I@opencontrail:control", 'zookeeper:backup:backup_dir')
+ backupDir = backupDirPillarKey['return'][0].values()[0]
+ if (backupDir == null || backupDir.isEmpty()) { backupDir='/var/backups/zookeeper' }
+ print(backupDir)
+ salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'file.remove', ["${backupDir}/dbrestored"])
// performs restore
salt.enforceState(pepperEnv, 'I@opencontrail:control', "zookeeper.backup")
- salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.start', ['zookeeper'], null, true)
- salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.start', ['supervisor-config'], null, true)
- salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.start', ['supervisor-control'], null, true)
+ if (ocVersion >= "4.0") {
+ for (service in ([zkService] + oc4ConfigServices + oc4ControlServices)) {
+ salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'cmd.run', ["doctrail controller systemctl start ${service}"])
+ }
+ } else {
+ for (service in ([zkService] + oc3SupervisorServices)) {
+ salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'service.start', ["${service}"])
+ }
+ }
// wait until contrail-status is up
- salt.commandStatus(pepperEnv, 'I@opencontrail:control', "contrail-status | grep -v == | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup", null, false)
+ salt.commandStatus(pepperEnv, 'I@opencontrail:control', contrailStatusCheckCmd, null, false)
- salt.cmdRun(pepperEnv, 'I@opencontrail:control', "ls /var/lib/zookeeper/version-2")
+ salt.cmdRun(pepperEnv, 'I@opencontrail:control', "ls ${zkDbPath}")
try {
salt.cmdRun(pepperEnv, 'I@opencontrail:control', "echo stat | nc localhost 2181")
} catch (Exception er) {
common.warningMsg('Check which node is zookeeper leader')
}
- salt.cmdRun(pepperEnv, 'I@opencontrail:control', "contrail-status")
}
}
}
diff --git a/test-cookiecutter-reclass-chunk.groovy b/test-cookiecutter-reclass-chunk.groovy
index 8fe0a2b..e0c9710 100644
--- a/test-cookiecutter-reclass-chunk.groovy
+++ b/test-cookiecutter-reclass-chunk.groovy
@@ -33,7 +33,8 @@
'reclassEnv': extraVars.testReclassEnv,
'distribRevision': extraVars.DISTRIB_REVISION,
'dockerContainerName': extraVars.DockerCName,
- 'testContext': extraVars.modelFile
+ 'testContext': extraVars.modelFile,
+ 'dockerExtraOpts': [ '--memory=3g' ]
]
if (extraVars.useExtraRepos) {
config['extraRepos'] = extraVars.extraRepos ? extraVars.extraRepos : [:]
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index 8076041..bd5ec1e 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -116,7 +116,7 @@
common.infoMsg("StepPrepareGit: ${gerrit_data}")
// fetch needed sources
dir(templateEnvFolder) {
- if (! gerrit_data['gerritRefSpec']) {
+ if (!gerrit_data['gerritRefSpec']) {
// Get clean HEAD
gerrit_data['useGerritTriggerBuildChooser'] = false
}
@@ -133,7 +133,26 @@
for (contextFile in _contextFileList) {
def basename = common.GetBaseName(contextFile, '.yml')
def context = readFile(file: "${_templateEnvDir}/contexts/${contextFile}")
- python.generateModel(context, basename, 'cfg01', _virtualenv, "${_templateEnvDir}/model", _templateEnvDir)
+ if (!fileExists(new File(_templateEnvDir, 'tox.ini').toString())) {
+ common.warningMsg('Forming NEW reclass-root structure...')
+ python.generateModel(context, basename, 'cfg01', _virtualenv, "${_templateEnvDir}/model", _templateEnvDir)
+ } else {
+ // tox-based CC generated structure of reclass,from the root. Otherwise for bw compat, modelEnv
+ // still expect only lower lvl of project, aka model/classes/cluster/XXX/. So,lets dump result into
+ // temp dir, and then copy it over initial structure.
+ def reclassTempRootDir = sh(script: "mktemp -d -p ${env.WORKSPACE}", returnStdout: true).trim()
+ python.generateModel(context, basename, 'cfg01', _virtualenv, reclassTempRootDir, _templateEnvDir)
+ dir("${_templateEnvDir}/model/${basename}/") {
+ if (fileExists(new File(reclassTempRootDir, 'reclass').toString())) {
+ common.warningMsg('Forming NEW reclass-root structure...')
+ sh("cp -ra ${reclassTempRootDir}/reclass/* .")
+ } else {
+ // those hack needed only for period release/2019.2.0 => current patch.
+ common.warningMsg('Forming OLD reclass-root structure...')
+ sh("mkdir -p classes/cluster/ ; cd classes/cluster/; cp -ra ${reclassTempRootDir}/* .")
+ }
+ }
+ }
}
}
}
diff --git a/test-salt-formulas-env.groovy b/test-salt-formulas-env.groovy
index 257c0ab..e007fe9 100644
--- a/test-salt-formulas-env.groovy
+++ b/test-salt-formulas-env.groovy
@@ -27,6 +27,8 @@
def travisLess = false /** TODO: Remove once formulas are witched to new config */
def cleanEnv = '' /** TODO: Remove once formulas are witched to new config */
def testSuite = ''
+envOverrides = []
+kitchenFileName = ''
throttle(['test-formula']) {
timeout(time: 1, unit: 'HOURS') {
@@ -114,26 +116,27 @@
} else {
if (checkouted) {
travisLess = true
- if (fileExists(".kitchen.yml") || fileExists(".kitchen.openstack.yml")) {
- if (fileExists(".kitchen.openstack.yml")) {
- common.infoMsg("Openstack Kitchen test configuration found, running Openstack kitchen tests.")
- if (fileExists(".kitchen.yml")) {
- common.infoMsg("Ignoring the docker Kitchen test configuration file.")
- }
- openstackTest = true
- withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: openstack_credentials_id,
- usernameVariable: 'OS_USERNAME', passwordVariable: 'OS_PASSWORD'], ]) {
- env.OS_USERNAME = OS_USERNAME
- env.OS_PASSWORD = OS_PASSWORD
- env.OS_AUTH_URL = OS_AUTH_URL
- env.OS_PROJECT_NAME = OS_PROJECT_NAME
- env.OS_DOMAIN_NAME = OS_DOMAIN_NAME
- env.OS_AZ = OS_AZ
- }
- } else {
- common.infoMsg("Docker Kitchen test configuration found, running Docker kitchen tests.")
+ if (fileExists(".kitchen.openstack.yml")) {
+ common.infoMsg("Openstack Kitchen test configuration found, running Openstack kitchen tests.")
+ kitchenFileName = ".kitchen.openstack.yml"
+ envOverrides.add("KITCHEN_YAML=${kitchenFileName}")
+ rubyVersion = '2.5.0'
+ withCredentials([[$class: 'UsernamePasswordMultiBinding', credentialsId: openstack_credentials_id,
+ usernameVariable: 'OS_USERNAME', passwordVariable: 'OS_PASSWORD'], ]) {
+ env.OS_USERNAME = OS_USERNAME
+ env.OS_PASSWORD = OS_PASSWORD
+ env.OS_AUTH_URL = OS_AUTH_URL
+ env.OS_PROJECT_NAME = OS_PROJECT_NAME
+ env.OS_DOMAIN_NAME = OS_DOMAIN_NAME
+ env.OS_AZ = OS_AZ
}
- ruby.ensureRubyEnv()
+ } else if (fileExists(".kitchen.yml")) {
+ common.infoMsg("Docker Kitchen test configuration found, running Docker kitchen tests.")
+ kitchenFileName = ".kitchen.yml"
+ rubyVersion = '2.4.1'
+ }
+ if (kitchenFileName) {
+ ruby.ensureRubyEnv(rubyVersion)
if (!fileExists("Gemfile")) {
sh("curl -s -o ./Gemfile 'https://gerrit.mcp.mirantis.com/gitweb?p=salt-formulas/salt-formulas-scripts.git;a=blob_plain;f=Gemfile;hb=refs/heads/master'")
ruby.installKitchen()
@@ -144,16 +147,15 @@
common.infoMsg("Running part of kitchen test")
if (KITCHEN_ENV != null && !KITCHEN_ENV.isEmpty() && KITCHEN_ENV != "") {
testSuite = KITCHEN_ENV.replaceAll("_", "-").trim()
- if (openstackTest) { testSuite = "KITCHEN_YAML=.kitchen.openstack.yml " + testSuite }
sh("grep apt.mirantis.com -Ril | xargs -I{} bash -c \"echo {}; sed -i 's/apt.mirantis.com/apt.mcp.mirantis.net/g' {}\"")
sh("grep apt-mk.mirantis.com -Ril | xargs -I{} bash -c \"echo {}; sed -i 's/apt-mk.mirantis.com/apt.mcp.mirantis.net/g' {}\"")
common.infoMsg("Running kitchen test with environment:" + testSuite)
- ruby.runKitchenTests("", testSuite)
+ ruby.runKitchenTests(envOverrides.join(' '), testSuite)
} else {
throw new Exception("KITCHEN_ENV parameter is empty or invalid. This may indicate wrong env settings of initial test job or .travis.yml file.")
}
} else {
- throw new Exception(".kitchen.yml file not found, no kitchen tests triggered.")
+ throw new Exception(".kitchen.yml nor .kitchen.openstack.yml file not found, no kitchen tests triggered.")
}
}
}
diff --git a/test-salt-formulas-pipeline.groovy b/test-salt-formulas-pipeline.groovy
index 088a744..4326433 100644
--- a/test-salt-formulas-pipeline.groovy
+++ b/test-salt-formulas-pipeline.groovy
@@ -19,8 +19,10 @@
def checkouted = false
+envOverrides = []
futureFormulas = []
failedFormulas = []
+kitchenFileName = ''
def setupRunner(defaultGitRef, defaultGitUrl) {
def branches = [:]
@@ -173,17 +175,19 @@
}/** TODO: End of block for removal */
} else {
if (checkouted) {
- if (fileExists(".kitchen.yml") || fileExists(".kitchen.openstack.yml")) {
- if (fileExists(".kitchen.openstack.yml")) {
- common.infoMsg("Openstack Kitchen test configuration found, running Openstack kitchen tests.")
- if (fileExists(".kitchen.yml")) {
- common.infoMsg("Ignoring the docker Kitchen test configuration file.")
- }
- } else {
- common.infoMsg("Docker Kitchen test configuration found, running Docker kitchen tests.")
- }
+ if (fileExists(".kitchen.openstack.yml")) {
+ common.infoMsg("Openstack Kitchen test configuration found, running Openstack kitchen tests.")
+ kitchenFileName = ".kitchen.openstack.yml"
+ envOverrides.add("KITCHEN_YAML=${kitchenFileName}")
+ rubyVersion = '2.5.0'
+ } else if (fileExists(".kitchen.yml")) {
+ common.infoMsg("Docker Kitchen test configuration found, running Docker kitchen tests.")
+ kitchenFileName = ".kitchen.yml"
+ rubyVersion = '2.4.1'
+ }
+ if (kitchenFileName) {
def kitchenEnvs = []
- ruby.ensureRubyEnv()
+ ruby.ensureRubyEnv(rubyVersion)
if (!fileExists("Gemfile")) {
sh("curl -s -o ./Gemfile 'https://gerrit.mcp.mirantis.com/gitweb?p=salt-formulas/salt-formulas-scripts.git;a=blob_plain;f=Gemfile;hb=refs/heads/master'")
ruby.installKitchen()
@@ -191,8 +195,8 @@
common.infoMsg("Override Gemfile found in the kitchen directory, using it.")
ruby.installKitchen()
}
- common.infoMsg = ruby.runKitchenCommand("list -b")
- kitchenEnvs = ruby.runKitchenCommand("list -b").split()
+ common.infoMsg = ruby.runKitchenCommand("list -b", envOverrides.join(' '))
+ kitchenEnvs = ruby.runKitchenCommand("list -b", envOverrides.join(' ')).split()
common.infoMsg(kitchenEnvs)
common.infoMsg("Running kitchen testing in parallel mode")
if (CUSTOM_KITCHEN_ENVS != null && CUSTOM_KITCHEN_ENVS != '') {
@@ -207,9 +211,11 @@
}
setupRunner(defaultGitRef, defaultGitUrl)
} else {
+ common.errorMsg("No enviroments defined in the Kitchen file: ${kitchenFileName}")
+ }
+ } else {
common.warningMsg(".kitchen.yml nor .kitchen.openstack.yml file not found, no kitchen tests triggered.")
}
- }
}
}
}
diff --git a/test-salt-model-node.groovy b/test-salt-model-node.groovy
index b9c0356..154df3d 100644
--- a/test-salt-model-node.groovy
+++ b/test-salt-model-node.groovy
@@ -73,6 +73,7 @@
'reclassEnv': workspace,
'distribRevision': distribRevision,
'dockerMaxCpus': MAX_CPU_PER_JOB.toInteger(),
+ 'dockerExtraOpts': [ '--memory=3g' ],
'ignoreClassNotfound': RECLASS_IGNORE_CLASS_NOTFOUND,
'aptRepoUrl': APT_REPOSITORY,
'aptRepoGPG': APT_REPOSITORY_GPG,
diff --git a/test-salt-model-wrapper.groovy b/test-salt-model-wrapper.groovy
index 9913c5c..42aa4e9 100644
--- a/test-salt-model-wrapper.groovy
+++ b/test-salt-model-wrapper.groovy
@@ -55,9 +55,9 @@
}
// run needed job with params
-def runTests(String jobName, ArrayList jobParams, String threadName = '') {
+def runTests(String jobName, ArrayList jobParams, String threadName = '', Boolean voteOverride = null) {
threadName = threadName ? threadName : jobName
- def propagateStatus = voteMatrix.get(jobName, true)
+ def propagateStatus = voteOverride != null ? voteOverride : voteMatrix.get(jobName, true)
return {
def jobBuild = build job: jobName, propagate: false, parameters: jobParams
jobResultComments[threadName] = ['url': jobBuild.absoluteUrl, 'status': jobBuild.result, 'job': jobName]
@@ -205,7 +205,9 @@
buildTestParamsOld['COOKIECUTTER_TEMPLATE_REF'] = ''
buildTestParamsOld['COOKIECUTTER_TEMPLATE_BRANCH'] = oldRef
String threadName = "${branchJobName}-${oldRef}"
- branches[threadName] = runTests(branchJobName, yamlJobParameters(buildTestParamsOld), threadName)
+ // disable votes for release/2018.11.0 branch
+ overrideVote = oldRef == 'release/2018.11.0' ? false : null
+ branches[threadName] = runTests(branchJobName, yamlJobParameters(buildTestParamsOld), threadName, overrideVote)
}
}
if (gerritProject == cookiecutterTemplatesRepo) {
diff --git a/update-package.groovy b/update-package.groovy
index 10f3a85..9d36f38 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -23,8 +23,15 @@
def packages
def command
def commandKwargs
-def installSaltStack(target, pkgs){
- salt.runSaltProcessStep(pepperEnv, target, 'pkg.install', ["force_yes=True", "pkgs='$pkgs'"], null, true, 30)
+
+def installSaltStack(target, pkgs, masterUpdate = false){
+ salt.cmdRun(pepperEnv, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
+ def minions_reachable = target
+ if (masterUpdate) {
+ // in case of update Salt Master packages - check all minions are good
+ minions_reachable = '*'
+ }
+ salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': target, 'target_reachable': minions_reachable])
}
timeout(time: 12, unit: 'HOURS') {
@@ -97,7 +104,7 @@
common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
common.retry(10, 5) {
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
+ installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
}
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
@@ -128,7 +135,7 @@
common.infoMsg("During salt-minion upgrade on cfg node, pipeline lose connectivy to salt-master for 2 min. If pipeline ended with error rerun pipeline again.")
common.retry(10, 5) {
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:master and ${saltTargets[i]}")){
- installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
+ installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
}
if(salt.minionsReachable(pepperEnv, 'I@salt:master', "I@salt:minion and not I@salt:master and ${saltTargets[i]}")){
installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 49f92c3..77c0a26 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -33,11 +33,9 @@
}
def updateSaltStack(target, pkgs) {
- // wait 2 mins when salt-* packages are updated which leads to salt-* services restart
- common.retry(2, 120) {
- salt.runSaltProcessStep(venvPepper, target, 'pkg.install', ["force_yes=True", "pkgs='$pkgs'"], null, true, 5)
- }
-
+ salt.cmdRun(venvPepper, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
+ // can't use same function from pipeline lib, as at the moment of running upgrade pipeline Jenkins
+ // still using pipeline lib from current old mcp-version
common.retry(20, 60) {
salt.minionsReachable(venvPepper, 'I@salt:master', '*')
def running = salt.runSaltProcessStep(venvPepper, target, 'saltutil.running', [], null, true, 5)