Merge "Added RECLASS_VERSION param passing to test-salt-model-node pipeline"
diff --git a/build-debian-packages-telegraf.groovy b/build-debian-packages-telegraf.groovy
index 566c097..16d6259 100644
--- a/build-debian-packages-telegraf.groovy
+++ b/build-debian-packages-telegraf.groovy
@@ -47,8 +47,8 @@
}
stage("build package") {
img.inside{
- sh("""wget https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz &&
- tar xf go1.9.2.linux-amd64.tar.gz &&
+ sh("""wget https://storage.googleapis.com/golang/go1.9.4.linux-amd64.tar.gz &&
+ tar xf go1.9.4.linux-amd64.tar.gz &&
export GOROOT=\$PWD/go &&
export PATH=\$PATH:\$GOROOT/bin &&
export GOPATH=\$PWD &&
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 39590e5..c2d4943 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -54,6 +54,8 @@
def clusterDomain = templateContext.default_context.cluster_domain
def clusterName = templateContext.default_context.cluster_name
def saltMaster = templateContext.default_context.salt_master_hostname
+ def localRepositories = templateContext.default_context.local_repositories.toBoolean()
+ def offlineDeployment = templateContext.default_context.offline_deployment.toBoolean()
def cutterEnv = "${env.WORKSPACE}/cutter"
def jinjaEnv = "${env.WORKSPACE}/jinja"
def outputDestination = "${modelEnv}/classes/cluster/${clusterName}"
@@ -98,6 +100,7 @@
sh "git init"
ssh.agentSh("git submodule add ${sharedReclassUrl} 'classes/system'")
}
+
def sharedReclassBranch = templateContext.default_context.shared_reclass_branch
// Use refspec if exists first of all
if (sharedReclassBranch.toString().startsWith('refs/')) {
@@ -161,6 +164,16 @@
}
}
+ if(localRepositories && !offlineDeployment){
+ def aptlyModelUrl = templateContext.default_context.local_model_url
+ dir(path: modelEnv) {
+ ssh.agentSh "git submodule add \"${aptlyModelUrl}\" \"classes/cluster/${clusterName}/cicd/aptly\""
+ if(!(mcpVersion in ["nightly", "testing", "stable"])){
+ ssh.agentSh "cd \"classes/cluster/${clusterName}/cicd/aptly\";git fetch --tags;git checkout ${mcpVersion}"
+ }
+ }
+ }
+
stage('Generate new SaltMaster node') {
def nodeFile = "${modelEnv}/nodes/${saltMaster}.${clusterDomain}.yml"
def nodeString = """classes:
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index 836b084..ba05406 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -96,7 +96,7 @@
while (nbTry < 5) {
nbTry++
try {
- saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", clusterName, EXTRA_FORMULAS, testEnv)
+ saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", clusterName, EXTRA_FORMULAS, testEnv, "pkg", DISTRIB_REVISION)
break
} catch (Exception e) {
if (e.getMessage() == "script returned exit code 124") {
diff --git a/update-mirror-image.groovy b/update-mirror-image.groovy
index e37d537..2f71d2a 100644
--- a/update-mirror-image.groovy
+++ b/update-mirror-image.groovy
@@ -1,37 +1,62 @@
/**
- * Update mirror image
+ * Update local mirror
*
* Expected parameters:
* SALT_MASTER_CREDENTIALS Credentials to the Salt API.
- * SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
+ * SALT_MASTER_URL Full Salt API address [https://10.10.10.1:6969].
* UPDATE_APTLY Option to update Aptly
* UPDATE_APTLY_MIRRORS List of mirrors
* PUBLISH_APTLY Publish aptly snapshots
+ * RECREATE_APTLY_MIRRORS Recreate Aptly mirrors
* RECREATE_APTLY_PUBLISHES Option to recreate Aptly publishes separated by comma
* FORCE_OVERWRITE_APTLY_PUBLISHES Option to force overwrite existing packages while publishing
* CLEANUP_APTLY Option to cleanup old Aptly snapshots
+ * UPDATE_DEBMIRRORS Option to update Debmirrors
* UPDATE_DOCKER_REGISTRY Option to update Docker Registry
* CLEANUP_DOCKER_CACHE Option to cleanup locally cached Docker images
- * UPDATE_PYPI Option to update Python Packages
* UPDATE_GIT Option to update Git repositories
- * UPDATE_IMAGES Option to update VM images
+ * UPDATE_FILES Option to update static files
*
**/
-def common = new com.mirantis.mk.Common()
-def salt = new com.mirantis.mk.Salt()
-def python = new com.mirantis.mk.Python()
-def venvPepper = "venvPepper"
+common = new com.mirantis.mk.Common()
+salt = new com.mirantis.mk.Salt()
+python = new com.mirantis.mk.Python()
+venvPepper = "venvPepper"
+
+@NonCPS
+def Boolean dockerExists() {
+ def engine = salt.getPillar(venvPepper, 'I@aptly:server', "aptly:server:source:engine")
+ def matches = (engine =~ /:docker/)
+ try{
+ def test = matches[position]
+ return false
+ }catch(Exception ex){
+ return true
+ }
+}
+
timeout(time: 12, unit: 'HOURS') {
node() {
try {
python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ def dockerExists = dockerExists()
if(UPDATE_APTLY.toBoolean()){
stage('Update Aptly mirrors'){
def aptlyMirrorArgs = "-s -v"
- salt.enforceState(venvPepper, '*apt*', ['aptly.server'], true)
+ if(RECREATE_APTLY_MIRRORS.toBoolean())
+ {
+ if(dockerExists){
+ salt.cmdRun(venvPepper, 'I@aptly:server', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true)
+ }
+ else{
+ salt.cmdRun(venvPepper, 'I@aptly:server', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true, ['runas=aptly'])
+ }
+ }
+
+ salt.enforceState(venvPepper, 'I@aptly:server', ['aptly.server'], true)
sleep(10)
if(UPDATE_APTLY_MIRRORS != ""){
@@ -39,21 +64,31 @@
UPDATE_APTLY_MIRRORS = UPDATE_APTLY_MIRRORS.replaceAll("\\s","")
def mirrors = UPDATE_APTLY_MIRRORS.tokenize(",")
for(mirror in mirrors){
- salt.runSaltProcessStep(venvPepper, '*apt*', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=\"${aptlyMirrorArgs} -m ${mirror}\"", 'runas=aptly'], null, true)
+ if(dockerExists){
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=\"${aptlyMirrorArgs} -m ${mirror}\""], null, true)
+ }else{
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=\"${aptlyMirrorArgs} -m ${mirror}\"", 'runas=aptly'], null, true)
+ }
}
}
else{
common.infoMsg("Updating all Aptly mirrors.")
- salt.runSaltProcessStep(venvPepper, '*apt*', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=\"${aptlyMirrorArgs}\"", 'runas=aptly'], null, true)
+
+ if(dockerExists){
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=\"${aptlyMirrorArgs}\""], null, true)
+ }
+ else{
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=\"${aptlyMirrorArgs}\"", 'runas=aptly'], null, true)
+ }
}
}
}
if(PUBLISH_APTLY.toBoolean()){
- def aptlyPublishArgs = "-av"
+ def aptlyPublishArgs = "-v"
common.infoMsg("Publishing all Aptly snapshots.")
- salt.enforceState(venvPepper, '*apt*', ['aptly.publisher'], true)
+ salt.enforceState(venvPepper, 'I@aptly:server', ['aptly.publisher'], true)
sleep(10)
if(CLEANUP_APTLY.toBoolean()){
@@ -65,33 +100,40 @@
if(FORCE_OVERWRITE_APTLY_PUBLISHES.toBoolean()){
aptlyPublishArgs += "f"
}
- salt.runSaltProcessStep(venvPepper, '*apt*', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=\"${aptlyPublishArgs}\"", 'runas=aptly'], null, true)
+ if(dockerExists){
+ aptlyPublishArgs += " -u http://10.99.0.1:8080"
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=\"${aptlyPublishArgs}\""], null, true)
+ }
+ else{
+ aptlyPublishArgs += "a"
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=\"${aptlyPublishArgs}\"", 'runas=aptly'], null, true)
+ }
+ }
+ if(UPDATE_DEBMIRRORS.toBoolean()){
+ stage('Update Debmirrors'){
+ common.infoMsg("Updating Debmirrors")
+ salt.enforceState(venvPepper, 'I@debmirror:client', 'debmirror')
+ }
}
if(UPDATE_DOCKER_REGISTRY.toBoolean()){
stage('Update Docker images'){
common.infoMsg("Updating Docker images.")
- salt.enforceState(venvPepper, '*apt*', 'docker.client.registry')
+ salt.enforceState(venvPepper, 'I@aptly:server', 'docker.client.registry')
if(CLEANUP_DOCKER_CACHE.toBoolean()){
- salt.cmdRun(venvPepper, '*apt*', 'docker system prune --all --force')
+ salt.cmdRun(venvPepper, 'I@aptly:server', 'docker system prune --all --force')
}
}
}
- if(UPDATE_PYPI.toBoolean()){
- stage('Update PyPi packages'){
- common.infoMsg("Updating PyPi packages.")
- salt.cmdRun(venvPepper, '*apt*', 'pip2pi /srv/pypi_mirror/packages/ -r /srv/pypi_mirror/requirements.txt')
- }
- }
if(UPDATE_GIT.toBoolean()){
stage('Update Git repositories'){
common.infoMsg("Updating Git repositories.")
- salt.enforceState(venvPepper, '*apt*', ['git.server'], true)
+ salt.enforceState(venvPepper, 'I@aptly:server', ['git.server'], true)
}
}
- if(UPDATE_IMAGES.toBoolean()){
- stage('Update VM images'){
- common.infoMsg("Updating VM images.")
- salt.runSaltProcessStep(venvPepper, '*apt*', '/srv/scripts/update-images.sh')
+ if(UPDATE_FILES.toBoolean()){
+ stage('Update static files'){
+ common.infoMsg("Updating static files.")
+ salt.enforceState(venvPepper, 'I@aptly:server', ['linux.system.file'], true)
}
}
} catch (Throwable e) {
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index d423347..70256b7 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -10,14 +10,28 @@
*/
// Load shared libs
-def salt = new com.mirantis.mk.Salt()
-def common = new com.mirantis.mk.Common()
-def python = new com.mirantis.mk.Python()
-def venvPepper = "venvPepper"
+salt = new com.mirantis.mk.Salt()
+common = new com.mirantis.mk.Common()
+python = new com.mirantis.mk.Python()
+venvPepper = "venvPepper"
+
+@NonCPS
+def Boolean dockerExists() {
+ def engine = salt.getPillar(venvPepper, 'I@aptly:server', "aptly:server:source:engine")
+ def matches = (engine =~ /:docker/)
+ try{
+ def test = matches[position]
+ return false
+ }catch(Exception ex){
+ return true
+ }
+}
+
timeout(time: 12, unit: 'HOURS') {
node("python") {
try {
python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ def dockerExists = dockerExists()
stage("Update Reclass"){
common.infoMsg("Updating reclass model")
@@ -29,20 +43,31 @@
if(UPDATE_LOCAL_REPOS.toBoolean()){
stage("Update local repos"){
common.infoMsg("Updating local repositories")
- salt.cmdRun(venvPepper, '*apt*', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true, ['runas=aptly'])
- salt.enforceState(venvPepper, '*apt*', 'aptly', true)
- salt.runSaltProcessStep(venvPepper, '*apt*', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv", 'runas=aptly'], null, true)
- salt.runSaltProcessStep(venvPepper, '*apt*', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-afrv", 'runas=aptly'], null, true)
+ if(dockerExists){
+ salt.cmdRun(venvPepper, 'I@aptly:server', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true)
+ }
+ else{
+ salt.cmdRun(venvPepper, 'I@aptly:server', "aptly mirror list --raw | grep -E '*' | xargs -n 1 aptly mirror drop -force", true, null, true, ['runas=aptly'])
+ }
- salt.enforceState(venvPepper, '*apt*', 'docker.client.registry', true)
+ salt.enforceState(venvPepper, 'I@aptly:server', 'aptly', true)
- salt.cmdRun(venvPepper, '*apt*', "export HOME='/root';export MCP_VERSION='${MCP_VERSION}';/srv/scripts/debmirror.sh")
+ if(dockerExists){
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv"], null, true)
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-frv -u http://10.99.0.1:8080"], null, true)
+ }
+ else{
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_mirror_update.sh', "args=-sv", 'runas=aptly'], null, true)
+ salt.runSaltProcessStep(venvPepper, 'I@aptly:server', 'cmd.script', ['salt://aptly/files/aptly_publish_update.sh', "args=-afrv", 'runas=aptly'], null, true)
+ }
- salt.enforceState(venvPepper, '*apt*', 'git server', true)
+ salt.enforceState(venvPepper, 'I@aptly:server', 'docker.client.registry', true)
- salt.cmdRun(venvPepper, '*apt*', 'pip2pi /srv/pypi_mirror/packages/ -r /srv/pypi_mirror/requirements.txt')
+ salt.enforceState(venvPepper, 'I@aptly:server', 'debmirror', true)
- salt.cmdRun(venvPepper, '*apt*', '/srv/scripts/update-images.sh')
+ salt.enforceState(venvPepper, 'I@aptly:server', 'git.server', true)
+
+ salt.enforceState(venvPepper, 'I@aptly:server', 'linux.system.file', true)
}
}