Merge the tip of origin/release/proposed/2019.2.0 into origin/release/2019.2.0
ade98ae Fix pillar check conditions for backupninja pipelines
e826127 Disable dogtag restore as it does not work
60e16ba Fix pillar for getting maas nodes
ba63133 Fix include order for opencontrail.control class for backup zookeeper
2129d0b Add elasticsearch scheme var to status check
3cb7375 Add xtrabackup pillar check before run Galera restore procedure
e312cc9 Enable Octavia WSGI by default during upgrade
31618f9 Fix default value for 'maasNodes' variable
1c8fccb Add ability to switch kernel from generic to hwe for cloud-update pipeline
6312047 ceph-remove-node: add additional checks before running ceph.setup.crush state
dfa3542 Add missed uncommitted changes in reclass for upgrade-mcp-release job
f309475 Refactor Backupninja restore pipeline to support Dogtag
3772d89 Minor fixes for backupninja backup pipeline
db1c9b0 Add ability to install dependencies during package update
20a4346 Place getWorkerThreads function inplace for upgrade pipeline
6f07b59 Update maas cluster model if exists during upgrade
09ef8b1 Update getting batch size for cloud deploy and upgrade mcp jobs
19d98e6 Fix condition and pipeline name
87fe5cd Trigger Dogtag backup when Galera backup is done
d4f9541 Refactor backupninja backup pipeline to support Dogtag backups
aaeeb1d ignore set flags while checking cluster health, additional healthchecks after each osd restart
c4da1a1 Update Galery verify/restore pipeline
2d12dde Wait for all cassandra dbs are up before starting analytics services
d5eec4c Fixed typos in upgrade-mcp-release pipeline
af18b25 Add opencontrail:common class during upgrade if doesnot exist
0665907 remove ceph osd pipeline - fix partition removal while using nvme drives
e13124c Convert workerThreads toString for isInteger check
ca6aeca Support Salt batch sizing for most important pipelines
b60e170 Use pytest traceback style option from pytest.ini
14fa6e1 Get rid of hardcoded node names in pipelines
Change-Id: I1a9291da3de499aa34e91266f02c81fa4e4941a7
diff --git a/backupninja-backup-pipeline.groovy b/backupninja-backup-pipeline.groovy
index 4410ea9..74f38e6 100644
--- a/backupninja-backup-pipeline.groovy
+++ b/backupninja-backup-pipeline.groovy
@@ -3,61 +3,128 @@
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
def askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
+def backupSaltMasterAndMaas = (env.getProperty('BACKUP_SALTMASTER_AND_MAAS') ?: true).toBoolean()
+def backupDogtag = (env.getProperty('BACKUP_DOGTAG') ?: true).toBoolean()
+def saltMasterTargetMatcher = "I@backupninja:client and I@salt:master"
+def dogtagTagetMatcher = "I@backupninja:client and I@dogtag:server"
+logBackupSuccess = []
+logBackupFailure = []
+
+def checkBackupninjaLog(output, backupName='', terminateOnFailure=true) {
+ def common = new com.mirantis.mk.Common()
+ def outputPattern = java.util.regex.Pattern.compile("\\d+")
+ def outputMatcher = outputPattern.matcher(output)
+ if (outputMatcher.find()) {
+ try {
+ result = outputMatcher.getAt([0, 1, 2, 3])
+ if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)) {
+ common.successMsg("[${backupName}] - Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
+ logBackupSuccess.add(backupName)
+ } else {
+ common.errorMsg("[${backupName}] - Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
+ logBackupFailure.add(backupName)
+ }
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("[${backupName}] - Backupninja log parsing failed.")
+ logBackupFailure.add(backupName)
+ }
+ }
+}
timeout(time: 12, unit: 'HOURS') {
node() {
- def backupNode = ''
+ def saltMasterBackupNode = ''
+ def dogtagBackupNode = ''
def backupServer = ''
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
stage('Verify pillar for backups') {
- try {
- def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
- if (masterPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-master pillar.')
+ if (backupSaltMasterAndMaas) {
+ try {
+ def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data').get('return')[0].values()[0]
+ if (masterPillar.isEmpty()) {
+ throw new Exception("Problem with salt-master pillar on 'I@salt:master' node.")
+ }
+ def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data').get('return')[0].values()[0]
+ if (minionPillar.isEmpty()) {
+ throw new Exception("Problem with salt-minion pillar on I@salt:master node.")
+ }
}
- def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
- if (minionPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-minion pillar.')
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master.html')
+ throw e
}
}
- catch (Exception e) {
- common.errorMsg(e.getMessage())
- common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master.html')
- return
+ if (backupDogtag) {
+ try {
+ def dogtagPillar = salt.getPillar(pepperEnv, "I@dogtag:server", "dogtag:server").get('return')[0].values()[0]
+ if (dogtagPillar.isEmpty()) {
+ throw new Exception("Problem with dogtag pillar on I@dogtag:server node.")
+ }
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Looks like dogtag pillar is not defined. Fix your pillar or disable dogtag backup by setting the BACKUP_DOGTAG parameter to False if you're using different barbican backend.")
+ throw e
+ }
}
}
stage('Check backup location') {
- try {
- backupNode = salt.getMinions(pepperEnv, "I@backupninja:client")[0]
- salt.minionsReachable(pepperEnv, "I@salt:master", backupNode)
- }
- catch (Exception e) {
- common.errorMsg(e.getMessage())
- common.errorMsg("Pipeline wasn't able to detect backupninja:client pillar or the minion is not reachable")
- currentBuild.result = "FAILURE"
- return
- }
-
- def postgresqlMajorVersion = salt.getPillar(pepperEnv, 'I@salt:master', '_param:postgresql_major_version').get('return')[0].values()[0]
- if (! postgresqlMajorVersion) {
- input message: "Can't get _param:postgresql_major_version parameter, which is required to determine postgresql-client version. Is it defined in pillar? Confirm to proceed anyway."
- } else {
- def postgresqlClientPackage = "postgresql-client-${postgresqlMajorVersion}"
+ if (backupSaltMasterAndMaas) {
try {
- if (!salt.isPackageInstalled(['saltId': pepperEnv, 'target': backupNode, 'packageName': postgresqlClientPackage, 'output': false])) {
- if (askConfirmation) {
- input message: "Do you want to install ${postgresqlClientPackage} package on targeted nodes: ${backupNode}? It's required to make backup. Click to confirm"
+ saltMasterBackupNode = salt.getMinionsSorted(pepperEnv, saltMasterTargetMatcher)[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", saltMasterBackupNode)
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Pipeline wasn't able to detect backupninja:client pillar on Salt master node or the minion is not reachable")
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+
+ def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
+ if (!maasNodes.isEmpty()) {
+ def postgresqlMajorVersion = salt.getPillar(pepperEnv, 'I@salt:master', '_param:postgresql_major_version').get('return')[0].values()[0]
+ if (! postgresqlMajorVersion) {
+ common.errorMsg("Can't get _param:postgresql_major_version parameter, which is required to determine postgresql-client version. Is it defined in pillar?")
+ if (askConfirmation) {
+ input message: "Confirm to proceed anyway."
+ }
+ } else {
+ def postgresqlClientPackages = "postgresql-client-${postgresqlMajorVersion}"
+ try {
+ if (!salt.isPackageInstalled(['saltId': pepperEnv, 'target': saltMasterBackupNode, 'packageName': postgresqlClientPackages, 'output': false])) {
+ if (askConfirmation) {
+ input message: "Do you want to install ${postgresqlClientPackages} package on targeted nodes: ${saltMasterBackupNode}? It's required to make backup. Click to confirm."
+ } else {
+ common.infoMsg("Package ${postgresqlClientPackages} will be installed. It's required to make backup.")
+ }
+ // update also common fake package
+ salt.runSaltProcessStep(pepperEnv, saltMasterBackupNode, 'pkg.install', ["postgresql-client,${postgresqlClientPackages}"])
+ }
+ } catch (Exception e) {
+ common.errorMsg("Unable to determine status of ${postgresqlClientPackages} packages on target nodes: ${saltMasterBackupNode}.")
+ if (askConfirmation) {
+ input message: "Do you want to continue? Click to confirm"
+ }
}
- // update also common fake package
- salt.runSaltProcessStep(pepperEnv, backupNode, 'pkg.install', ["postgresql-client,${postgresqlClientPackage}"])
}
- } catch (Exception e) {
- common.errorMsg("Unable to determine status of ${postgresqlClientPackage} packages on target nodes: ${backupNode}.")
- if (askConfirmation) {
- input message: "Do you want to continue? Click to confirm"
- }
+ }
+ }
+ if (backupDogtag) {
+ try {
+ dogtagBackupNode = salt.getMinionsSorted(pepperEnv, dogtagTagetMatcher)[0]
+ salt.minionsReachable(pepperEnv, "I@salt:master", dogtagBackupNode)
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg("Pipeline wasn't able to detect node with backupninja:client and dogtag:server pillars defined or the minion is not reachable")
+ currentBuild.result = "FAILURE"
+ throw e
}
}
@@ -69,45 +136,48 @@
common.errorMsg(e.getMessage())
common.errorMsg("Pipeline wasn't able to detect backupninja:server pillar or the minion is not reachable")
currentBuild.result = "FAILURE"
- return
+ throw e
}
}
stage('Prepare for backup') {
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:client', 'state': 'backupninja'])
- def backupMasterSource = salt.getReturnValues(salt.getPillar(pepperEnv, backupNode, 'salt:master:initial_data:source'))
- def backupMinionSource = salt.getReturnValues(salt.getPillar(pepperEnv, backupNode, 'salt:minion:initial_data:source'))
- [backupServer, backupMasterSource, backupMinionSource].unique().each {
- salt.cmdRun(pepperEnv, backupNode, "ssh-keygen -F ${it} || ssh-keyscan -H ${it} >> /root/.ssh/known_hosts")
+ if (backupSaltMasterAndMaas) {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
+ salt.enforceState(['saltId': pepperEnv, 'target': saltMasterTargetMatcher, 'state': 'backupninja'])
+ def backupMasterSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:master:initial_data:source'))
+ def backupMinionSource = salt.getReturnValues(salt.getPillar(pepperEnv, saltMasterBackupNode, 'salt:minion:initial_data:source'))
+ // TODO: Remove ssh-keyscan once we have openssh meta for backupninja implemented
+ [backupServer, backupMasterSource, backupMinionSource].unique().each {
+ salt.cmdRun(pepperEnv, saltMasterBackupNode, "ssh-keygen -F ${it} || ssh-keyscan -H ${it} >> /root/.ssh/known_hosts")
+ }
+ def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
+ if (!maasNodes.isEmpty()) {
+ common.infoMsg("Trying to save maas file permissions on ${maasNodes} if possible")
+ salt.cmdRun(pepperEnv, 'I@maas:region', 'which getfacl && getfacl -pR /var/lib/maas/ > /var/lib/maas/file_permissions.txt || true')
+ }
}
- def maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
- if (!maasNodes.isEmpty()) {
- common.infoMsg("Trying to save maas file permissions on ${maasNodes} if possible")
- salt.cmdRun(pepperEnv, 'I@maas:region', 'which getfacl && getfacl -pR /var/lib/maas/ > /var/lib/maas/file_permissions.txt || true')
+ if (backupDogtag) {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@backupninja:server', 'state': 'backupninja'])
+ salt.enforceState(['saltId': pepperEnv, 'target': dogtagTagetMatcher, 'state': 'backupninja'])
}
}
stage('Backup') {
- def output = salt.getReturnValues(salt.cmdRun(pepperEnv, backupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
- def outputPattern = java.util.regex.Pattern.compile("\\d+")
- def outputMatcher = outputPattern.matcher(output)
- if (outputMatcher.find()) {
- try {
- result = outputMatcher.getAt([0, 1, 2, 3])
- }
- catch (Exception e) {
- common.errorMsg(e.getMessage())
- common.errorMsg("Parsing failed.")
- currentBuild.result = "FAILURE"
- return
- }
+ if (backupSaltMasterAndMaas) {
+ def output = salt.getReturnValues(salt.cmdRun(pepperEnv, saltMasterBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
+ checkBackupninjaLog(output, "Salt Master/MAAS")
}
- if (result[1] != null && result[1] instanceof String && result[1].isInteger() && (result[1].toInteger() < 1)) {
- common.successMsg("Backup successfully finished " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
- } else {
- common.errorMsg("Backup failed. Found " + result[1] + " fatals, " + result[2] + " errors " + result[3] + " warnings.")
+ if (backupDogtag) {
+ def output = salt.getReturnValues(salt.cmdRun(pepperEnv, dogtagBackupNode, "su root -c 'backupninja --now -d'")).readLines()[-2]
+ checkBackupninjaLog(output, "Dogtag")
+ }
+ }
+ stage('Results') {
+ if (logBackupSuccess.size() > 0) {
+ common.infoMsg("Following backups finished successfully: ${logBackupSuccess.join(",")}")
+ }
+ if (logBackupFailure.size() > 0) {
+ common.errorMsg("Following backups has failed: ${logBackupFailure.join(",")}. Make sure to check the logs.")
currentBuild.result = "FAILURE"
- return
}
}
}
-}
+}
\ No newline at end of file
diff --git a/backupninja-restore-pipeline.groovy b/backupninja-restore-pipeline.groovy
index b58756e..f617f1b 100644
--- a/backupninja-restore-pipeline.groovy
+++ b/backupninja-restore-pipeline.groovy
@@ -2,57 +2,102 @@
def salt = new com.mirantis.mk.Salt()
def python = new com.mirantis.mk.Python()
def pepperEnv = "pepperEnv"
-def maasNodes
+def maasNodes = []
+def restoreSaltMasterAndMaas = (env.getProperty('RESTORE_SALTMASTER_AND_MAAS') ?: true).toBoolean()
+def restoreDogtag = (env.getProperty('RESTORE_DOGTAG') ?: true).toBoolean()
timeout(time: 12, unit: 'HOURS') {
node() {
+ if (restoreDogtag) {
+ common.warningMsg("Dogtag restore does not work and disabled by default. For more information check the docs https://docs.mirantis.com/mcp/q4-18/mcp-operations-guide/backup-restore.html")
+ }
+ restoreDogtag = false
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
- stage('Salt-Master restore') {
- common.infoMsg('Verify pillar for salt-master backups')
- try {
- def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data')
- if(masterPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-master pillar.')
- }
- def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data')
- if(minionPillar['return'].isEmpty()) {
- throw new Exception('Problem with salt-minion pillar.')
- }
- }
- catch (Exception e){
- common.errorMsg(e.getMessage())
- common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master/salt-master-restore.html')
- return
- }
- maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
- common.infoMsg('Performing restore')
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.master.restore'])
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.minion.restore'])
- salt.fullRefresh(pepperEnv, '*')
-
- common.infoMsg('Validating output')
- common.infoMsg('Salt-Keys')
- salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key")
- common.infoMsg('Salt-master CA')
- salt.cmdRun(pepperEnv, 'I@salt:master', "ls -la /etc/pki/ca/salt_master_ca/")
- }
- if (!maasNodes.isEmpty()) {
- stage('MAAS Restore') {
- common.infoMsg('Verify pillar for MaaS backup')
+ stage('Verify pillar for restore') {
+ if (restoreSaltMasterAndMaas) {
try {
- def maaSPillar = salt.getPillar(pepperEnv, "I@maas:region", 'maas:region:database:initial_data')
- if (maaSPillar['return'].isEmpty()) {
- throw new Exception('Problem with MaaS pillar.')
+ def masterPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:master:initial_data').get('return')[0].values()[0]
+ if(masterPillar.isEmpty()) {
+ throw new Exception("Problem with salt-master pillar on 'I@salt:master' node.")
+ }
+ def minionPillar = salt.getPillar(pepperEnv, "I@salt:master", 'salt:minion:initial_data').get('return')[0].values()[0]
+ if(minionPillar.isEmpty()) {
+ throw new Exception("Problem with salt-minion pillar on 'I@salt:master' node.")
+ }
+ }
+ catch (Exception e){
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/salt-master/salt-master-restore.html')
+ throw e
+ }
+ maasNodes = salt.getMinions(pepperEnv, 'I@maas:region')
+ }
+ if (!maasNodes.isEmpty()) {
+ try {
+ def maaSPillar = salt.getPillar(pepperEnv, "I@maas:region", 'maas:region:database:initial_data').get('return')[0].values()[0]
+ if (maaSPillar.isEmpty()) {
+ throw new Exception("Problem with MaaS pillar on 'I@maas:region' node.")
}
}
catch (Exception e) {
common.errorMsg(e.getMessage())
- common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/backupninja-postgresql/backupninja-postgresql-restore.html')
- return
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/maas-postgresql/backupninja-postgresql-restore.html')
+ throw e
}
- salt.enforceState(['saltId': pepperEnv, 'target': 'I@maas:region', 'state': 'maas.region'])
+ } else {
+ common.warningMsg("No MaaS Pillar was found. You can ignore this if it's expected. Otherwise you should fix you pillar. Check: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/maas-postgresql/backupninja-postgresql-restore.html")
+ }
+ if (restoreDogtag) {
+ try {
+ def dogtagPillar = salt.getPillar(pepperEnv, "I@dogtag:server:role:master", 'dogtag:server:initial_data').get('return')[0].values()[0]
+ if (dogtagPillar.isEmpty()) {
+ throw new Exception("Problem with Dogtag pillar on 'I@dogtag:server:role:master' node.")
+ }
+ }
+ catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/dogtag/restore-dogtag.html')
+ throw e
+ }
+ }
+ }
+ stage('Restore') {
+ if (restoreSaltMasterAndMaas) {
+ common.infoMsg('Starting salt-master restore')
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.master.restore'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': 'salt.minion.restore'])
+ salt.fullRefresh(pepperEnv, '*')
+ common.infoMsg('Validating output')
+ common.infoMsg('Salt-Keys')
+ salt.cmdRun(pepperEnv, 'I@salt:master', "salt-key")
+ common.infoMsg('Salt-master CA')
+ salt.cmdRun(pepperEnv, 'I@salt:master', "ls -la /etc/pki/ca/salt_master_ca/")
+ if (!maasNodes.isEmpty()) {
+ common.infoMsg('Starting MaaS restore')
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@maas:region', 'state': 'maas.region'])
+ }
+ }
+ if (restoreDogtag) {
+ salt.runSaltProcessStep(pepperEnv, 'I@dogtag:server:role:slave', 'service.stop', ['dirsrv@pki-tomcat.service'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@dogtag:server:role:master', 'state': 'dogtag.server.restore'])
+ salt.runSaltProcessStep(pepperEnv, 'I@dogtag:server:role:slave', 'service.start', ['dirsrv@pki-tomcat.service'])
+ }
+ }
+ stage('After restore steps') {
+ if (restoreSaltMasterAndMaas) {
+ common.infoMsg("No more steps for Salt Master and MaaS restore are required.")
+ }
+ if (restoreDogtag) {
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@salt:master', 'state': ['salt', 'reclass']])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@dogtag:server:role:master', 'state': 'dogtag.server'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@dogtag:server', 'state': 'dogtag.server'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@haproxy:proxy', 'state': 'haproxy'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@barbican:server:role:primary', 'state': 'barbican.server'])
+ salt.enforceState(['saltId': pepperEnv, 'target': 'I@barbican:server', 'state': 'barbican.server'])
+ salt.cmdRun(pepperEnv, 'I@barbican:server', 'rm /etc/barbican/alias/*')
+ salt.runSaltProcessStep(pepperEnv, 'I@barbican:server', 'service.restart', 'apache2')
}
}
}
diff --git a/ceph-remove-node.groovy b/ceph-remove-node.groovy
index 0fba6a0..766dda1 100644
--- a/ceph-remove-node.groovy
+++ b/ceph-remove-node.groovy
@@ -304,7 +304,8 @@
}
}
- if (HOST_TYPE.toLowerCase() == 'osd' && GENERATE_CRUSHMAP.toBoolean() == true) {
+ def crushmap_target = salt.getMinions(pepperEnv, "I@ceph:setup:crush")
+ if (HOST_TYPE.toLowerCase() == 'osd' && GENERATE_CRUSHMAP.toBoolean() == true && crushmap_target ) {
stage('Generate CRUSHMAP') {
salt.enforceState(pepperEnv, 'I@ceph:setup:crush', 'ceph.setup.crush', true)
}
diff --git a/ceph-remove-osd.groovy b/ceph-remove-osd.groovy
index b89f3f2..098fb98 100644
--- a/ceph-remove-osd.groovy
+++ b/ceph-remove-osd.groovy
@@ -56,11 +56,22 @@
}
}
if (partition?.trim()) {
- // dev = /dev/sdi
- def dev = partition.replaceAll('\\d+$', "")
- // part_id = 2
- def part_id = partition.substring(partition.lastIndexOf("/")+1).replaceAll("[^0-9]+", "")
- runCephCommand(master, target, "Ignore | parted ${dev} rm ${part_id}")
+ if (partition.contains("nvme")) {
+ // dev = /dev/nvme1n1p1
+ def dev = partition.replaceAll('\\d+$', "")
+ print("Skipping " + dev)
+ // part_id = 2
+ def part_id = partition.substring(partition.lastIndexOf("p")+1).replaceAll("[^0-9]+", "")
+ print("Skipping" + part_id)
+ runCephCommand(master, target, "Ignore | parted ${dev} rm ${part_id}")
+ }
+ else {
+ // dev = /dev/sdi
+ def dev = partition.replaceAll('\\d+$', "")
+ // part_id = 2
+ def part_id = partition.substring(partition.lastIndexOf("/")+1).replaceAll("[^0-9]+", "")
+ runCephCommand(master, target, "Ignore | parted ${dev} rm ${part_id}")
+ }
}
return
}
diff --git a/ceph-upgrade.groovy b/ceph-upgrade.groovy
index c4881bc..dd75875 100644
--- a/ceph-upgrade.groovy
+++ b/ceph-upgrade.groovy
@@ -33,20 +33,27 @@
return salt.cmdRun(master, target, cmd)
}
-def waitForHealthy(master, count=0, attempts=300) {
+def waitForHealthy(master, flags, count=0, attempts=300) {
// wait for healthy cluster
while (count<attempts) {
def health = runCephCommand(master, ADMIN_HOST, 'ceph health')['return'][0].values()[0]
if (health.contains('HEALTH_OK')) {
common.infoMsg('Cluster is healthy')
break;
+ } else {
+ for (flag in flags) {
+ if (health.contains(flag + ' flag(s) set') && !(health.contains('down'))) {
+ common.infoMsg('Cluster is healthy')
+ return;
+ }
+ }
}
count++
sleep(10)
}
}
-def backup(master, target) {
+def backup(master, flags, target) {
stage("backup ${target}") {
if (target == 'osd') {
@@ -72,7 +79,7 @@
def provider_pillar = salt.getPillar(master, "${kvm01}", "salt:control:cluster:internal:node:${minion_name}:provider")
def minionProvider = provider_pillar['return'][0].values()[0]
- waitForHealthy(master)
+ waitForHealthy(master, flags)
try {
salt.cmdRun(master, "${minionProvider}", "[ ! -f ${BACKUP_DIR}/${minion_name}.${domain}.qcow2.bak ] && virsh destroy ${minion_name}.${domain}")
} catch (Exception e) {
@@ -89,14 +96,14 @@
common.warningMsg(e)
}
salt.minionsReachable(master, 'I@salt:master', "${minion_name}*")
- waitForHealthy(master)
+ waitForHealthy(master, flags)
}
}
}
return
}
-def upgrade(master, target) {
+def upgrade(master, target, flags) {
stage("Change ${target} repos") {
salt.runSaltProcessStep(master, "I@ceph:${target}", 'saltutil.refresh_pillar', [], null, true, 5)
@@ -127,12 +134,21 @@
}
// restart services
stage("Restart ${target} services on ${minion}") {
- runCephCommand(master, "${minion}", "systemctl restart ceph-${target}.target")
+ if (target == 'osd') {
+ def osds = salt.getGrain(master, "${minion}", 'ceph:ceph_disk').values()[0]
+ osds[0].values()[0].values()[0].each { osd,param ->
+ runCephCommand(master, "${minion}", "systemctl restart ceph-${target}@${osd}")
+ waitForHealthy(master, flags)
+ }
+ } else {
+ runCephCommand(master, "${minion}", "systemctl restart ceph-${target}.target")
+ waitForHealthy(master, flags)
+ }
}
stage("Verify services for ${minion}") {
sleep(10)
- runCephCommand(master, ADMIN_HOST, "ceph -s")
+ runCephCommand(master, "${minion}", "systemctl status ceph-${target}.target")
}
stage('Ask for manual confirmation') {
@@ -196,23 +212,23 @@
}
if (STAGE_UPGRADE_MON.toBoolean() == true) {
- upgrade(pepperEnv, 'mon')
+ upgrade(pepperEnv, 'mon', flags)
}
if (STAGE_UPGRADE_MGR.toBoolean() == true) {
- upgrade(pepperEnv, 'mgr')
+ upgrade(pepperEnv, 'mgr', flags)
}
if (STAGE_UPGRADE_OSD.toBoolean() == true) {
- upgrade(pepperEnv, 'osd')
+ upgrade(pepperEnv, 'osd', flags)
}
if (STAGE_UPGRADE_RGW.toBoolean() == true) {
- upgrade(pepperEnv, 'radosgw')
+ upgrade(pepperEnv, 'radosgw', flags)
}
if (STAGE_UPGRADE_CLIENT.toBoolean() == true) {
- upgrade(pepperEnv, 'common')
+ upgrade(pepperEnv, 'common', flags)
}
// remove cluster flags
@@ -246,7 +262,7 @@
// wait for healthy cluster
if (WAIT_FOR_HEALTHY.toBoolean() == true) {
- waitForHealthy(pepperEnv)
+ waitForHealthy(pepperEnv, flags)
}
}
}
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index aa1a644..5f19480 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -46,6 +46,10 @@
* SALT_VERSION Version of Salt which is going to be installed i.e. 'stable 2016.3' or 'stable 2017.7' etc.
*
* EXTRA_TARGET The value will be added to target nodes
+ * BATCH_SIZE Use batching for states, which may be targeted for huge amount of nodes. Format:
+ - 10 - number of nodes
+ - 10% - percentage of all targeted nodes
+
*
* Test settings:
* TEST_K8S_API_SERVER Kubernetes API address
@@ -104,6 +108,10 @@
if (common.validInputParam('EXTRA_TARGET')) {
extra_tgt = "${EXTRA_TARGET}"
}
+def batch_size = ''
+if (common.validInputParam('BATCH_SIZE')) {
+ batch_size = "${BATCH_SIZE}"
+}
timeout(time: 12, unit: 'HOURS') {
node(slave_node) {
@@ -333,9 +341,12 @@
//
// Install
//
+ if (!batch_size) {
+ batch_size = salt.getWorkerThreads(venvPepper)
+ }
// Check if all minions are reachable and ready
- salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': '*'])
+ salt.checkTargetMinionsReady(['saltId': venvPepper, 'target': '*', batch: batch_size])
if (common.checkContains('STACK_INSTALL', 'core')) {
stage('Install core infrastructure') {
@@ -343,7 +354,7 @@
if (common.validInputParam('STATIC_MGMT_NETWORK')) {
staticMgmtNetwork = STATIC_MGMT_NETWORK.toBoolean()
}
- orchestrate.installFoundationInfra(venvPepper, staticMgmtNetwork, extra_tgt)
+ orchestrate.installFoundationInfra(venvPepper, staticMgmtNetwork, extra_tgt, batch_size)
if (common.checkContains('STACK_INSTALL', 'kvm')) {
orchestrate.installInfraKvm(venvPepper, extra_tgt)
@@ -467,7 +478,7 @@
// Workaround for PROD-17765 issue to prevent crashes of keystone.role_present state.
// More details: https://mirantis.jira.com/browse/PROD-17765
salt.restartSaltMinion(venvPepper, "I@keystone:client ${extra_tgt}")
- salt.minionsReachable(venvPepper, "I@salt:master and *01* ${extra_tgt}", 'I@keystone:client', null, 10, 6)
+ salt.minionsReachable(venvPepper, 'I@salt:master', 'I@keystone:client ${extra_tgt}', null, 10, 6)
stage('Install OpenStack network') {
@@ -497,7 +508,7 @@
}
stage('Install OpenStack compute') {
- orchestrate.installOpenstackCompute(venvPepper, extra_tgt)
+ orchestrate.installOpenstackCompute(venvPepper, extra_tgt, batch_size)
if (common.checkContains('STACK_INSTALL', 'contrail')) {
orchestrate.installContrailCompute(venvPepper, extra_tgt)
@@ -574,7 +585,7 @@
test.executeConformance(config)
} else {
def output_file = image.replaceAll('/', '-') + '.output'
- def target = "ctl01* ${extra_tgt}"
+ def target = "I@keystone:server:role:primary ${extra_tgt}"
def conformance_output_file = 'conformance_test.tar'
// run image
@@ -645,7 +656,7 @@
"py.test --junit-xml=${report_dir}report.xml" +
" --html=${report_dir}report.html -v vapor/tests/ -k 'not destructive' "
- salt.runSaltProcessStep(venvPepper, 'cfg*', 'saltutil.refresh_pillar', [], null, true)
+ salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'saltutil.refresh_pillar', [], null, true)
salt.enforceState(venvPepper, 'I@opencontrail:test' , 'opencontrail.test' , true)
salt.cmdRun(venvPepper, 'I@opencontrail:test', cmd, false)
@@ -662,7 +673,7 @@
def gluster_compound = "I@glusterfs:server ${extra_tgt}"
def salt_ca_compound = "I@salt:minion:ca:salt_master_ca ${extra_tgt}"
// Enforce highstate asynchronous only on the nodes which are not glusterfs servers
- salt.enforceHighstate(venvPepper, '* and not ' + gluster_compound + ' and not ' + salt_ca_compound)
+ salt.enforceHighstate(venvPepper, '* and not ' + gluster_compound + ' and not ' + salt_ca_compound, batch_size)
// Iterate over nonempty set of gluster servers and apply highstates one by one
// TODO: switch to batch once salt 2017.7+ would be used
def saltcaMinions = salt.getMinionsSorted(venvPepper, salt_ca_compound)
diff --git a/cloud-update.groovy b/cloud-update.groovy
index 0ea5fea..9945d33 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -190,6 +190,70 @@
}
} else {
if (distUpgrade) {
+ common.infoMsg("Checking availability of Linux HWE Kernel...")
+ def switchHwe = false
+ def nodesOut = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], 'linux_kernel_switch.check_hwe_kernel').get('return')[0]
+ def targetHWE = []
+ for (node in nodesOut) {
+ def nodeName = node.getKey()
+ def statusPkgs = node.getValue()
+ if (statusPkgs) {
+ statusPkgs.each { pkg, pkgStatus ->
+ if (pkgStatus instanceof String) {
+ common.warningMsg("Target ${nodeName} has no installed Linux HWE Kernel package: ${pkg}")
+ if (! targetHWE.contains(nodeName)) {
+ targetHWE.add(nodeName)
+ }
+ }
+ }
+ } else {
+ common.warningMsg("Target ${nodeName} has no info about Linux HWE Kernel, check formula or resync minion data.")
+ }
+ }
+ if (targetHWE) {
+ if (INTERACTIVE.toBoolean()) {
+ try {
+ input message: "Do you want to switch from generic to hwe kernel for ${targetHWE} nodes? Click to confirm", ok: 'Switch to HWE'
+ switchHwe = true
+ } catch (Exception ex) {
+ common.warningMsg("Kernel switch from generic to hwe for ${targetHWE} cancelled. Continue dist-upgrade with existing kernel.")
+ }
+ } else {
+ switchHwe = true
+ }
+ }
+ if (switchHwe) {
+ def onlyKernel='True'
+ def targetHWECompound = targetHWE.join(' or ')
+ if (INTERACTIVE.toBoolean()) {
+ try {
+ input message: "Install HWE headers and generic packages?", ok: 'Install'
+ onlyKernel='False'
+ common.infoMsg("HWE Kernel, headers and generic packages will be installed.")
+ } catch (Exception e) {
+ common.infoMsg("Only HWE Kernel packages will be installed.")
+ }
+ } else {
+ onlyKernel='False'
+ }
+ salt.runSaltCommand(pepperEnv, 'local', ['expression': targetHWECompound, 'type': 'compound'], 'linux_kernel_switch.switch_kernel', false, "only_kernel=${onlyKernel}")
+ common.infoMsg("HWE Kernel has been installed on ${targetHWE} nodes")
+ def rebootNow = true
+ if (INTERACTIVE.toBoolean()) {
+ try {
+ input message: "To finish switch on HWE kernel it is needed to reboot. Reboot nodes ${targetHWE} now?", ok: 'Reboot'
+ } catch (Exception e) {
+ common.warningMsg("HWE Kernel is not used. Please reboot nodes ${targetHWE} manually to finish kernel switch.")
+ rebootNow = false
+ }
+ }
+ if (rebootNow) {
+ common.infoMsg('Performing nodes reboot after kernel install...')
+ salt.runSaltCommand(pepperEnv, 'local', ['expression': targetHWECompound, 'type': 'compound'], 'system.reboot', null, 'at_time=1')
+ sleep(180)
+ salt.minionsReachable(pepperEnv, 'I@salt:master', targetHWECompound, null, 10, 20)
+ }
+ }
common.retry(3){
out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' dist-upgrade'])
}
@@ -335,6 +399,22 @@
}
} else {
if (distUpgrade) {
+ salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], 'linux_kernel_switch.rollback_switch_kernel', false)
+ def rebootNow = true
+ if (INTERACTIVE.toBoolean()) {
+ try {
+ input message: "To finish kernel downgrade it is needed to reboot. Reboot nodes ${target} now?", ok: 'Reboot'
+ } catch (Exception e) {
+ common.warningMsg("Please reboot nodes ${target} manually to finish kernel downgrade.")
+ rebootNow = false
+ }
+ }
+ if (rebootNow) {
+ common.infoMsg('Performing nodes reboot after kernel downgrade...')
+ salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], 'system.reboot', null, 'at_time=1')
+ sleep(180)
+ salt.minionsReachable(pepperEnv, 'I@salt:master', target, null, 10, 20)
+ }
common.retry(3){
out = salt.runSaltProcessStep(pepperEnv, target, 'cmd.run', [args + ' dist-upgrade'])
}
diff --git a/cvp-runner.groovy b/cvp-runner.groovy
index 4a4a8b6..edbe902 100644
--- a/cvp-runner.groovy
+++ b/cvp-runner.groovy
@@ -48,7 +48,7 @@
def container_workdir = '/var/lib'
def workdir = "${container_workdir}/${test_suite_name}"
def tests_set = (env.getProperty('tests_set')) ?: ''
- def script = "pytest --junitxml ${container_workdir}/${artifacts_dir}/${xml_file} --tb=short -vv ${tests_set}"
+ def script = "pytest --junitxml ${container_workdir}/${artifacts_dir}/${xml_file} -vv ${tests_set}"
sh "mkdir -p ${artifacts_dir}"
diff --git a/deploy-virtual-edge-mom.groovy b/deploy-virtual-edge-mom.groovy
index 875195b..8d35d37 100644
--- a/deploy-virtual-edge-mom.groovy
+++ b/deploy-virtual-edge-mom.groovy
@@ -171,7 +171,7 @@
saltMasterURL = "http://${edgeBuildsInfra[ed_].description.tokenize(' ')[1]}:6969"
- enableSyndic(saltMasterURL, 'cfg01*', SALT_MASTER_CREDENTIALS, salt_mom_ip)
+ enableSyndic(saltMasterURL, 'I@salt:master', SALT_MASTER_CREDENTIALS, salt_mom_ip)
props_ = edge_deploy_schemas[ed_]['properties']
deploy_job = edge_deploy_schemas[ed_]['deploy_job_name']
diff --git a/galera-cluster-verify-restore.groovy b/galera-cluster-verify-restore.groovy
index 3faedc7..243baff 100644
--- a/galera-cluster-verify-restore.groovy
+++ b/galera-cluster-verify-restore.groovy
@@ -20,6 +20,7 @@
def restoreType = env.RESTORE_TYPE
def runRestoreDb = false
def runBackupDb = false
+def restartCluster = false
askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
checkTimeSync = (env.getProperty('CHECK_TIME_SYNC') ?: true).toBoolean()
@@ -35,12 +36,17 @@
if (restoreType.equals("BACKUP_AND_RESTORE")) {
runBackupDb = true
}
+if (restoreType.equals("RESTART_CLUSTER")) {
+ restartCluster = true
+}
timeout(time: 12, unit: 'HOURS') {
node() {
stage('Setup virtualenv for Pepper') {
python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
}
+
+ def galeraStatus = [:]
stage('Verify status') {
def sysstatTargets = 'I@xtrabackup:client or I@xtrabackup:server'
def sysstatTargetsNodes = salt.getMinions(pepperEnv, sysstatTargets)
@@ -58,50 +64,60 @@
input message: "Do you want to continue? Click to confirm"
}
}
- resultCode = galera.verifyGaleraStatus(pepperEnv, false, checkTimeSync)
- if (resultCode == 128) {
- common.errorMsg("Unable to connect to Galera Master. Trying slaves...")
- resultCode = galera.verifyGaleraStatus(pepperEnv, true, checkTimeSync)
- if (resultCode == 129) {
- common.errorMsg("Unable to obtain Galera slave minions list. Without fixing this issue, pipeline cannot continue in verification, backup and restoration. This may be caused by wrong Galera configuration or corrupted pillar data.")
+ try {
+ common.infoMsg('Checking required xtrabackup pillars...')
+ def xtrabackupRestoreFrom = salt.getPillar(pepperEnv, 'I@galera:master or I@galera:slave', 'xtrabackup:client:restore_from')
+ def xtrabackupRestoreLatest = salt.getPillar(pepperEnv, 'I@galera:master or I@galera:slave', 'xtrabackup:client:restore_full_latest')
+ if ('' in xtrabackupRestoreFrom['return'][0].values() || '' in xtrabackupRestoreLatest['return'][0].values()) {
+ throw new Exception('Pillars xtrabackup:client:restore_from or xtrabackup:client:restore_full_latest are missed for \'I@galera:master or I@galera:slave\' nodes.')
+ }
+ } catch (Exception e) {
+ common.errorMsg(e.getMessage())
+ common.errorMsg('Please fix your pillar data. For more information check docs: https://docs.mirantis.com/mcp/latest/mcp-operations-guide/backup-restore/openstack/database/xtrabackup-restore-database.html')
+ return
+ }
+ galeraStatus = galera.verifyGaleraStatus(pepperEnv, checkTimeSync)
+
+ switch (galeraStatus.error) {
+ case 128:
+ common.errorMsg("Unable to obtain Galera members minions list. Without fixing this issue, pipeline cannot continue in verification, backup and restoration. This may be caused by wrong Galera configuration or corrupted pillar data.")
currentBuild.result = "FAILURE"
return
- } else if (resultCode == 130) {
+ case 130:
common.errorMsg("Neither master or slaves are reachable. Without fixing this issue, pipeline cannot continue in verification, backup and restoration. Is at least one member of the Galera cluster up and running?")
currentBuild.result = "FAILURE"
return
- }
- }
- if (resultCode == 131) {
- common.errorMsg("Time desynced - Please fix this issue and rerun the pipeline.")
- currentBuild.result = "FAILURE"
- return
- }
- if (resultCode == 140 || resultCode == 141) {
- common.errorMsg("Disk utilization check failed - Please fix this issue and rerun the pipeline.")
- currentBuild.result = "FAILURE"
- return
- }
- if (resultCode == 1) {
- if (askConfirmation) {
- input message: "There was a problem with parsing the status output or with determining it. Do you want to run a restore?"
- } else {
- common.warningMsg("There was a problem with parsing the status output or with determining it. Try to restore.")
- }
- } else if (resultCode > 1) {
- if (askConfirmation) {
- input message: "There's something wrong with the cluster, do you want to continue with backup and/or restore?"
- } else {
- common.warningMsg("There's something wrong with the cluster, try to backup and/or restore.")
- }
- } else {
- if (askConfirmation) {
- input message: "There seems to be everything alright with the cluster, do you still want to continue with backup and/or restore?"
- } else {
- common.warningMsg("There seems to be everything alright with the cluster, no backup and no restoration will be done.")
- currentBuild.result = "SUCCESS"
+ case 131:
+ common.errorMsg("Time desynced - Please fix this issue and rerun the pipeline.")
+ currentBuild.result = "FAILURE"
return
- }
+ case 140..141:
+ common.errorMsg("Disk utilization check failed - Please fix this issue and rerun the pipeline.")
+ currentBuild.result = "FAILURE"
+ return
+ case 1:
+ if (askConfirmation) {
+ input message: "There was a problem with parsing the status output or with determining it. Do you want to run a next action: ${restoreType}?"
+ } else {
+ common.warningMsg("There was a problem with parsing the status output or with determining it. Trying to perform action: ${restoreType}.")
+ }
+ break
+ case 0:
+ if (askConfirmation) {
+ input message: "There seems to be everything alright with the cluster, do you still want to continue with next action: ${restoreType}?"
+ break
+ } else {
+ common.warningMsg("There seems to be everything alright with the cluster, no backup and no restoration will be done.")
+ currentBuild.result = "SUCCESS"
+ return
+ }
+ default:
+ if (askConfirmation) {
+ input message: "There's something wrong with the cluster, do you want to continue with action: ${restoreType}?"
+ } else {
+ common.warningMsg("There's something wrong with the cluster, trying to perform action: ${restoreType}")
+ }
+ break
}
}
if (runBackupDb) {
@@ -117,24 +133,41 @@
)
}
}
- if (runRestoreDb) {
- stage('Restore') {
- if (askConfirmation) {
- input message: "Are you sure you want to run a restore? Click to confirm"
- }
- try {
- if ((!askConfirmation && resultCode > 0) || askConfirmation) {
- galera.restoreGaleraCluster(pepperEnv, runRestoreDb)
+ if (runRestoreDb || restartCluster) {
+ if (runRestoreDb) {
+ stage('Restore') {
+ if (askConfirmation) {
+ input message: "Are you sure you want to run a restore? Click to confirm"
}
- } catch (Exception e) {
- common.errorMsg("Restoration process has failed.")
- common.errorMsg(e.getMessage())
+ try {
+ if ((!askConfirmation && resultCode > 0) || askConfirmation) {
+ galera.restoreGaleraCluster(pepperEnv, galeraStatus)
+ }
+ } catch (Exception e) {
+ common.errorMsg("Restoration process has failed.")
+ common.errorMsg(e.getMessage())
+ }
+ }
+ }
+ if (restartCluster) {
+ stage('Restart cluster') {
+ if (askConfirmation) {
+ input message: "Are you sure you want to run a restart? Click to confirm"
+ }
+ try {
+ if ((!askConfirmation && resultCode > 0) || askConfirmation) {
+ galera.restoreGaleraCluster(pepperEnv, galeraStatus, false)
+ }
+ } catch (Exception e) {
+ common.errorMsg("Restart process has failed.")
+ common.errorMsg(e.getMessage())
+ }
}
}
stage('Verify restoration result') {
common.retry(verificationRetries, 15) {
- exitCode = galera.verifyGaleraStatus(pepperEnv, false, false)
- if (exitCode >= 1) {
+ def status = galera.verifyGaleraStatus(pepperEnv, false)
+ if (status.error >= 1) {
error("Verification attempt finished with an error. This may be caused by cluster not having enough time to come up or to sync. Next verification attempt in 5 seconds.")
} else {
common.infoMsg("Restoration procedure seems to be successful. See verification report to be sure.")
diff --git a/galera-database-backup-pipeline.groovy b/galera-database-backup-pipeline.groovy
index a6d0af5..8239aa6 100644
--- a/galera-database-backup-pipeline.groovy
+++ b/galera-database-backup-pipeline.groovy
@@ -57,5 +57,19 @@
stage('Clean-up') {
salt.cmdRun(pepperEnv, backupNode, "su root -c '/usr/local/bin/innobackupex-runner.sh -c'")
}
+ stage('Backup Dogtag') {
+ if (!salt.getMinions(pepperEnv, "I@dogtag:server:enabled").isEmpty()) {
+ dogtagBackupBuild = build(job: 'backupninja_backup', parameters: [
+ [$class: 'StringParameterValue', name: 'SALT_MASTER_URL', value: SALT_MASTER_URL],
+ [$class: 'StringParameterValue', name: 'SALT_MASTER_CREDENTIALS', value: SALT_MASTER_CREDENTIALS],
+ [$class: 'BooleanParameterValue', name: 'ASK_CONFIRMATION', value: "false"],
+ [$class: 'BooleanParameterValue', name: 'BACKUP_SALTMASTER_AND_MAAS', value: "false"],
+ [$class: 'BooleanParameterValue', name: 'BACKUP_DOGTAG', value: "true"],
+ ]
+ )
+ } else {
+ common.warningMsg("Dogtag pillar not found. This is fine if you are using different Barbican backend.")
+ }
+ }
}
}
\ No newline at end of file
diff --git a/opencontrail-upgrade.groovy b/opencontrail-upgrade.groovy
index a358222..7c761a0 100644
--- a/opencontrail-upgrade.groovy
+++ b/opencontrail-upgrade.groovy
@@ -66,7 +66,7 @@
stage('Opencontrail controllers upgrade') {
- oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:primary', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
oc_component_repo = oc_component_repo['return'][0].values()[0]
@@ -103,22 +103,14 @@
args = 'apt install contrail-database -y;'
check = 'nodetool status'
- // ntw01
- runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, pepperEnv, common)
- // ntw02
- runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, pepperEnv, common)
- // ntw03
- runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv, common)
args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
check = 'contrail-status'
- // ntw01
- runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, pepperEnv, common)
- // ntw02
- runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, pepperEnv, common)
- // ntw03
- runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv, common)
try {
salt.enforceState(pepperEnv, 'I@opencontrail:control', 'opencontrail')
@@ -144,7 +136,7 @@
stage('Opencontrail analytics upgrade') {
- oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector:role:primary', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
oc_component_repo = oc_component_repo['return'][0].values()[0]
@@ -161,22 +153,14 @@
args = 'apt install contrail-database -y;'
check = 'nodetool status'
- // nal01
- runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, pepperEnv, common)
- // nal02
- runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, pepperEnv, common)
- // nal03
- runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:secondary', command, args, check, salt, pepperEnv, common)
args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
check = 'contrail-status'
- // nal01
- runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, pepperEnv, common)
- // nal02
- runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, pepperEnv, common)
- // nal03
- runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:secondary', command, args, check, salt, pepperEnv, common)
try {
salt.enforceState(pepperEnv, 'I@opencontrail:collector', 'opencontrail')
@@ -302,7 +286,7 @@
stage('Opencontrail controllers rollback') {
- oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:primary', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
oc_component_repo = oc_component_repo['return'][0].values()[0]
try {
@@ -318,22 +302,14 @@
args = 'apt install contrail-database -y --force-yes;'
check = 'nodetool status'
- // ntw01
- runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, pepperEnv, common)
- // ntw02
- runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, pepperEnv, common)
- // ntw03
- runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv, common)
args = "apt install -o Dpkg::Options::=\"--force-confold\" ${CONTROL_PKGS} -y --force-yes;"
check = 'contrail-status'
- // ntw01
- runCommonCommands('I@opencontrail:control and *01*', command, args, check, salt, pepperEnv, common)
- // ntw02
- runCommonCommands('I@opencontrail:control and *02*', command, args, check, salt, pepperEnv, common)
- // ntw03
- runCommonCommands('I@opencontrail:control and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:control:role:secondary', command, args, check, salt, pepperEnv, common)
try {
salt.enforceState(pepperEnv, 'I@opencontrail:control', 'opencontrail')
@@ -361,7 +337,7 @@
stage('Opencontrail analytics rollback') {
- oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector and *01*', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
+ oc_component_repo = salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector:role:primary', 'cmd.shell', ['grep -RE \'oc[0-9]{2,3}\' /etc/apt/sources.list* | awk \'{print $1}\' | sed \'s/ *:.*//\''], null, true)
oc_component_repo = oc_component_repo['return'][0].values()[0]
try {
@@ -377,22 +353,14 @@
args = 'apt install contrail-database -y --force-yes;'
check = 'nodetool status'
- // nal01
- runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, pepperEnv, common)
- // nal02
- runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, pepperEnv, common)
- // nal03
- runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:secondary', command, args, check, salt, pepperEnv, common)
args = "apt install -o Dpkg::Options::=\"--force-confold\" ${ANALYTIC_PKGS} -y --force-yes;"
check = 'contrail-status'
- // nal01
- runCommonCommands('I@opencontrail:collector and *01*', command, args, check, salt, pepperEnv, common)
- // nal02
- runCommonCommands('I@opencontrail:collector and *02*', command, args, check, salt, pepperEnv, common)
- // nal03
- runCommonCommands('I@opencontrail:collector and *03*', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:primary', command, args, check, salt, pepperEnv, common)
+ runCommonCommands('I@opencontrail:collector:role:secondary', command, args, check, salt, pepperEnv, common)
try {
salt.enforceState(pepperEnv, 'I@opencontrail:collector', 'opencontrail')
diff --git a/openstack-compute-install.groovy b/openstack-compute-install.groovy
index 581168a..780beac 100644
--- a/openstack-compute-install.groovy
+++ b/openstack-compute-install.groovy
@@ -5,6 +5,7 @@
* SALT_MASTER_CREDENTIALS Credentials to the Salt API.
* SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
+ * BATCH_SIZE Use batching for large amount of target nodes
*
**/
@@ -18,6 +19,11 @@
def command
def commandKwargs
+def batch_size = ''
+if (common.validInputParam('BATCH_SIZE')) {
+ batch_size = "${BATCH_SIZE}"
+}
+
timeout(time: 12, unit: 'HOURS') {
node() {
try {
@@ -43,76 +49,76 @@
common.infoMsg("First node %nodename% has trusty")
common.infoMsg("Assuming trusty on all cluster, running extra network states...")
common.infoMsg("Network iteration #1. Bonding")
- salt.enforceState(pepperEnv, targetLiveAll, 'linux.network', true)
+ salt.enforceState(pepperEnv, targetLiveAll, 'linux.network', true, true, batch_size)
common.infoMsg("Network iteration #2. Vlan tagging and bridging")
- salt.enforceState(pepperEnv, targetLiveAll, 'linux.network', true)
+ salt.enforceState(pepperEnv, targetLiveAll, 'linux.network', true, true, batch_size)
}
}
stage("Setup repositories") {
- salt.enforceState(pepperEnv, targetLiveAll, 'linux.system.repo', true)
+ salt.enforceState(pepperEnv, targetLiveAll, 'linux.system.repo', true, true, batch_size)
}
stage("Upgrade packages") {
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'pkg.upgrade', [], null, true)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'pkg.upgrade', [], batch_size, true)
}
stage("Update Hosts file") {
- salt.enforceState(pepperEnv, "I@linux:system", 'linux.network.host', true)
+ salt.enforceState(pepperEnv, "I@linux:system", 'linux.network.host', true, true, batch_size)
}
stage("Setup networking") {
// Sync all of the modules from the salt master.
- salt.syncAll(pepperEnv, targetLiveAll)
+ salt.syncAll(pepperEnv, targetLiveAll, batch_size)
// Apply state 'salt' to install python-psutil for network configuration without restarting salt-minion to avoid losing connection.
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.apply', ['salt', 'exclude=[{\'id\': \'salt_minion_service\'}, {\'id\': \'salt_minion_service_restart\'}, {\'id\': \'salt_minion_sync_all\'}]'], null, true)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.apply', ['salt', 'exclude=[{\'id\': \'salt_minion_service\'}, {\'id\': \'salt_minion_service_restart\'}, {\'id\': \'salt_minion_sync_all\'}]'], batch_size, true)
// Restart salt-minion to take effect.
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'service.restart', ['salt-minion'], null, true, 10)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'service.restart', ['salt-minion'], batch_size, true, 10)
// Configure networking excluding vhost0 interface.
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.apply', ['linux.network', 'exclude=[{\'id\': \'linux_interface_vhost0\'}]'], null, true)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.apply', ['linux.network', 'exclude=[{\'id\': \'linux_interface_vhost0\'}]'], batch_size, true)
// Kill unnecessary processes ifup/ifdown which is stuck from previous state linux.network.
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'ps.pkill', ['ifup'], null, false)
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'ps.pkill', ['ifdown'], null, false)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'ps.pkill', ['ifup'], batch_size, false)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'ps.pkill', ['ifdown'], batch_size, false)
// Restart networking to bring UP all interfaces.
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'service.restart', ['networking'], null, true, 300)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'service.restart', ['networking'], batch_size, true, 300)
}
stage("Highstate compute") {
// Execute highstate without state opencontrail.client.
common.retry(2){
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.highstate', ['exclude=opencontrail.client'], null, true)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'state.highstate', ['exclude=opencontrail.client'], batch_size, true)
}
// Apply nova state to remove libvirt default bridge virbr0.
- salt.enforceState(pepperEnv, targetLiveAll, 'nova', true)
+ salt.enforceState(pepperEnv, targetLiveAll, 'nova', true, true, batch_size)
// Execute highstate.
- salt.enforceHighstate(pepperEnv, targetLiveAll, true)
+ salt.enforceHighstate(pepperEnv, targetLiveAll, true, true, batch_size)
// Restart supervisor-vrouter.
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'service.restart', ['supervisor-vrouter'], null, true, 300)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'service.restart', ['supervisor-vrouter'], batch_size, true, 300)
// Apply salt and collectd if is present to update information about current network interfaces.
- salt.enforceState(pepperEnv, targetLiveAll, 'salt', true)
+ salt.enforceState(pepperEnv, targetLiveAll, 'salt', true, true, batch_size)
if(!salt.getPillar(pepperEnv, minions[0], "collectd")['return'][0].values()[0].isEmpty()) {
- salt.enforceState(pepperEnv, targetLiveAll, 'collectd', true)
+ salt.enforceState(pepperEnv, targetLiveAll, 'collectd', true, true, batch_size)
}
}
stage("Update/Install monitoring") {
//Collect Grains
- salt.enforceState(pepperEnv, targetLiveAll, 'salt.minion.grains')
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'saltutil.refresh_modules')
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'mine.update')
+ salt.enforceState(pepperEnv, targetLiveAll, 'salt.minion.grains', true, true, batch_size)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'saltutil.refresh_modules', [], batch_size)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'mine.update', [], batch_size)
sleep(5)
- salt.enforceState(pepperEnv, targetLiveAll, 'prometheus')
- salt.enforceState(pepperEnv, 'I@prometheus:server', 'prometheus')
+ salt.enforceState(pepperEnv, targetLiveAll, 'prometheus', true, true, batch_size)
+ salt.enforceState(pepperEnv, 'I@prometheus:server', 'prometheus', true, true, batch_size)
}
} catch (Throwable e) {
diff --git a/restore-cassandra.groovy b/restore-cassandra.groovy
index 7285c40..f1964ab 100644
--- a/restore-cassandra.groovy
+++ b/restore-cassandra.groovy
@@ -91,8 +91,11 @@
common.errorMsg('An error has been occurred during cassandra db startup on I@opencontrail:control and not I@cassandra:backup:client nodes: ' + err.getMessage())
throw err
}
- // another mantra, wait till all services are up
- sleep(60)
+ // wait till outstanding cassandra dbs are up
+ common.retry(6, 20){
+ common.infoMsg("Trying to connect to casandra db on I@opencontrail:control and not I@cassandra:backup:client nodes ...")
+ salt.cmdRun(pepperEnv, 'I@opencontrail:control and not I@cassandra:backup:client', "nc -v -z -w2 ${configDbIp} ${configDbPort}")
+ }
try {
common.infoMsg("Start analytics containers node")
salt.cmdRun(pepperEnv, 'I@opencontrail:collector', 'cd /etc/docker/compose/opencontrail/; docker-compose up -d')
diff --git a/stacklight-upgrade.groovy b/stacklight-upgrade.groovy
index 78765bb..c7a90db 100644
--- a/stacklight-upgrade.groovy
+++ b/stacklight-upgrade.groovy
@@ -70,10 +70,10 @@
common.errorMsg('[ERROR] Elasticsearch VIP port could not be retrieved')
}
- pillar = salt.getReturnValues(salt.getPillar(master, "I@elasticsearch:client", 'elasticsearch:client:server:scheme'))
+ pillar = salt.getPillar(master, "I@elasticsearch:client ${extra_tgt}", 'elasticsearch:client:server:scheme')
def elasticsearch_scheme
- if(pillar) {
- elasticsearch_scheme = pillar
+ if (!pillar['return'].isEmpty()) {
+ elasticsearch_scheme = pillar['return'][0].values()[0]
common.infoMsg("[INFO] Using elasticsearch scheme: ${elasticsearch_scheme}")
} else {
common.infoMsg('[INFO] No pillar with Elasticsearch server scheme, using scheme: http')
@@ -82,7 +82,7 @@
common.retry(retries,retries_wait) {
common.infoMsg('Waiting for Elasticsearch to become green..')
- salt.cmdRun(master, "I@elasticsearch:client", "curl -sf ${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
+ salt.cmdRun(master, "I@elasticsearch:client", "curl -sfk ${elasticsearch_scheme}://${elasticsearch_vip}:${elasticsearch_port}/_cat/health | awk '{print \$4}' | grep green")
}
} catch (Exception er) {
errorOccured = true
diff --git a/update-package.groovy b/update-package.groovy
index df7655b..14c2056 100644
--- a/update-package.groovy
+++ b/update-package.groovy
@@ -6,6 +6,7 @@
* SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* TARGET_PACKAGES Space delimited list of packages to be updates [package1=version package2=version], empty string means all updating all packages to the latest version.
+ * BATCH_SIZE Use batching for large amount of target nodes
*
**/
@@ -13,14 +14,19 @@
salt = new com.mirantis.mk.Salt()
common = new com.mirantis.mk.Common()
-def installSaltStack(target, pkgs, masterUpdate = false){
+def batch_size = ''
+if (common.validInputParam('BATCH_SIZE')) {
+ batch_size = "${BATCH_SIZE}"
+}
+
+def installSaltStack(target, pkgs, batch, masterUpdate = false){
salt.cmdRun(pepperEnv, "I@salt:master", "salt -C '${target}' --async pkg.install force_yes=True pkgs='$pkgs'")
def minions_reachable = target
if (masterUpdate) {
// in case of update Salt Master packages - check all minions are good
minions_reachable = '*'
}
- salt.checkTargetMinionsReady(['saltId': pepperEnv, 'target': target, 'target_reachable': minions_reachable])
+ salt.checkTargetMinionsReady(['saltId': pepperEnv, 'target': target, 'target_reachable': minions_reachable, 'batch': batch])
}
timeout(time: 12, unit: 'HOURS') {
@@ -46,12 +52,16 @@
stage("List package upgrades") {
common.infoMsg("Listing all the packages that have a new update available on nodes: ${targetLiveAll}")
- salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'pkg.list_upgrades', [], null, true)
+ salt.runSaltProcessStep(pepperEnv, targetLiveAll, 'pkg.list_upgrades', [], batch_size, true)
if (TARGET_PACKAGES != '' && TARGET_PACKAGES != '*') {
- common.warningMsg("Note that only the \"${TARGET_PACKAGES}\" would be installed from the above list of available updates on the ${targetLiveAll}")
+ if (ALLOW_DEPENDENCY_UPDATE.toBoolean()) {
+ common.warningMsg("Note that the \"${TARGET_PACKAGES}\" and it new dependencies would be installed from the above list of available updates on the ${targetLiveAll}")
+ } else {
+ common.warningMsg("Note that only the \"${TARGET_PACKAGES}\" would be installed from the above list of available updates on the ${targetLiveAll}")
+ commandKwargs = ['only_upgrade': 'true']
+ }
command = "pkg.install"
packages = TARGET_PACKAGES.tokenize(' ')
- commandKwargs = ['only_upgrade': 'true']
}
}
@@ -68,9 +78,9 @@
for (int i = 0; i < saltTargets.size(); i++ ) {
common.retry(10, 5) {
if (salt.getMinions(pepperEnv, "I@salt:master and ${saltTargets[i]}")) {
- installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', true)
+ installSaltStack("I@salt:master and ${saltTargets[i]}", '["salt-master", "salt-common", "salt-api", "salt-minion"]', null, true)
} else if (salt.getMinions(pepperEnv, "I@salt:minion and not I@salt:master and ${saltTargets[i]}")) {
- installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]')
+ installSaltStack("I@salt:minion and not I@salt:master and ${saltTargets[i]}", '["salt-minion"]', batch_size)
} else {
error("Minion ${saltTargets[i]} is not reachable!")
}
@@ -78,7 +88,7 @@
}
}
common.infoMsg('Starting package upgrades...')
- out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, null, packages, commandKwargs)
+ out = salt.runSaltCommand(pepperEnv, 'local', ['expression': targetLiveAll, 'type': 'compound'], command, batch_size, packages, commandKwargs)
salt.printSaltCommandResult(out)
for(value in out.get("return")[0].values()){
if (value.containsKey('result') && value.result == false) {
diff --git a/update-salt-environment.groovy b/update-salt-environment.groovy
index b91f385..2ae408f 100644
--- a/update-salt-environment.groovy
+++ b/update-salt-environment.groovy
@@ -27,7 +27,7 @@
'apt-get update && apt-get install -y salt-formula-*'
)
common.infoMsg("Running salt sync-all")
- salt.runSaltProcessStep(venvPepper, 'jma*', 'saltutil.sync_all', [], null, true)
+ salt.runSaltProcessStep(venvPepper, '*', 'saltutil.sync_all', [], null, true)
}
}
stage("Update Reclass") {
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index a809ba5..203c7af 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -8,6 +8,7 @@
* DRIVE_TRAIN_PARAMS Yaml, DriveTrain releated params:
* SALT_MASTER_URL Salt API server location
* SALT_MASTER_CREDENTIALS Credentials to the Salt API
+ * BATCH_SIZE Use batch sizing during upgrade for large envs
* UPGRADE_SALTSTACK Upgrade SaltStack packages to new version.
* UPDATE_CLUSTER_MODEL Update MCP version parameter in cluster model
* UPDATE_PIPELINES Update pipeline repositories on Gerrit
@@ -69,6 +70,14 @@
}
}
+def getWorkerThreads(saltId) {
+ if (env.getEnvironment().containsKey('SALT_MASTER_OPT_WORKER_THREADS')) {
+ return env['SALT_MASTER_OPT_WORKER_THREADS'].toString()
+ }
+ def threads = salt.cmdRun(saltId, "I@salt:master", "cat /etc/salt/master.d/master.conf | grep worker_threads | cut -f 2 -d ':'", true, null, true)
+ return threads['return'][0].values()[0].replaceAll('Salt command execution success','').trim()
+}
+
def wa29352(ArrayList saltMinions, String cname) {
// WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/12/openssh/client/root.yml
// Default soft-param has been removed, what now makes not possible to render some old env's.
@@ -82,7 +91,7 @@
salt.cmdRun(venvPepper, 'I@salt:master', "test ! -f ${wa29352File}", true, null, false)
}
catch (Exception ex) {
- common.infoMsg('Work-around for PROD-29352 already apply, nothing todo')
+ common.infoMsg('Work-around for PROD-29352 already applied, nothing todo')
return
}
def rKeysDict = [
@@ -111,6 +120,9 @@
"grep -q '${wa29352ClassName}' infra/secrets.yml || sed -i '/classes:/ a - $wa29352ClassName' infra/secrets.yml")
salt.fullRefresh(venvPepper, '*')
sh('rm -fv ' + _tempFile)
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cname && git status && " +
+ "git add ${wa29352File} && git add -u && git commit --allow-empty -m 'Cluster model updated with WA for PROD-29352. Issue cause due patch https://gerrit.mcp.mirantis.com/#/c/37932/ at ${common.getDatetime()}' ")
+ common.infoMsg('Work-around for PROD-29352 successfully applied')
}
def wa29155(ArrayList saltMinions, String cname) {
@@ -128,12 +140,9 @@
common.infoMsg('Work-around for PROD-29155 already apply, nothing todo')
return
}
- salt.fullRefresh(venvPepper, 'cfg*')
- salt.fullRefresh(venvPepper, 'cmp*')
+ salt.fullRefresh(venvPepper, 'I@salt:master')
+ salt.fullRefresh(venvPepper, 'I@nova:compute')
for (String minion in saltMinions) {
- if (!minion.startsWith('cmp')) {
- continue
- }
// First attempt, second will be performed in next validateReclassModel() stages
try {
salt.cmdRun(venvPepper, 'I@salt:master', "reclass -n ${minion}", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
@@ -208,6 +217,62 @@
}
}
+def wa32182(String cluster_name) {
+ if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
+ def clusterModelPath = "/srv/salt/reclass/classes/cluster/${cluster_name}"
+ def fixFile = "${clusterModelPath}/opencontrail/common_wa32182.yml"
+ def usualFile = "${clusterModelPath}/opencontrail/common.yml"
+ def fixFileContent = "classes:\n- system.opencontrail.common\n"
+ salt.cmdRun(venvPepper, 'I@salt:master', "test -f ${fixFile} -o -f ${usualFile} || echo '${fixFileContent}' > ${fixFile}")
+ def contrailFiles = ['opencontrail/analytics.yml', 'opencontrail/control.yml', 'openstack/compute/init.yml']
+ if (salt.testTarget(venvPepper, "I@kubernetes:master")) {
+ contrailFiles.add('kubernetes/compute.yml')
+ }
+ for(String contrailFile in contrailFiles) {
+ contrailFile = "${clusterModelPath}/${contrailFile}"
+ def containsFix = salt.cmdRun(venvPepper, 'I@salt:master', "grep -E '^- cluster\\.${cluster_name}\\.opencontrail\\.common(_wa32182)?\$' ${contrailFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ if (containsFix) {
+ continue
+ } else {
+ salt.cmdRun(venvPepper, 'I@salt:master', "grep -q -E '^parameters:' ${contrailFile} && sed -i '/^parameters:/i - cluster.${cluster_name}.opencontrail.common_wa32182' ${contrailFile} || " +
+ "echo '- cluster.${cluster_name}.opencontrail.common_wa32182' >> ${contrailFile}")
+ }
+ }
+ }
+}
+
+def wa33867(String cluster_name) {
+ if (salt.testTarget(venvPepper, 'I@opencontrail:control or I@opencontrail:collector')) {
+ def contrailControlFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/opencontrail/control.yml"
+ def line = salt.cmdRun(venvPepper, 'I@salt:master', "awk '/^- cluster.${cluster_name}.infra.backup.client_zookeeper/ {getline; print \$0}' ${contrailControlFile}", false, null, true).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
+ if (line == "- cluster.${cluster_name}.infra") {
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^- cluster.${cluster_name}.infra\$/d' ${contrailControlFile}")
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^- cluster.${cluster_name}.infra.backup.client_zookeeper\$/i - cluster.${cluster_name}.infra' ${contrailControlFile}")
+ }
+ }
+}
+
+def wa33771(String cluster_name) {
+ def octaviaEnabled = salt.getMinions(venvPepper, 'I@octavia:api:enabled')
+ def octaviaWSGI = salt.getMinions(venvPepper, 'I@apache:server:site:octavia_api')
+ if (octaviaEnabled && ! octaviaWSGI) {
+ def openstackControl = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/control.yml"
+ def octaviaFile = "/srv/salt/reclass/classes/cluster/${cluster_name}/openstack/octavia_wa33771.yml"
+ def octaviaContext = [
+ 'classes': [ 'system.apache.server.site.octavia' ],
+ 'parameters': [
+ '_param': [ 'apache_octavia_api_address' : '${_param:cluster_local_address}' ],
+ 'apache': [ 'server': [ 'site': [ 'apache_proxy_openstack_api_octavia': [ 'enabled': false ] ] ] ]
+ ]
+ ]
+ def _tempFile = '/tmp/wa33771' + UUID.randomUUID().toString().take(8)
+ writeYaml file: _tempFile , data: octaviaContext
+ def octaviaFileContent = sh(script: "cat ${_tempFile} | base64", returnStdout: true).trim()
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i '/^parameters:/i - cluster.${cluster_name}.openstack.octavia_wa33771' ${openstackControl}")
+ salt.cmdRun(venvPepper, 'I@salt:master', "echo '${octaviaFileContent}' | base64 -d > ${octaviaFile}", false, null, false)
+ }
+}
+
def archiveReclassInventory(filename) {
def _tmp_file = '/tmp/' + filename + UUID.randomUUID().toString().take(8)
// jenkins may fail at overheap. Compress data with gzip like WA
@@ -302,19 +367,21 @@
def updateLocalRepos = ''
def reclassSystemBranch = ''
def reclassSystemBranchDefault = gitTargetMcpVersion
+ def batchSize = ''
if (gitTargetMcpVersion != 'proposed') {
reclassSystemBranchDefault = "origin/${gitTargetMcpVersion}"
}
- def driteTrainParamsYaml = env.getProperty('DRIVE_TRAIN_PARAMS')
- if (driteTrainParamsYaml) {
- def driteTrainParams = readYaml text: driteTrainParamsYaml
- saltMastURL = driteTrainParams.get('SALT_MASTER_URL')
- saltMastCreds = driteTrainParams.get('SALT_MASTER_CREDENTIALS')
- upgradeSaltStack = driteTrainParams.get('UPGRADE_SALTSTACK', false).toBoolean()
- updateClusterModel = driteTrainParams.get('UPDATE_CLUSTER_MODEL', false).toBoolean()
- updatePipelines = driteTrainParams.get('UPDATE_PIPELINES', false).toBoolean()
- updateLocalRepos = driteTrainParams.get('UPDATE_LOCAL_REPOS', false).toBoolean()
- reclassSystemBranch = driteTrainParams.get('RECLASS_SYSTEM_BRANCH', reclassSystemBranchDefault)
+ def driveTrainParamsYaml = env.getProperty('DRIVE_TRAIN_PARAMS')
+ if (driveTrainParamsYaml) {
+ def driveTrainParams = readYaml text: driveTrainParamsYaml
+ saltMastURL = driveTrainParams.get('SALT_MASTER_URL')
+ saltMastCreds = driveTrainParams.get('SALT_MASTER_CREDENTIALS')
+ upgradeSaltStack = driveTrainParams.get('UPGRADE_SALTSTACK', false).toBoolean()
+ updateClusterModel = driveTrainParams.get('UPDATE_CLUSTER_MODEL', false).toBoolean()
+ updatePipelines = driveTrainParams.get('UPDATE_PIPELINES', false).toBoolean()
+ updateLocalRepos = driveTrainParams.get('UPDATE_LOCAL_REPOS', false).toBoolean()
+ reclassSystemBranch = driveTrainParams.get('RECLASS_SYSTEM_BRANCH', reclassSystemBranchDefault)
+ batchSize = driveTrainParams.get('BATCH_SIZE', '')
} else {
// backward compatibility for 2018.11.0
saltMastURL = env.getProperty('SALT_MASTER_URL')
@@ -331,6 +398,9 @@
if (cluster_name == '' || cluster_name == 'null' || cluster_name == null) {
error('Pillar data is broken for Salt master node! Please check it manually and re-run pipeline.')
}
+ if (!batchSize) {
+ batchSize = getWorkerThreads(venvPepper)
+ }
stage('Update Reclass and Salt-Formulas') {
common.infoMsg('Perform: Full salt sync')
@@ -371,6 +441,9 @@
salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly -l 'system.linux.system.repo.mcp.extra' * | xargs --no-run-if-empty sed -i 's/system.linux.system.repo.mcp.extra/system.linux.system.repo.mcp.apt_mirantis.extra/g'")
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/linux_system_repo_mcp_maas_url/d' maas.yml")
+ salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name/infra && sed -i '/maas_region_main_archive/d' maas.yml")
+
// Switch Jenkins/Gerrit to use LDAP SSL/TLS
def gerritldapURI = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && " +
"grep -r --exclude-dir=aptly 'gerrit_ldap_server: .*' * | grep -Po 'gerrit_ldap_server: \\K.*' | tr -d '\"'", true, null, false).get('return')[0].values()[0].replaceAll('Salt command execution success', '').trim()
@@ -407,7 +480,7 @@
}
// Add all update repositories
def repoIncludeBase = '- system.linux.system.repo.mcp.apt_mirantis.'
- def updateRepoList = ['cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'percona', 'salt-formulas', 'saltstack', 'ubuntu']
+ def updateRepoList = ['cassandra', 'ceph', 'contrail', 'docker', 'elastic', 'extra', 'openstack', 'maas', 'percona', 'salt-formulas', 'saltstack', 'ubuntu']
updateRepoList.each { repo ->
def repoNameUpdateInclude = "${repoIncludeBase}update.${repo}"
def filesWithInclude = salt.cmdRun(venvPepper, 'I@salt:master', "cd /srv/salt/reclass/classes/cluster/$cluster_name && grep -Plr '\\${repoIncludeBase}${repo}\$' . || true", false).get('return')[0].values()[0].trim().tokenize('\n')
@@ -421,6 +494,9 @@
}
}
}
+ wa32182(cluster_name)
+ wa33771(cluster_name)
+ wa33867(cluster_name)
// Add new defaults
common.infoMsg("Add new defaults")
salt.cmdRun(venvPepper, 'I@salt:master', "grep '^ mcp_version: ' /srv/salt/reclass/classes/cluster/$cluster_name/infra/init.yml || " +
@@ -451,7 +527,8 @@
}
wa29352(minions, cluster_name)
- wa29155(minions, cluster_name)
+ def computeMinions = salt.getMinions(venvPepper, 'I@nova:compute')
+ wa29155(computeMinions, cluster_name)
try {
common.infoMsg('Perform: UPDATE Reclass package')
@@ -535,7 +612,7 @@
if (upgradeSaltStack) {
updateSaltStack('I@salt:master', '["salt-master", "salt-common", "salt-api", "salt-minion"]')
- salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.repo', true, true, null, false, 60, 2)
+ salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.repo', true, true, batchSize, false, 60, 2)
updateSaltStack('I@salt:minion and not I@salt:master', '["salt-minion"]')
}
@@ -548,16 +625,16 @@
// update minions certs
// call for `salt.minion.ca` state on related nodes to make sure
// mine was updated with required data after salt-minion/salt-master restart salt:minion:ca
- salt.enforceState(venvPepper, 'I@salt:minion:ca', 'salt.minion.ca', true, true, null, false, 60, 2)
- salt.enforceState(venvPepper, 'I@salt:minion', 'salt.minion.cert', true, true, null, false, 60, 2)
+ salt.enforceState(venvPepper, 'I@salt:minion:ca', 'salt.minion.ca', true, true, batchSize, false, 60, 2)
+ salt.enforceState(venvPepper, 'I@salt:minion', 'salt.minion.cert', true, true, batchSize, false, 60, 2)
// run `salt.minion` to refresh all minion configs (for example _keystone.conf)
salt.enforceState(venvPepper, 'I@salt:minion', 'salt.minion', true, true, null, false, 60, 2)
// Retry needed only for rare race-condition in user appearance
common.infoMsg('Perform: updating users and keys')
- salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.user', true, true, null, false, 60, 2)
+ salt.enforceState(venvPepper, 'I@linux:system', 'linux.system.user', true, true, batchSize, false, 60, 2)
common.infoMsg('Perform: updating openssh')
- salt.enforceState(venvPepper, 'I@linux:system', 'openssh', true, true, null, false, 60, 2)
+ salt.enforceState(venvPepper, 'I@linux:system', 'openssh', true, true, batchSize, false, 60, 2)
// apply salt API TLS if needed
def nginxAtMaster = salt.getPillar(venvPepper, 'I@salt:master', 'nginx:server:enabled').get('return')[0].values()[0]