Merge "Cloud upgrade pipeline - add option to run CVP tests."
diff --git a/cloud-deploy-pipeline.groovy b/cloud-deploy-pipeline.groovy
index 7f7337a..a7f35d5 100644
--- a/cloud-deploy-pipeline.groovy
+++ b/cloud-deploy-pipeline.groovy
@@ -43,6 +43,10 @@
* which have to be added during bootstrap.
* Format: repo 1, repo priority 1, repo pin 1; repo 2, repo priority 2, repo pin 2;
+ * SALT_VERSION Version of Salt which is going to be installed i.e. 'stable 2016.3' or 'stable 2017.7' etc.
+ *
+ * EXTRA_TARGET The value will be added to target nodes
+ *
* Test settings:
* TEST_K8S_API_SERVER Kubernetes API address
* TEST_K8S_CONFORMANCE_IMAGE Path to docker image with conformance e2e tests
@@ -95,6 +99,12 @@
if (common.validInputParam('SLAVE_NODE')) {
slave_node = SLAVE_NODE
}
+
+def extra_tgt = ''
+if (common.validInputParam('EXTRA_TARGET')) {
+ extra_tgt = "${EXTRA_TARGET}"
+}
+
timeout(time: 12, unit: 'HOURS') {
node(slave_node) {
try {
@@ -208,6 +218,12 @@
}
}
+
+ if (common.validInputParam('SALT_VERSION')) {
+ common.infoMsg("Setting salt version to ${SALT_VERSION}")
+ envParams.put('cfg_saltversion', SALT_VERSION)
+ }
+
openstack.createHeatStack(openstackCloud, STACK_NAME, STACK_TEMPLATE, envParams, HEAT_STACK_ENVIRONMENT, venv)
}
@@ -295,7 +311,7 @@
// Set up override params
if (common.validInputParam('SALT_OVERRIDES')) {
stage('Set Salt overrides') {
- salt.setSaltOverrides(venvPepper, SALT_OVERRIDES)
+ salt.setSaltOverrides(venvPepper, SALT_OVERRIDES, '/srv/salt/reclass', extra_tgt)
}
}
@@ -309,14 +325,14 @@
if (common.validInputParam('STATIC_MGMT_NETWORK')) {
staticMgmtNetwork = STATIC_MGMT_NETWORK.toBoolean()
}
- orchestrate.installFoundationInfra(venvPepper, staticMgmtNetwork)
+ orchestrate.installFoundationInfra(venvPepper, staticMgmtNetwork, extra_tgt)
if (common.checkContains('STACK_INSTALL', 'kvm')) {
- orchestrate.installInfraKvm(venvPepper)
- orchestrate.installFoundationInfra(venvPepper, staticMgmtNetwork)
+ orchestrate.installInfraKvm(venvPepper, extra_tgt)
+ orchestrate.installFoundationInfra(venvPepper, staticMgmtNetwork, extra_target)
}
- orchestrate.validateFoundationInfra(venvPepper)
+ orchestrate.validateFoundationInfra(venvPepper, extra_tgt)
}
}
@@ -329,35 +345,35 @@
def awsOutputs = aws.getOutputs(venv, aws_env_vars, STACK_NAME)
common.prettyPrint(awsOutputs)
if (awsOutputs.containsKey('ControlLoadBalancer')) {
- salt.runSaltProcessStep(venvPepper, 'I@salt:master', 'reclass.cluster_meta_set', ['kubernetes_control_address', awsOutputs['ControlLoadBalancer']], null, true)
+ salt.runSaltProcessStep(venvPepper, "I@salt:master ${extra_tgt}", 'reclass.cluster_meta_set', ['kubernetes_control_address', awsOutputs['ControlLoadBalancer']], null, true)
outputs.put('kubernetes_apiserver', 'https://' + awsOutputs['ControlLoadBalancer'])
}
}
// ensure certificates are generated properly
- salt.runSaltProcessStep(venvPepper, '*', 'saltutil.refresh_pillar', [], null, true)
- salt.enforceState(venvPepper, '*', ['salt.minion.cert'], true)
+ salt.runSaltProcessStep(venvPepper, "* ${extra_tgt}", 'saltutil.refresh_pillar', [], null, true)
+ salt.enforceState(venvPepper, "* ${extra_tgt}", ['salt.minion.cert'], true)
- orchestrate.installKubernetesInfra(venvPepper)
+ orchestrate.installKubernetesInfra(venvPepper, extra_tgt)
}
if (common.checkContains('STACK_INSTALL', 'contrail')) {
stage('Install Contrail control') {
- orchestrate.installContrailNetwork(venvPepper)
+ orchestrate.installContrailNetwork(venvPepper, extra_tgt)
}
}
stage('Install Kubernetes control') {
- orchestrate.installKubernetesControl(venvPepper)
+ orchestrate.installKubernetesControl(venvPepper, extra_tgt)
// collect artifacts (kubeconfig)
- writeFile(file: 'kubeconfig', text: salt.getFileContent(venvPepper, 'I@kubernetes:master and *01*', '/etc/kubernetes/admin-kube-config'))
+ writeFile(file: 'kubeconfig', text: salt.getFileContent(venvPepper, "I@kubernetes:master and *01* ${extra_tgt}", '/etc/kubernetes/admin-kube-config'))
archiveArtifacts(artifacts: 'kubeconfig')
}
if (common.checkContains('STACK_INSTALL', 'contrail')) {
stage('Install Contrail compute') {
- orchestrate.installContrailCompute(venvPepper)
+ orchestrate.installContrailCompute(venvPepper, extra_tgt)
}
}
@@ -385,7 +401,7 @@
}
}
- orchestrate.installKubernetesCompute(venvPepper)
+ orchestrate.installKubernetesCompute(venvPepper, extra_tgt)
}
}
@@ -394,49 +410,49 @@
// install Infra and control, tests, ...
stage('Install OpenStack infra') {
- orchestrate.installOpenstackInfra(venvPepper)
+ orchestrate.installOpenstackInfra(venvPepper, extra_tgt)
}
stage('Install OpenStack control') {
- orchestrate.installOpenstackControl(venvPepper)
+ orchestrate.installOpenstackControl(venvPepper, extra_tgt)
}
// Workaround for PROD-17765 issue to prevent crashes of keystone.role_present state.
// More details: https://mirantis.jira.com/browse/PROD-17765
- salt.runSaltProcessStep(venvPepper, 'I@keystone:client', 'service.restart', ['salt-minion'])
- salt.minionsReachable(venvPepper, 'I@salt:master and *01*', 'I@keystone:client', null, 10, 6)
+ salt.runSaltProcessStep(venvPepper, "I@keystone:client ${extra_tgt}", 'service.restart', ['salt-minion'])
+ salt.minionsReachable(venvPepper, "I@salt:master and *01* ${extra_tgt}", 'I@keystone:client', null, 10, 6)
stage('Install OpenStack network') {
if (common.checkContains('STACK_INSTALL', 'contrail')) {
- orchestrate.installContrailNetwork(venvPepper)
+ orchestrate.installContrailNetwork(venvPepper, extra_tgt)
} else if (common.checkContains('STACK_INSTALL', 'ovs')) {
- orchestrate.installOpenstackNetwork(venvPepper)
+ orchestrate.installOpenstackNetwork(venvPepper, extra_tgt)
}
// Wait for network to come up, 150s should be enough
common.retry(10, 15) {
- salt.cmdRun(venvPepper, 'I@keystone:server', '. /root/keystonercv3; openstack network list')
+ salt.cmdRun(venvPepper, "I@keystone:server ${extra_tgt}", '. /root/keystonercv3; openstack network list')
}
}
- if (salt.testTarget(venvPepper, 'I@ironic:conductor')){
+ if (salt.testTarget(venvPepper, "I@ironic:conductor ${extra_tgt}")){
stage('Install OpenStack Ironic conductor') {
- orchestrate.installIronicConductor(venvPepper)
+ orchestrate.installIronicConductor(venvPepper, extra_tgt)
}
}
- if (salt.testTarget(venvPepper, 'I@manila:share')){
+ if (salt.testTarget(venvPepper, "I@manila:share ${extra_tgt}")){
stage('Install OpenStack Manila data and share') {
- orchestrate.installManilaShare(venvPepper)
+ orchestrate.installManilaShare(venvPepper, extra_tgt)
}
}
stage('Install OpenStack compute') {
- orchestrate.installOpenstackCompute(venvPepper)
+ orchestrate.installOpenstackCompute(venvPepper, extra_tgt)
if (common.checkContains('STACK_INSTALL', 'contrail')) {
- orchestrate.installContrailCompute(venvPepper)
+ orchestrate.installContrailCompute(venvPepper, extra_tgt)
}
}
@@ -445,48 +461,48 @@
// install ceph
if (common.checkContains('STACK_INSTALL', 'ceph')) {
stage('Install Ceph MONs') {
- orchestrate.installCephMon(venvPepper)
+ orchestrate.installCephMon(venvPepper, "I@ceph:mon ${extra_tgt}", extra_tgt)
}
stage('Install Ceph OSDs') {
- orchestrate.installCephOsd(venvPepper)
+ orchestrate.installCephOsd(venvPepper, "I@ceph:osd ${extra_tgt}", true, extra_tgt)
}
stage('Install Ceph clients') {
- orchestrate.installCephClient(venvPepper)
+ orchestrate.installCephClient(venvPepper, extra_tgt)
}
stage('Connect Ceph') {
- orchestrate.connectCeph(venvPepper)
+ orchestrate.connectCeph(venvPepper, extra_tgt)
}
}
if (common.checkContains('STACK_INSTALL', 'oss')) {
stage('Install Oss infra') {
- orchestrate.installOssInfra(venvPepper)
+ orchestrate.installOssInfra(venvPepper, extra_tgt)
}
}
if (common.checkContains('STACK_INSTALL', 'cicd')) {
stage('Install Cicd') {
- orchestrate.installInfra(venvPepper)
- orchestrate.installDockerSwarm(venvPepper)
- orchestrate.installCicd(venvPepper)
+ orchestrate.installInfra(venvPepper, extra_tgt)
+ orchestrate.installDockerSwarm(venvPepper, extra_tgt)
+ orchestrate.installCicd(venvPepper, extra_tgt)
}
}
if (common.checkContains('STACK_INSTALL', 'sl-legacy')) {
stage('Install StackLight v1') {
- orchestrate.installStacklightv1Control(venvPepper)
- orchestrate.installStacklightv1Client(venvPepper)
+ orchestrate.installStacklightv1Control(venvPepper, extra_tgt)
+ orchestrate.installStacklightv1Client(venvPepper, extra_tgt)
}
}
if (common.checkContains('STACK_INSTALL', 'stacklight')) {
stage('Install StackLight') {
- orchestrate.installDockerSwarm(venvPepper)
- orchestrate.installStacklight(venvPepper)
+ orchestrate.installDockerSwarm(venvPepper, extra_tgt)
+ orchestrate.installStacklight(venvPepper, extra_tgt)
}
}
@@ -494,10 +510,10 @@
stage('Install OSS') {
if (!common.checkContains('STACK_INSTALL', 'stacklight')) {
// In case if StackLightv2 enabled containers already started
- orchestrate.installDockerSwarm(venvPepper)
- salt.enforceState(venvPepper, 'I@docker:swarm:role:master and I@devops_portal:config', 'docker.client', true)
+ orchestrate.installDockerSwarm(venvPepper, extra_tgt)
+ salt.enforceState(venvPepper, "I@docker:swarm:role:master and I@devops_portal:config ${extra_tgt}", 'docker.client', true)
}
- orchestrate.installOss(venvPepper)
+ orchestrate.installOss(venvPepper, extra_tgt)
}
}
@@ -510,7 +526,7 @@
stage('Run k8s conformance e2e tests') {
def image = TEST_K8S_CONFORMANCE_IMAGE
def output_file = image.replaceAll('/', '-') + '.output'
- def target = 'ctl01*'
+ def target = "ctl01* ${extra_tgt}"
def conformance_output_file = 'conformance_test.tar'
// run image
@@ -594,8 +610,8 @@
stage('Finalize') {
if (common.checkContains('STACK_INSTALL', 'finalize')) {
- def gluster_compound = 'I@glusterfs:server'
- def salt_ca_compound = 'I@salt:minion:ca:salt_master_ca'
+ def gluster_compound = "I@glusterfs:server ${extra_tgt}"
+ def salt_ca_compound = "I@salt:minion:ca:salt_master_ca ${extra_tgt}"
// Enforce highstate asynchronous only on the nodes which are not glusterfs servers
salt.enforceHighstate(venvPepper, '* and not ' + gluster_compound + ' and not ' + salt_ca_compound)
// Iterate over nonempty set of gluster servers and apply highstates one by one
diff --git a/cloud-update.groovy b/cloud-update.groovy
index 151b653..19d563f 100644
--- a/cloud-update.groovy
+++ b/cloud-update.groovy
@@ -708,13 +708,26 @@
def verifyAPIs(pepperEnv, target) {
def salt = new com.mirantis.mk.Salt()
def common = new com.mirantis.mk.Common()
- def out = salt.cmdRun(pepperEnv, target, '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
- if (out.toString().toLowerCase().contains('error')) {
- common.errorMsg(out)
- if (INTERACTIVE.toBoolean()) {
- input message: "APIs are not working as expected. Please fix it manually."
- } else {
- throw new Exception("APIs are not working as expected")
+ def cmds = ["openstack service list",
+ "openstack image list",
+ "openstack flavor list",
+ "openstack compute service list",
+ "openstack server list",
+ "openstack network list",
+ "openstack volume list",
+ "openstack orchestration service list"]
+ def sourcerc = ". /root/keystonercv3;"
+ def cmdOut = ">/dev/null 2>&1;echo \$?"
+ for (c in cmds) {
+ def command = sourcerc + c + cmdOut
+ def out = salt.cmdRun(pepperEnv, target, "${command}")
+ if (!out.toString().toLowerCase().contains('0')) {
+ common.errorMsg(out)
+ if (INTERACTIVE.toBoolean()) {
+ input message: "APIs are not working as expected. Please fix it manually."
+ } else {
+ throw new Exception("APIs are not working as expected")
+ }
}
}
}
@@ -1530,7 +1543,7 @@
if (merges.contains("log")) {
if (salt.testTarget(pepperEnv, LOG_TARGET)) {
- mergeSnapshot(pepperEnv, LOG_TARGET. 'log')
+ mergeSnapshot(pepperEnv, LOG_TARGET, 'log')
}
}
diff --git a/cvp-func.groovy b/cvp-func.groovy
index ef50945..d1fff1a 100644
--- a/cvp-func.groovy
+++ b/cvp-func.groovy
@@ -33,6 +33,7 @@
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
validate.runBasicContainer(saltMaster, TARGET_NODE, TEST_IMAGE)
sh "rm -rf ${artifacts_dir}"
+ salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
validate.configureContainer(saltMaster, TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO, TEMPEST_ENDPOINT_TYPE)
}
diff --git a/cvp-ha.groovy b/cvp-ha.groovy
index 99a49d3..ab5b5d4 100644
--- a/cvp-ha.groovy
+++ b/cvp-ha.groovy
@@ -39,6 +39,7 @@
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
validate.runBasicContainer(saltMaster, TEMPEST_TARGET_NODE, TEST_IMAGE)
sh "rm -rf ${artifacts_dir}"
+ salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
validate.configureContainer(saltMaster, TEMPEST_TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO)
}
diff --git a/cvp-perf.groovy b/cvp-perf.groovy
index d0c7156..fe86197 100644
--- a/cvp-perf.groovy
+++ b/cvp-perf.groovy
@@ -28,6 +28,7 @@
stage('Initialization') {
saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
sh "rm -rf ${artifacts_dir}"
+ salt.cmdRun(saltMaster, TARGET_NODE, "rm -rf ${remote_artifacts_dir}")
salt.cmdRun(saltMaster, TARGET_NODE, "mkdir -p ${remote_artifacts_dir}")
validate.runBasicContainer(saltMaster, TARGET_NODE, TEST_IMAGE)
validate.configureContainer(saltMaster, TARGET_NODE, PROXY, TOOLS_REPO, "")
diff --git a/cvp-spt.groovy b/cvp-spt.groovy
new file mode 100644
index 0000000..ea4680f
--- /dev/null
+++ b/cvp-spt.groovy
@@ -0,0 +1,63 @@
+/**
+ *
+ * Launch pytest frameworks in Jenkins
+ *
+ * Expected parameters:
+ * SALT_MASTER_URL URL of Salt master
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API
+ *
+ * TESTS_SET Leave empty for full run or choose a file (test)
+ * TESTS_REPO Repo to clone
+ * TESTS_SETTINGS Additional environment varibales to apply
+ * PROXY Proxy to use for cloning repo or for pip
+ *
+ */
+
+validate = new com.mirantis.mcp.Validate()
+
+node() {
+ try{
+ stage('Initialization') {
+ validate.prepareVenv(TESTS_REPO, PROXY)
+ }
+
+ stage('Run Tests') {
+ validate.runTests(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS, TESTS_SET, '', TESTS_SETTINGS)
+ }
+ stage ('Publish results') {
+ archiveArtifacts artifacts: "*"
+ junit "*.xml"
+ plot csvFileName: 'plot-8634d2fe-dc48-4713-99f9-b69a381483aa.csv',
+ group: 'SPT',
+ style: 'line',
+ title: 'SPT Glance results',
+ xmlSeries: [[
+ file: "report.xml",
+ nodeType: 'NODESET',
+ url: '',
+ xpath: '/testsuite/testcase[@name="test_speed_glance"]/properties/property']]
+ plot csvFileName: 'plot-8634d2fe-dc48-4713-99f9-b69a381483bb.csv',
+ group: 'SPT',
+ style: 'line',
+ title: 'SPT HW2HW results',
+ xmlSeries: [[
+ file: "report.xml",
+ nodeType: 'NODESET',
+ url: '',
+ xpath: '/testsuite/testcase[@classname="cvp-spt.cvp_spt.tests.test_hw2hw"]/properties/property']]
+ plot csvFileName: 'plot-8634d2fe-dc48-4713-99f9-b69a381483bc.csv',
+ group: 'SPT',
+ style: 'line',
+ title: 'SPT VM2VM results',
+ xmlSeries: [[
+ file: "report.xml",
+ nodeType: 'NODESET',
+ url: '',
+ xpath: '/testsuite/testcase[@classname="cvp-spt.cvp_spt.tests.test_vm2vm"]/properties/property']]
+ }
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ throw e
+ }
+}
diff --git a/deploy-virtual-edge-mom.groovy b/deploy-virtual-edge-mom.groovy
new file mode 100644
index 0000000..af0d8a7
--- /dev/null
+++ b/deploy-virtual-edge-mom.groovy
@@ -0,0 +1,211 @@
+/**
+ *
+ * Deploy Master of Masters and edge clouds
+ *
+ * Expected parameters:
+ * FORMULA_PKG_REVISION Formula revision
+ * STACK_CLUSTER_NAME The name of cluster model to use
+ * STACK_RECLASS_ADDRESS Reclass repository
+ * STACK_RECLASS_BRANCH Reclass barnch
+ * OPENSTACK_API_PROJECT OpenStack project to connect to
+ * HEAT_STACK_ZONE Heat stack availability zone
+ * STACK_TEMPLATE_URL URL to git repo with stack templates
+ * STACK_TEMPLATE_BRANCH Stack templates repo branch
+ * STACK_TEMPLATE File with stack template
+ * STACK_TEST Run tests (bool)
+ * EDGE_DEPLOY_SCHEMAS Env schemas to deploy as edge clouds
+ * MOM_JOB Type of Master-of-Masters stack
+ */
+
+common = new com.mirantis.mk.Common()
+salt = new com.mirantis.mk.Salt()
+
+def slave_node = 'python'
+
+if (common.validInputParam('SLAVE_NODE')) {
+ slave_node = SLAVE_NODE
+}
+
+def deployMoMJob = 'deploy-heat-virtual_mcp11_aio'
+if (common.validInputParam('MOM_JOB')) {
+ deployMoMJob = MOM_JOB
+}
+
+def deploy_schemas = '{os_ha_ovs: {deploy_job_name: "deploy-heat-os_ha_ovs", properties: {SLAVE_NODE: "python", STACK_INSTALL: "openstack,ovs", STACK_TEMPLATE: "os_ha_ovs", STACK_TYPE: "heat", FORMULA_PKG_REVISION: "testing", STACK_DELETE: false, STACK_CLUSTER_NAME: "os-ha-ovs"}}}'
+if (common.validInputParam('EDGE_DEPLOY_SCHEMAS')) {
+ deploy_schemas = EDGE_DEPLOY_SCHEMAS
+}
+
+def salt_overrides_list = SALT_OVERRIDES.tokenize('\n')
+
+def enableSyndic(saltMasterURL, nodeName, saltMasterCred, saltMoMIP) {
+
+ def saltMaster
+ saltMaster = salt.connection(saltMasterURL, saltMasterCred)
+
+ // Set up test_target parameter on node level
+ def fullnodename = salt.getMinions(saltMaster, nodeName).get(0)
+ def saltMasterExpression = 'I@salt:master'
+ def saltMasterTarget = ['expression': saltMasterExpression, 'type': 'compound']
+ def result
+ def classes_to_add = ['system.salt.syndic.single']
+ def params_to_add = ['salt_syndic_master_address': saltMoMIP]
+
+ result = salt.runSaltCommand(saltMaster, 'local', saltMasterTarget, 'reclass.node_update', null, null, ['name': "${fullnodename}", 'classes': classes_to_add, 'parameters': params_to_add])
+ salt.checkResult(result)
+
+ common.infoMsg('Perform full refresh for all nodes')
+ salt.fullRefresh(saltMaster, '*')
+
+ if (salt.testTarget(saltMaster, 'I@salt:syndic:enabled:True')) {
+ salt.enforceState(saltMaster, 'I@salt:syndic:enabled', 'salt.syndic', true, true, null, false, 180, 2)
+ }
+
+
+}
+
+node(slave_node) {
+
+ def momBuild
+ def salt_mom_url
+ def salt_mom_ip
+ def deploy_edges_infra = [:]
+ def deploy_edges = [:]
+ def edgeBuildsInfra = [:]
+ def edgeBuilds = [:]
+ def edge_deploy_schemas = readJSON text: deploy_schemas
+
+ stage('Deploy MoM stack'){
+ momBuild = build job: deployMoMJob, propagate: true, parameters: [
+ [$class: 'StringParameterValue', name: 'FORMULA_PKG_REVISION', value: FORMULA_PKG_REVISION],
+ [$class: 'StringParameterValue', name: 'STACK_CLUSTER_NAME', value: STACK_CLUSTER_NAME],
+ [$class: 'StringParameterValue', name: 'STACK_INSTALL', value: 'core'],
+ [$class: 'BooleanParameterValue', name: 'STACK_DELETE', value: STACK_DELETE.toBoolean()],
+ [$class: 'StringParameterValue', name: 'STACK_RECLASS_ADDRESS', value: STACK_RECLASS_ADDRESS],
+ [$class: 'StringParameterValue', name: 'STACK_RECLASS_BRANCH', value: STACK_RECLASS_BRANCH],
+ [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
+ [$class: 'StringParameterValue', name: 'HEAT_STACK_ZONE', value: HEAT_STACK_ZONE],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE_URL', value: STACK_TEMPLATE_URL],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE_BRANCH', value: STACK_TEMPLATE_BRANCH],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE', value: STACK_TEMPLATE],
+ [$class: 'StringParameterValue', name: 'STACK_TEST', value: STACK_TEST],
+ [$class: 'BooleanParameterValue', name: 'TEST_DOCKER_INSTALL', value: false],
+ [$class: 'StringParameterValue', name: 'SLAVE_NODE', value: slave_node],
+ ]
+
+ if (momBuild.result == 'SUCCESS') {
+ // get salt master url
+ salt_mom_url = "http://${momBuild.description.tokenize(' ')[1]}:6969"
+ salt_mom_ip = "${momBuild.description.tokenize(' ')[1]}"
+ node_name = "${momBuild.description.tokenize(' ')[2]}"
+ salt_overrides_list.add("salt_syndic_master_address: ${momBuild.description.tokenize(' ')[1]}")
+ common.infoMsg("Salt API is accessible via ${salt_mom_url}")
+ common.infoMsg("Enabling salt_syndic_enabled through overrides")
+ salt_overrides_list.add("salt_syndic_enabled: true")
+ } else {
+ common.errorMsg("Deployment of MoM has failed with result: " + momBuild.result)
+
+ }
+
+ }
+
+ stage('Deploy edge clouds'){
+ salt_overrides_list.add("salt_syndic_enabled: true")
+
+ for (edge_deploy_schema in edge_deploy_schemas.keySet()) {
+ def props
+ def deploy_job
+ def stack_name
+ def ed = edge_deploy_schema
+
+ deploy_job = edge_deploy_schemas[edge_deploy_schema]['deploy_job_name']
+
+ common.infoMsg("Edge cloud: ${edge_deploy_schema}")
+ common.infoMsg("Deploy job name: ${edge_deploy_schemas[edge_deploy_schema]['deploy_job_name']}")
+
+ props = edge_deploy_schemas[edge_deploy_schema]['properties']
+
+ if (env.BUILD_USER_ID) {
+ stack_name = "${env.BUILD_USER_ID}-${edge_deploy_schema}-${BUILD_NUMBER}"
+ } else {
+ stack_name = "replayed-${edge_deploy_schema}-${BUILD_NUMBER}"
+ }
+ deploy_edges_infra["Deploy ${ed} infra"] = {
+ node(slave_node) {
+ edgeBuildsInfra["${ed}"] = build job: deploy_job, propagate: false, parameters: [
+ [$class: 'StringParameterValue', name: 'HEAT_STACK_ZONE', value: HEAT_STACK_ZONE],
+ [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
+ [$class: 'StringParameterValue', name: 'SLAVE_NODE', value: props['SLAVE_NODE']],
+ [$class: 'StringParameterValue', name: 'STACK_INSTALL', value: 'core'],
+ [$class: 'StringParameterValue', name: 'STACK_NAME', value: stack_name],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE', value: props['STACK_TEMPLATE']],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE_URL', value: STACK_TEMPLATE_URL],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE_BRANCH', value: 'master'],
+ [$class: 'StringParameterValue', name: 'STACK_TYPE', value: 'heat'],
+ [$class: 'StringParameterValue', name: 'FORMULA_PKG_REVISION', value: props['FORMULA_PKG_REVISION']],
+ [$class: 'StringParameterValue', name: 'STACK_CLUSTER_NAME', value: props['STACK_CLUSTER_NAME']],
+ [$class: 'StringParameterValue', name: 'STACK_TEST', value: ''],
+ [$class: 'StringParameterValue', name: 'SALT_VERSION', value: 'stable 2017.7'],
+ [$class: 'BooleanParameterValue', name: 'TEST_DOCKER_INSTALL', value: false],
+ [$class: 'TextParameterValue', name: 'SALT_OVERRIDES', value: salt_overrides_list.join('\n')],
+ [$class: 'BooleanParameterValue', name: 'STACK_DELETE', value: props['STACK_DELETE'].toBoolean()],
+ ]
+ }
+ }
+ }
+
+ parallel deploy_edges_infra
+
+ for (k in edgeBuildsInfra.keySet()) {
+ common.infoMsg("keyset1: ${[k]}")
+ def ed_ = k
+ def deploy_job
+ def props_
+ def extra_target
+ def saltMasterURL
+
+ if (edgeBuildsInfra[ed_].result == 'SUCCESS') {
+ extra_target = "and *${edgeBuildsInfra[ed_].description.tokenize(' ')[0]}*"
+ saltMasterURL = "http://${edgeBuildsInfra[ed_].description.tokenize(' ')[1]}:6969"
+
+
+ enableSyndic(saltMasterURL, 'cfg01*', SALT_MASTER_CREDENTIALS, salt_mom_ip)
+
+ props_ = edge_deploy_schemas[ed_]['properties']
+ deploy_job = edge_deploy_schemas[ed_]['deploy_job_name']
+
+
+ deploy_edges["Deploy ${ed_} with MoM"] = {
+ node(slave_node) {
+ edgeBuilds["${ed_}"] = build job: deploy_job, propagate: false, parameters: [
+ [$class: 'StringParameterValue', name: 'HEAT_STACK_ZONE', value: HEAT_STACK_ZONE],
+ [$class: 'StringParameterValue', name: 'OPENSTACK_API_PROJECT', value: OPENSTACK_API_PROJECT],
+ [$class: 'StringParameterValue', name: 'SLAVE_NODE', value: props_['SLAVE_NODE']],
+ [$class: 'StringParameterValue', name: 'STACK_INSTALL', value: props_['STACK_INSTALL']],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE', value: props_['STACK_TEMPLATE']],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE_URL', value: STACK_TEMPLATE_URL],
+ [$class: 'StringParameterValue', name: 'STACK_TEMPLATE_BRANCH', value: 'master'],
+ [$class: 'StringParameterValue', name: 'STACK_TYPE', value: 'physical'],
+ [$class: 'StringParameterValue', name: 'SALT_MASTER_URL', value: salt_mom_url],
+ [$class: 'StringParameterValue', name: 'EXTRA_TARGET', value: extra_target],
+ [$class: 'StringParameterValue', name: 'FORMULA_PKG_REVISION', value: props_['FORMULA_PKG_REVISION']],
+ [$class: 'StringParameterValue', name: 'STACK_CLUSTER_NAME', value: props_['STACK_CLUSTER_NAME']],
+ [$class: 'StringParameterValue', name: 'STACK_TEST', value: ''],
+ [$class: 'BooleanParameterValue', name: 'TEST_DOCKER_INSTALL', value: false],
+ [$class: 'TextParameterValue', name: 'SALT_OVERRIDES', value: salt_overrides_list.join('\n')],
+ [$class: 'BooleanParameterValue', name: 'STACK_DELETE', value: props_['STACK_DELETE'].toBoolean()],
+ ]
+ }
+ }
+ } else {
+ common.successMsg("${k} : " + edgeBuilds[k].result)
+ common.errorMsg("${k} : " + edgeBuilds[k].result)
+ }
+ }
+
+ parallel deploy_edges
+
+ }
+
+
+}
diff --git a/generate-cookiecutter-products.groovy b/generate-cookiecutter-products.groovy
index 0a2888e..6964653 100644
--- a/generate-cookiecutter-products.groovy
+++ b/generate-cookiecutter-products.groovy
@@ -169,7 +169,21 @@
stage("Test") {
if (sharedReclassUrl != "" && TEST_MODEL && TEST_MODEL.toBoolean()) {
sh("cp -r ${modelEnv} ${testEnv}")
- saltModelTesting.setupAndTestNode("${saltMaster}.${clusterDomain}", "", testEnv)
+ def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
+ saltModelTesting.setupAndTestNode(
+ "${saltMaster}.${clusterDomain}",
+ "",
+ "",
+ testEnv,
+ 'pkg',
+ 'stable',
+ 'master',
+ 0,
+ false,
+ false,
+ '',
+ '',
+ DockerCName)
}
}
diff --git a/git-mirror-2way-pipeline.groovy b/git-mirror-2way-pipeline.groovy
index 21d72bf..bd9dcb8 100644
--- a/git-mirror-2way-pipeline.groovy
+++ b/git-mirror-2way-pipeline.groovy
@@ -6,7 +6,7 @@
try{
def branches = BRANCHES.tokenize(',')
def pollBranches = []
- for (i=0; i < branches.size; i++) {
+ for (i=0; i < branches.size(); i++) {
pollBranches.add([name:branches[i]])
}
dir("target") {
diff --git a/git-mirror-pipeline.groovy b/git-mirror-pipeline.groovy
index 45536d0..8bfe467 100644
--- a/git-mirror-pipeline.groovy
+++ b/git-mirror-pipeline.groovy
@@ -6,7 +6,7 @@
try{
def branches = BRANCHES.tokenize(',')
def pollBranches = []
- for (i=0; i < branches.size; i++) {
+ for (i=0; i < branches.size(); i++) {
pollBranches.add([name:branches[i]])
}
dir("source") {
diff --git a/k8s-upgrade-pipeline.groovy b/k8s-upgrade-pipeline.groovy
new file mode 100644
index 0000000..3e7828b
--- /dev/null
+++ b/k8s-upgrade-pipeline.groovy
@@ -0,0 +1,103 @@
+/**
+ * Update kuberentes cluster
+ *
+ * Expected parameters:
+ * SALT_MASTER_CREDENTIALS Credentials to the Salt API.
+ * SALT_MASTER_URL Full Salt API address [https://10.10.10.1:8000].
+ * KUBERNETES_HYPERKUBE_IMAGE Target kubernetes version. May be null in case of reclass-system rollout
+ * KUBERNETES_PAUSE_IMAGE Kubernetes pause image should have same version as hyperkube. May be null in case of reclass-system rollout
+ * TARGET_UPDATES Comma separated list of nodes to update (Valid values are ctl,cmp)
+ * CTL_TARGET Salt targeted kubernetes CTL nodes (ex. I@kubernetes:master). Kubernetes control plane
+ * CMP_TARGET Salt targeted compute nodes (ex. cmp* and 'I@kubernetes:pool') Kubernetes computes
+ * PER_NODE Target nodes will be managed one by one (bool)
+ *
+**/
+def common = new com.mirantis.mk.Common()
+def salt = new com.mirantis.mk.Salt()
+def python = new com.mirantis.mk.Python()
+
+def updates = TARGET_UPDATES.tokenize(",").collect{it -> it.trim()}
+def pepperEnv = "pepperEnv"
+
+def overrideKubernetesImage(pepperEnv) {
+ def salt = new com.mirantis.mk.Salt()
+
+ def k8sSaltOverrides = """
+ kubernetes_hyperkube_image: ${KUBERNETES_HYPERKUBE_IMAGE}
+ kubernetes_pause_image: ${KUBERNETES_PAUSE_IMAGE}
+ """
+ stage("Override kubernetes images to target version") {
+ salt.setSaltOverrides(pepperEnv, k8sSaltOverrides)
+ }
+}
+
+def performKubernetesComputeUpdate(pepperEnv, target) {
+ def salt = new com.mirantis.mk.Salt()
+
+ stage("Execute Kubernetes compute update on ${target}") {
+ salt.enforceState(pepperEnv, target, 'kubernetes.pool')
+ salt.runSaltProcessStep(pepperEnv, target, 'service.restart', ['kubelet'])
+ }
+}
+
+def performKubernetesControlUpdate(pepperEnv, target) {
+ def salt = new com.mirantis.mk.Salt()
+
+ stage("Execute Kubernetes control plane update on ${target}") {
+ salt.enforceStateWithExclude(pepperEnv, target, "kubernetes", "kubernetes.master.setup")
+ // Restart kubelet
+ salt.runSaltProcessStep(pepperEnv, target, 'service.restart', ['kubelet'])
+ }
+}
+
+
+timeout(time: 12, unit: 'HOURS') {
+ node() {
+ try {
+
+ stage("Setup virtualenv for Pepper") {
+ python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
+ }
+
+ if ((common.validInputParam('KUBERNETES_HYPERKUBE_IMAGE')) && (common.validInputParam('KUBERNETES_PAUSE_IMAGE'))) {
+ overrideKubernetesImage(pepperEnv)
+ }
+
+ /*
+ * Execute update
+ */
+ if (updates.contains("ctl")) {
+ def target = CTL_TARGET
+
+ if (PER_NODE.toBoolean()) {
+ def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+
+ for (t in targetHosts) {
+ performKubernetesControlUpdate(pepperEnv, t)
+ }
+ } else {
+ performKubernetesControlUpdate(pepperEnv, target)
+ }
+ }
+
+ if (updates.contains("cmp")) {
+ def target = CMP_TARGET
+
+ if (PER_NODE.toBoolean()) {
+ def targetHosts = salt.getMinionsSorted(pepperEnv, target)
+
+ for (t in targetHosts) {
+ performKubernetesComputeUpdate(pepperEnv, t)
+ }
+ } else {
+ performKubernetesComputeUpdate(pepperEnv, target)
+ }
+ }
+ } catch (Throwable e) {
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+ throw e
+ }
+ }
+}
\ No newline at end of file
diff --git a/openstack-compute-upgrade.groovy b/openstack-compute-upgrade.groovy
index a59fc08..2984b55 100644
--- a/openstack-compute-upgrade.groovy
+++ b/openstack-compute-upgrade.groovy
@@ -7,6 +7,7 @@
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* TARGET_SUBSET_TEST Number of nodes to list package updates, empty string means all targetted nodes.
* TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
*
**/
@@ -96,8 +97,10 @@
salt.runSaltProcessStep(pepperEnv, targetTestSubset, 'pkg.list_upgrades', [], null, true)
}
- stage('Confirm upgrade on sample nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample nodes') {
input message: "Please verify the list of packages that you want to be upgraded. Do you want to continue with upgrade?"
+ }
}
stage("Add new repos on sample nodes") {
@@ -136,8 +139,10 @@
}
}
- stage('Confirm upgrade on sample') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample') {
input message: "Please verify if there are packages that it wants to downgrade. If so, execute apt-cache policy on them and verify if everything is fine. Do you want to continue with upgrade?"
+ }
}
command = "cmd.run"
@@ -180,10 +185,12 @@
}
}
- stage('Confirm upgrade on all targeted nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on all targeted nodes') {
timeout(time: 2, unit: 'HOURS') {
input message: "Verify that the upgraded sample nodes are working correctly. If so, do you want to approve live upgrade on ${targetLiveAll} nodes?"
}
+ }
}
stage("Add new repos on all targeted nodes") {
diff --git a/openstack-control-upgrade.groovy b/openstack-control-upgrade.groovy
index d306e34..89b5e77 100644
--- a/openstack-control-upgrade.groovy
+++ b/openstack-control-upgrade.groovy
@@ -9,6 +9,7 @@
* STAGE_ROLLBACK_UPGRADE Run rollback upgrade stage (bool)
* SKIP_VM_RELAUNCH Set to true if vms should not be recreated (bool)
* OPERATING_SYSTEM_RELEASE_UPGRADE Set to true if operating system of vms should be upgraded to newer release (bool)
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
*
**/
@@ -195,10 +196,10 @@
salt.cmdRun(pepperEnv, "${test_upgrade_node}*", '. /root/keystonercv3; openstack service list; openstack image list; openstack flavor list; openstack compute service list; openstack server list; openstack network list; openstack volume list; openstack orchestration service list')
- if (STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
- stage('Ask for manual confirmation') {
- input message: "Do you want to continue with upgrade?"
- }
+ if (INTERACTIVE.toBoolean() && STAGE_TEST_UPGRADE.toBoolean() == true && STAGE_REAL_UPGRADE.toBoolean() == true) {
+ stage('Ask for manual confirmation') {
+ input message: "Do you want to continue with upgrade?"
+ }
}
}
@@ -303,8 +304,12 @@
stopServices(pepperEnv, proxy_node, proxy_general_target, 'prx')
stopServices(pepperEnv, control_node, control_general_target, 'ctl')
salt.printSaltCommandResult(out)
- if (out.toString().contains("dpkg returned an error code")) {
- input message: "Apt dist-upgrade failed, please fix it manually and then click on proceed. If unable to fix it, click on abort and run the rollback stage."
+ if (out.toString().contains("dpkg returned an error code")){
+ if (INTERACTIVE.toBoolean()) {
+ input message: "Apt dist-upgrade failed, please fix it manually and then click on proceed. If unable to fix it, click on abort and run the rollback stage."
+ } else {
+ error("Apt dist-upgrade failed. And interactive mode was disabled, failing...")
+ }
}
// run base states
try {
@@ -355,10 +360,14 @@
retryStateRun(pepperEnv, control_general_target, 'heat')
} catch (Exception e) {
errorOccured = true
- if (OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == false) {
- input message: "Some states that require syncdb failed. Please check the reason.Click proceed only if you want to restore database into it's pre-upgrade state. If you want restore production database and also the VMs into its pre-upgrade state please click on abort and run the rollback stage."
+ if (INTERACTIVE.toBoolean()){
+ if (OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == false) {
+ input message: "Some states that require syncdb failed. Please check the reason. Click proceed only if you want to restore database into it's pre-upgrade state. If you want restore production database and also the VMs into its pre-upgrade state please click on abort and run the rollback stage."
+ } else {
+ input message: "Some states that require syncdb failed. Please check the reason and click proceed only if you want to restore database into it's pre-upgrade state. Otherwise, click abort."
+ }
} else {
- input message: "Some states that require syncdb failed. Please check the reason and click proceed only if you want to restore database into it's pre-upgrade state. Otherwise, click abort."
+ error("Stage Real control upgrade failed. And interactive mode was disabled, failing...")
}
openstack.restoreGaleraDb(pepperEnv)
common.errorMsg("Stage Real control upgrade failed")
@@ -420,7 +429,9 @@
/*
if (OPERATING_SYSTEM_RELEASE_UPGRADE.toBoolean() == false) {
- input message: "Please verify if the control upgrade was successful! If so, by clicking proceed the original VMs disk images will be backed up and snapshot will be merged to the upgraded VMs which will finalize the upgrade procedure"
+ if (INTERACTIVE.toBoolean()){
+ input message: "Please verify if the control upgrade was successful! If so, by clicking proceed the original VMs disk images will be backed up and snapshot will be merged to the upgraded VMs which will finalize the upgrade procedure"
+ }
node_count = 1
for (t in proxy_target_hosts) {
def target = salt.stripDomainName(t)
@@ -445,8 +456,10 @@
virsh.liveSnapshotMerge(pepperEnv, nodeProvider, target, snapshotName)
node_count++
}
- input message: "Please scroll up and look for red highlighted messages containing 'virsh blockcommit' string.
- If there are any fix it manually. Otherwise click on proceed."
+ if (INTERACTIVE.toBoolean()){
+ input message: "Please scroll up and look for red highlighted messages containing 'virsh blockcommit' string.
+ If there are any fix it manually. Otherwise click on proceed."
+ }
}
*/
}
@@ -548,7 +561,7 @@
vcpRealUpgrade(pepperEnv)
}
- if (STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
+ if (INTERACTIVE.toBoolean() && STAGE_REAL_UPGRADE.toBoolean() == true && STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
stage('Ask for manual confirmation') {
input message: "Please verify if the control upgrade was successful. If it did not succeed, in the worst scenario, you can click on proceed to continue with control-upgrade-rollback. Do you want to continue with the rollback?"
}
@@ -557,11 +570,13 @@
if (STAGE_ROLLBACK_UPGRADE.toBoolean() == true) {
stage('Rollback upgrade') {
- stage('Ask for manual confirmation') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Ask for manual confirmation') {
input message: "Before rollback please check the documentation for reclass model changes. Do you really want to continue with the rollback?"
+ }
}
vcpRollback(pepperEnv)
}
}
}
-}
\ No newline at end of file
+}
diff --git a/ovs-gateway-upgrade.groovy b/ovs-gateway-upgrade.groovy
index 8d131c8..87cf828 100644
--- a/ovs-gateway-upgrade.groovy
+++ b/ovs-gateway-upgrade.groovy
@@ -7,6 +7,7 @@
* TARGET_SERVERS Salt compound target to match nodes to be updated [*, G@osfamily:debian].
* TARGET_SUBSET_TEST Number of nodes to list package updates, empty string means all targetted nodes.
* TARGET_SUBSET_LIVE Number of selected nodes to live apply selected package update.
+ * INTERACTIVE Ask interactive questions during pipeline run (bool).
*
**/
@@ -63,8 +64,10 @@
salt.runSaltProcessStep(pepperEnv, targetTestSubset, 'pkg.list_upgrades', [], null, true)
}
- stage('Confirm upgrade on sample nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample nodes') {
input message: "Please verify the list of packages that you want to be upgraded. Do you want to continue with upgrade?"
+ }
}
stage("Add new repos on sample nodes") {
@@ -81,8 +84,10 @@
}
}
- stage('Confirm upgrade on sample') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on sample') {
input message: "Please verify if there are packages that it wants to downgrade. If so, execute apt-cache policy on them and verify if everything is fine. Do you want to continue with upgrade?"
+ }
}
command = "cmd.run"
@@ -111,10 +116,12 @@
}
}
- stage('Confirm upgrade on all targeted nodes') {
+ if (INTERACTIVE.toBoolean()){
+ stage('Confirm upgrade on all targeted nodes') {
timeout(time: 2, unit: 'HOURS') {
- input message: "Verify that the upgraded sample nodes are working correctly. If so, do you want to approve live upgrade on ${targetLiveAll} nodes?"
+ input message: "Verify that the upgraded sample nodes are working correctly. If so, do you want to approve live upgrade on ${targetLiveAll} nodes?"
}
+ }
}
stage("Add new repos on all targeted nodes") {
diff --git a/test-cookiecutter-reclass.groovy b/test-cookiecutter-reclass.groovy
index ba05406..fedbd14 100644
--- a/test-cookiecutter-reclass.groovy
+++ b/test-cookiecutter-reclass.groovy
@@ -96,7 +96,21 @@
while (nbTry < 5) {
nbTry++
try {
- saltModelTesting.setupAndTestNode("cfg01.${clusterDomain}", clusterName, EXTRA_FORMULAS, testEnv, "pkg", DISTRIB_REVISION)
+ def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
+ saltModelTesting.setupAndTestNode(
+ "cfg01.${clusterDomain}",
+ clusterName,
+ EXTRA_FORMULAS,
+ testEnv,
+ 'pkg',
+ DISTRIB_REVISION,
+ 'master',
+ 0,
+ false,
+ false,
+ '',
+ '',
+ DockerCName)
break
} catch (Exception e) {
if (e.getMessage() == "script returned exit code 124") {
@@ -174,11 +188,10 @@
stage("test-nodes") {
def partitions = common.partitionList(contextFileList, PARALLEL_NODE_GROUP_SIZE.toInteger())
def buildSteps = [:]
- for (int i = 0; i < partitions.size(); i++) {
- def partition = partitions[i]
+ partitions.eachWithIndex { partition, i ->
buildSteps.put("partition-${i}", new HashMap<String,org.jenkinsci.plugins.workflow.cps.CpsClosure2>())
- for(int k = 0; k < partition.size; k++){
- def basename = sh(script: "basename ${partition[k]} .yml", returnStdout: true).trim()
+ for(part in partition){
+ def basename = sh(script: "basename ${part} .yml", returnStdout: true).trim()
def testEnv = "${env.WORKSPACE}/model/${basename}"
buildSteps.get("partition-${i}").put(basename, { testModel(basename, testEnv) })
}
diff --git a/test-salt-formulas-pipeline.groovy b/test-salt-formulas-pipeline.groovy
index 401628b..ca4eb67 100644
--- a/test-salt-formulas-pipeline.groovy
+++ b/test-salt-formulas-pipeline.groovy
@@ -182,9 +182,6 @@
common.errorMsg("----------------KITCHEN LOG:---------------")
println readFile(".kitchen/logs/kitchen.log")
}
- def slack = new com.mirantis.mcp.SlackNotification()
- slack.jobResultNotification("success", "#test_reclass_notify")
- common.sendNotification(currentBuild.result, "", ["slack"])
}
}
}
diff --git a/test-salt-model-node.groovy b/test-salt-model-node.groovy
index 7fc333d..694f048 100644
--- a/test-salt-model-node.groovy
+++ b/test-salt-model-node.groovy
@@ -64,7 +64,22 @@
def workspace = common.getWorkspace()
common.infoMsg("Running salt model test for node ${NODE_TARGET} in cluster ${CLUSTER_NAME}")
try {
- saltModelTesting.setupAndTestNode(NODE_TARGET, CLUSTER_NAME, EXTRA_FORMULAS, workspace, FORMULAS_SOURCE, FORMULAS_REVISION, RECLASS_VERSION, MAX_CPU_PER_JOB.toInteger(), RECLASS_IGNORE_CLASS_NOTFOUND, LEGACY_TEST_MODE, APT_REPOSITORY, APT_REPOSITORY_GPG)
+ def DockerCName = "${env.JOB_NAME.toLowerCase()}_${env.BUILD_TAG.toLowerCase()}"
+
+ test_result = saltModelTesting.setupAndTestNode(
+ NODE_TARGET,
+ CLUSTER_NAME,
+ EXTRA_FORMULAS,
+ workspace,
+ FORMULAS_SOURCE,
+ FORMULAS_REVISION,
+ RECLASS_VERSION,
+ MAX_CPU_PER_JOB.toInteger(),
+ RECLASS_IGNORE_CLASS_NOTFOUND,
+ LEGACY_TEST_MODE,
+ APT_REPOSITORY,
+ APT_REPOSITORY_GPG,
+ DockerCName)
} catch (Exception e) {
if (e.getMessage() == "script returned exit code 124") {
common.errorMsg("Impossible to test node due to timeout of salt-master, ABORTING BUILD")
@@ -73,13 +88,19 @@
throw e
}
}
+ if (test_result) {
+ common.infoMsg("Test finished: SUCCESS")
+ } else {
+ common.warningMsg("Test finished: FAILURE")
+ currentBuild.result = "FAILURE"
+ }
}
}
} catch (Throwable e) {
- // If there was an error or exception thrown, the build failed
- currentBuild.result = "FAILURE"
- currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
- throw e
+ // If there was an error or exception thrown, the build failed
+ currentBuild.result = "FAILURE"
+ currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
+ throw e
}
}
}
diff --git a/test-system-reclass-pipeline.groovy b/test-system-reclass-pipeline.groovy
index 7fc1181..fa16739 100644
--- a/test-system-reclass-pipeline.groovy
+++ b/test-system-reclass-pipeline.groovy
@@ -26,6 +26,7 @@
def checkouted = false
def merged = false
def systemRefspec = "HEAD"
+def formulasRevision = 'testing'
timeout(time: 12, unit: 'HOURS') {
node() {
try {
@@ -71,14 +72,17 @@
[$class: 'StringParameterValue', name: 'DEFAULT_GIT_URL', value: clusterGitUrl],
[$class: 'StringParameterValue', name: 'DEFAULT_GIT_REF', value: "HEAD"],
[$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: defaultGitUrl],
- [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec]
+ [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec],
+ [$class: 'StringParameterValue', name: 'FORMULAS_REVISION', value: formulasRevision],
]
}
}
branches["cookiecutter"] = {
build job: "test-mk-cookiecutter-templates", parameters: [
[$class: 'StringParameterValue', name: 'SYSTEM_GIT_URL', value: defaultGitUrl],
- [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec]
+ [$class: 'StringParameterValue', name: 'SYSTEM_GIT_REF', value: systemRefspec],
+ [$class: 'StringParameterValue', name: 'DISTRIB_REVISION', value: formulasRevision]
+
]
}
parallel branches
diff --git a/upgrade-mcp-release.groovy b/upgrade-mcp-release.groovy
index 0c5c971..08796c9 100644
--- a/upgrade-mcp-release.groovy
+++ b/upgrade-mcp-release.groovy
@@ -6,19 +6,19 @@
* SALT_MASTER_URL Salt API server location
* SALT_MASTER_CREDENTIALS Credentials to the Salt API
* MCP_VERSION Version of MCP to upgrade to
+ * UPGRADE_SALTSTACK Upgrade SaltStack packages to new version.
* UPDATE_CLUSTER_MODEL Update MCP version parameter in cluster model
* UPDATE_PIPELINES Update pipeline repositories on Gerrit
* UPDATE_LOCAL_REPOS Update local repositories
*/
-import hudson.model.*
-
// Load shared libs
salt = new com.mirantis.mk.Salt()
common = new com.mirantis.mk.Common()
python = new com.mirantis.mk.Python()
jenkinsUtils = new com.mirantis.mk.JenkinsUtils()
venvPepper = "venvPepper"
+workspace = ""
def triggerMirrorJob(jobName){
params = jenkinsUtils.getJobParameters(jobName)
@@ -30,9 +30,44 @@
]
}
+def updateSaltStack(target, pkgs){
+ try{
+ salt.runSaltProcessStep(venvPepper, target, 'pkg.install', ["force_yes=True", "pkgs='$pkgs'"], null, true, 5)
+ }catch(Exception ex){}
+
+ common.retry(10, 30){
+ salt.minionsReachable(venvPepper, 'I@salt:master', '*')
+ def running = salt.runSaltProcessStep(venvPepper, target, 'saltutil.running', [], null, true, 5)
+ for(value in running.get("return")[0].values()){
+ if(value != []){
+ throw new Exception("Not all salt-minions are ready for execution")
+ }
+ }
+ }
+
+ def saltVersion = salt.getPillar(venvPepper, 'I@salt:master', "_param:salt_version").get("return")[0].values()[0]
+ def saltMinionVersions = salt.cmdRun(venvPepper, "*", "apt-cache policy salt-common | awk '/Installed/ && /$saltVersion/'").get("return")
+ def saltMinionVersion = ""
+
+ for(minion in saltMinionVersions[0].keySet()){
+ saltMinionVersion = saltMinionVersions[0].get(minion).replace("Salt command execution success","").trim()
+ if(saltMinionVersion == ""){
+ error("Installed version of Salt on $minion doesn't match specified version in the model.")
+ }
+ }
+}
+
+def archiveReclassInventory(filename){
+ def ret = salt.cmdRun(venvPepper, 'I@salt:master', "reclass -i", true, null, false)
+ def reclassInv = ret.values()[0]
+ writeFile file: filename, text: reclassInv.toString()
+ archiveArtifacts artifacts: "$filename"
+}
+
timeout(time: 12, unit: 'HOURS') {
node("python") {
try {
+ workspace = common.getWorkspace()
python.setupPepperVirtualenv(venvPepper, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
if(MCP_VERSION == ""){
@@ -108,6 +143,16 @@
salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_salt.list")
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_salt.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades salt-formula-*")
+
+ def inventoryBeforeFilename = "reclass-inventory-before.out"
+ def inventoryAfterFilename = "reclass-inventory-after.out"
+
+ archiveReclassInventory(inventoryBeforeFilename)
+
+ salt.cmdRun(venvPepper, 'I@salt:master', "sed -i -e 's/[^ ]*[^ ]/$MCP_VERSION/4' /etc/apt/sources.list.d/mcp_extra.list")
+ salt.cmdRun(venvPepper, 'I@salt:master', "apt-get -o Dir::Etc::sourcelist='/etc/apt/sources.list.d/mcp_extra.list' -o Dir::Etc::sourceparts='-' -o APT::Get::List-Cleanup='0' update")
+ salt.cmdRun(venvPepper, 'I@salt:master', "apt-get install -y --allow-downgrades reclass")
+
salt.fullRefresh(venvPepper, 'I@salt:master')
try{
@@ -126,6 +171,19 @@
error("Reclass fails rendering. Pay attention to your cluster model.")
}
+ archiveReclassInventory(inventoryAfterFilename)
+
+ sh "diff -u $inventoryBeforeFilename $inventoryAfterFilename > reclass-inventory-diff.out || true"
+ archiveArtifacts artifacts: "reclass-inventory-diff.out"
+
+ if(UPGRADE_SALTSTACK.toBoolean()){
+ salt.enforceState(venvPepper, "I@linux:system", 'linux.system.repo', true)
+
+ updateSaltStack("I@salt:master", '["salt-master", "salt-common", "salt-api", "salt-minion"]')
+
+ updateSaltStack("I@salt:minion and not I@salt:master", '["salt-minion"]')
+ }
+
if(UPDATE_PIPELINES.toBoolean()){
triggerMirrorJob("git-mirror-downstream-mk-pipelines")
triggerMirrorJob("git-mirror-downstream-pipeline-library")