| /** |
| * |
| * Launch HA test for the cloud |
| * |
| * Expected parameters: |
| * |
| * SALT_MASTER_URL URL of Salt master |
| * SALT_MASTER_CREDENTIALS Credentials that are used in this Jenkins for accessing Salt master (usually "salt") |
| * PROXY Proxy address (if any) for accessing the Internet. It will be used for cloning repos and installing pip dependencies |
| * TEST_IMAGE Docker image link to use for running container with testing tools. |
| * TOOLS_REPO URL of repo where testing tools, scenarios, configs are located |
| * |
| * DEBUG_MODE If you need to debug (keep container after test), please enabled this |
| * MANUAL_CONFIRMATION Ask for confirmation before doing something destructive (reboot/shutdown node) |
| * RETRY_CHECK_STATUS Number of retries to check node status |
| * SKIP_LIST_PATH Path to tempest skip list file in TOOLS_REPO |
| * TARGET_NODES Nodes to test |
| * TEMPEST_REPO Tempest repo to clone and use |
| * TEMPEST_TARGET_NODE Node, where tests will be executed |
| * TEMPEST_TEST_PATTERN Tests to run during HA scenarios |
| * |
| */ |
| |
| common = new com.mirantis.mk.Common() |
| salt = new com.mirantis.mk.Salt() |
| validate = new com.mirantis.mcp.Validate() |
| |
| def saltMaster |
| def artifacts_dir = 'validation_artifacts/' |
| def remote_artifacts_dir = '/root/qa_results/' |
| def current_target_node = null |
| def first_node = null |
| def tempest_result = '' |
| timeout(time: 12, unit: 'HOURS') { |
| node() { |
| def num_retries = Integer.parseInt(RETRY_CHECK_STATUS) |
| try { |
| stage('Initialization') { |
| saltMaster = salt.connection(SALT_MASTER_URL, SALT_MASTER_CREDENTIALS) |
| validate.runBasicContainer(saltMaster, TEMPEST_TARGET_NODE, TEST_IMAGE) |
| sh "rm -rf ${artifacts_dir}" |
| salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}") |
| salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "mkdir -p ${remote_artifacts_dir}") |
| validate.configureContainer(saltMaster, TEMPEST_TARGET_NODE, PROXY, TOOLS_REPO, TEMPEST_REPO) |
| } |
| |
| stage('Initial env check') { |
| sh "mkdir -p ${artifacts_dir}" |
| tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_initial") |
| validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (tempest_result != "finished") { |
| currentBuild.result = "FAILURE" |
| throw new Exception("Tempest tests failed") |
| } |
| } |
| |
| stage('Soft Shutdown') { |
| if (MANUAL_CONFIRMATION.toBoolean() == true) { |
| stage('Ask for manual confirmation') { |
| input message: "Are you sure you want to shutdown current vip node?" |
| } |
| } |
| current_target_node = validate.get_vip_node(saltMaster, TARGET_NODES) |
| if (current_target_node == null) { |
| throw new Exception("Cannot current vip node in ${TARGET_NODES} nodes") |
| } |
| common.warningMsg("Shutdown current vip node ${current_target_node}") |
| validate.shutdown_vm_node(saltMaster, current_target_node, 'soft_shutdown') |
| sleep 15 |
| } |
| stage('Check during shutdown') { |
| tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_during_shutdown") |
| validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (tempest_result != "finished") { |
| currentBuild.result = "FAILURE" |
| throw new Exception("Tempest tests failed") |
| } |
| } |
| stage('Power on') { |
| common.infoMsg('Powering on node') |
| kvm = validate.locate_node_on_kvm(saltMaster, current_target_node) |
| salt.cmdRun(saltMaster, kvm, "virsh start ${current_target_node}") |
| common.infoMsg("Checking that node is UP") |
| status = salt.minionsReachable(saltMaster, 'I@salt:master', current_target_node, null, 10, num_retries) |
| if (status == null) { |
| throw new Exception("Node ${current_target_node} cannot start") |
| } |
| first_node = current_target_node |
| current_target_node = null |
| sleep 30 |
| } |
| stage('Check after shutdown') { |
| tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_after_shutdown") |
| validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (tempest_result != "finished") { |
| currentBuild.result = "FAILURE" |
| throw new Exception("Tempest tests failed") |
| } |
| sleep 15 |
| } |
| |
| stage('Hard Shutdown') { |
| if (MANUAL_CONFIRMATION.toBoolean() == true) { |
| stage('Ask for manual confirmation') { |
| input message: "Are you sure you want to hard shutdown current vip node?" |
| } |
| } |
| salt.cmdRun(saltMaster, first_node, "service keepalived stop") |
| current_target_node = validate.get_vip_node(saltMaster, TARGET_NODES) |
| common.warningMsg("Shutdown current vip node ${current_target_node}") |
| validate.shutdown_vm_node(saltMaster, current_target_node, 'hard_shutdown') |
| //TODO:if previous command fails, keeaplived will not be started on first_node |
| sleep 10 |
| salt.cmdRun(saltMaster, first_node, "service keepalived start") |
| } |
| stage('Check during hard shutdown') { |
| tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_during_hard_shutdown") |
| validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (tempest_result != "finished") { |
| currentBuild.result = "FAILURE" |
| throw new Exception("Tempest tests failed") |
| } |
| } |
| stage('Power on') { |
| common.infoMsg('Powering on node') |
| kvm = validate.locate_node_on_kvm(saltMaster, current_target_node) |
| salt.cmdRun(saltMaster, kvm, "virsh start ${current_target_node}") |
| common.infoMsg("Checking that node is UP") |
| status = salt.minionsReachable(saltMaster, 'I@salt:master', current_target_node, null, 10, num_retries) |
| if (status == null) { |
| throw new Exception("Node ${current_target_node} cannot start") |
| } |
| current_target_node = null |
| sleep 30 |
| } |
| stage('Check after hard shutdown') { |
| tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_after_hard_shutdown") |
| validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (tempest_result != "finished") { |
| currentBuild.result = "FAILURE" |
| throw new Exception("Tempest tests failed") |
| } |
| sleep 5 |
| } |
| |
| stage('Reboot') { |
| if (MANUAL_CONFIRMATION.toBoolean() == true) { |
| stage('Ask for manual confirmation') { |
| input message: "Are you sure you want to reboot current vip node?" |
| } |
| } |
| current_target_node = validate.get_vip_node(saltMaster, TARGET_NODES) |
| common.warningMsg("Rebooting current vip node ${current_target_node}") |
| validate.shutdown_vm_node(saltMaster, current_target_node, 'reboot') |
| sleep 5 |
| } |
| stage('Check during reboot') { |
| tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_during_reboot") |
| validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (tempest_result != "finished") { |
| currentBuild.result = "FAILURE" |
| throw new Exception("Tempest tests failed") |
| } |
| sleep 30 |
| } |
| stage('Check after reboot') { |
| common.warningMsg("Checking that node is UP") |
| status = salt.minionsReachable(saltMaster, 'I@salt:master', current_target_node, null, 10, num_retries) |
| if (status == null) { |
| throw new Exception("Node ${current_target_node} cannot start") |
| } |
| current_target_node = null |
| tempest_result = validate.runCVPtempest(saltMaster, TEMPEST_TARGET_NODE, TEMPEST_TEST_PATTERN, SKIP_LIST_PATH, remote_artifacts_dir, "docker_tempest_after") |
| validate.openstack_cleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (tempest_result != "finished") { |
| currentBuild.result = "FAILURE" |
| throw new Exception("Tempest tests failed") |
| } |
| } |
| |
| stage('Collect results') { |
| validate.addFiles(saltMaster, TEMPEST_TARGET_NODE, remote_artifacts_dir, artifacts_dir) |
| archiveArtifacts artifacts: "${artifacts_dir}/*" |
| if (DEBUG_MODE == 'false') { |
| validate.runCleanup(saltMaster, TEMPEST_TARGET_NODE) |
| salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}") |
| } |
| } |
| } finally { |
| if (DEBUG_MODE == 'false') { |
| salt.cmdRun(saltMaster, TEMPEST_TARGET_NODE, "rm -rf ${remote_artifacts_dir}") |
| validate.runCleanup(saltMaster, TEMPEST_TARGET_NODE) |
| if (current_target_node != null) { |
| common.warningMsg("Powering on node ${current_target_node}") |
| kvm = validate.locate_node_on_kvm(saltMaster, current_target_node) |
| salt.cmdRun(saltMaster, kvm, "virsh start ${current_target_node}") |
| } |
| } |
| } |
| } |
| } |