Merge "Add retry for salttimeout errors and ceph sporadic Related-Prod:PROD-37137"
diff --git a/jobs/global.yaml b/jobs/global.yaml
index 8f1b680..c111b77 100755
--- a/jobs/global.yaml
+++ b/jobs/global.yaml
@@ -10,8 +10,8 @@
<!--- INSERT ALERT HERE ^^^^^^^ --->
</font></h1>
- current-version: 2019.2.23
- previous-version: 2019.2.22
- disabled-proposed: true
- disabled-2019-2-0: false
- upgrade-to-tag: true
+ current-version: 2019.2.24
+ previous-version: 2019.2.23
+ disabled-proposed: false
+ disabled-2019-2-0: true
+ upgrade-to-tag: false
diff --git a/jobs/pipelines/deploy-cicd-and-run-tests.groovy b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
index ff8a9f8..3c0b328 100644
--- a/jobs/pipelines/deploy-cicd-and-run-tests.groovy
+++ b/jobs/pipelines/deploy-cicd-and-run-tests.groovy
@@ -11,6 +11,7 @@
def batch_size = env.BATCH_SIZE ?: ''
def dist_upgrade_nodes = "${env.DIST_UPGRADE_NODES}" != "false" ? true : false
def upgrade_saltstack = "${env.UPGRADE_SALTSTACK}" != "false" ? true : false
+def upgrade_version_tag = "${env.UPGRADE_VERSION_TAG}"
if (env_manager == 'devops') {
jenkins_slave_node_name = "${NODE_NAME}"
@@ -228,7 +229,7 @@
upgrade_to_tag = ''
contrail_upgrade_line = ''
if(env.UPGRADE_TO_TAG == "true") {
- upgrade_to_tag = '--update-to-tag ${env.UPGRADE_VERSION_TAG} '
+ upgrade_to_tag = "--update-to-tag ${upgrade_version_tag} "
}
if(env.PLATFORM_STACK_INSTALL.contains("contrail")) {
contrail_upgrade_line = "tcp_tests/tests/system/test_upgrade_contrail.py::TestUpdateContrail "
diff --git a/jobs/pipelines/swarm-create-cfg-config-drive.groovy b/jobs/pipelines/swarm-create-cfg-config-drive.groovy
index c55005f..aff36df 100644
--- a/jobs/pipelines/swarm-create-cfg-config-drive.groovy
+++ b/jobs/pipelines/swarm-create-cfg-config-drive.groovy
@@ -209,7 +209,7 @@
apt:
sources:
jenkins-formula:
- source: deb [trusted=yes] http://srv08-srt.infra.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main
+ source: deb [trusted=yes] http://cz7780.bud.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main
merge_how: "dict(recurse_array)+list(append)"
"""
diff --git a/tcp_tests/templates/cookied-model-generator/salt_heat-cicd-pike-contrail-stb-sl.yaml b/tcp_tests/templates/cookied-model-generator/salt_heat-cicd-pike-contrail-stb-sl.yaml
index 0e2b9da..50829d3 100755
--- a/tcp_tests/templates/cookied-model-generator/salt_heat-cicd-pike-contrail-stb-sl.yaml
+++ b/tcp_tests/templates/cookied-model-generator/salt_heat-cicd-pike-contrail-stb-sl.yaml
@@ -40,7 +40,7 @@
- description: Inject jenkins-formula extra repo to cluster model
cmd: |
. venv-reclass-tools/bin/activate;
- reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://srv08-srt.infra.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
+ reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://cz7780.bud.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
diff --git a/tcp_tests/templates/cookied-model-generator/salt_released-bm-b300-cicd-queens-ovs-maas.yaml b/tcp_tests/templates/cookied-model-generator/salt_released-bm-b300-cicd-queens-ovs-maas.yaml
index 1496add..c0292a4 100644
--- a/tcp_tests/templates/cookied-model-generator/salt_released-bm-b300-cicd-queens-ovs-maas.yaml
+++ b/tcp_tests/templates/cookied-model-generator/salt_released-bm-b300-cicd-queens-ovs-maas.yaml
@@ -72,7 +72,7 @@
- description: Inject jenkins-formula extra repo to cluster model
cmd: |
. venv-reclass-tools/bin/activate;
- reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://srv08-srt.infra.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
+ reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://cz7780.bud.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
diff --git a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-contrail41-sl.yaml b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-contrail41-sl.yaml
index 9733ccb..c3f1158 100644
--- a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-contrail41-sl.yaml
+++ b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-contrail41-sl.yaml
@@ -40,7 +40,7 @@
- description: Inject jenkins-formula extra repo to cluster model
cmd: |
. venv-reclass-tools/bin/activate;
- reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://srv08-srt.infra.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
+ reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://cz7780.bud.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
diff --git a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-dvr-sl.yaml b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-dvr-sl.yaml
index 346e8ed..089c1b6 100644
--- a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-dvr-sl.yaml
+++ b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-pike-dvr-sl.yaml
@@ -36,7 +36,7 @@
- description: Inject jenkins-formula extra repo to cluster model
cmd: |
. venv-reclass-tools/bin/activate;
- reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://srv08-srt.infra.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
+ reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://cz7780.bud.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
diff --git a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-contrail41-sl.yaml b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-contrail41-sl.yaml
index 9acdc56..277b051 100644
--- a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-contrail41-sl.yaml
+++ b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-contrail41-sl.yaml
@@ -40,7 +40,7 @@
- description: Inject jenkins-formula extra repo to cluster model
cmd: |
. venv-reclass-tools/bin/activate;
- reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://srv08-srt.infra.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
+ reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://cz7780.bud.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
diff --git a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-dvr-sl.yaml b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-dvr-sl.yaml
index 02c19a8..7d2bc29 100644
--- a/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-dvr-sl.yaml
+++ b/tcp_tests/templates/cookied-model-generator/salt_released-heat-cicd-queens-dvr-sl.yaml
@@ -36,7 +36,7 @@
- description: Inject jenkins-formula extra repo to cluster model
cmd: |
. venv-reclass-tools/bin/activate;
- reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://srv08-srt.infra.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
+ reclass-tools add-key parameters.linux.system.repo.salt-formula-jenkins-2019-2-19.source "deb [trusted=yes] http://cz7780.bud.mirantis.net:8080/salt-formula-jenkins-2019.2.19/ xenial main" /srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/infra/config/init.yml
node_name: {{ HOSTNAME_CFG01 }}
retry: {count: 1, delay: 1}
skip_fail: false
diff --git a/tcp_tests/templates/heat-cicd-queens-dvr-sl/salt-context-cookiecutter.yaml b/tcp_tests/templates/heat-cicd-queens-dvr-sl/salt-context-cookiecutter.yaml
index 86344d9..d53bbca 100644
--- a/tcp_tests/templates/heat-cicd-queens-dvr-sl/salt-context-cookiecutter.yaml
+++ b/tcp_tests/templates/heat-cicd-queens-dvr-sl/salt-context-cookiecutter.yaml
@@ -323,7 +323,7 @@
openstack_barbican_node03_address: 10.167.4.47
openstack_barbican_node03_hostname: kmn03
- designate_backend: powerdns
+ designate_backend: 'bind'
designate_enabled: 'True'
openstack_dns_node01_address: 10.167.4.113
openstack_dns_node02_address: 10.167.4.114
diff --git a/tcp_tests/templates/shared-salt.yaml b/tcp_tests/templates/shared-salt.yaml
index 354d59e..1f43386 100644
--- a/tcp_tests/templates/shared-salt.yaml
+++ b/tcp_tests/templates/shared-salt.yaml
@@ -1559,23 +1559,12 @@
{%- macro MACRO_CHECK_SALT_VERSION_ON_NODES() %}
{#####################################################}
-{%- for ssh in config.underlay.ssh %}
- {%- set salt_roles = [] %}
- {%- for role in ssh['roles'] %}
- {%- if role in config.salt_deploy.salt_roles %}
- {%- set _ = salt_roles.append(role) %}
- {%- endif %}
- {%- endfor %}
-
- {%- if salt_roles %}
-- description: 'Check salt version is as expected'
- cmd: salt-call test.version | grep {{ SALT_VERSION }}
- node_name: {{ ssh['node_name'] }}
- retry: {count: 1, delay: 1}
+- description: 'Check salt version is as expected on nodes'
+ cmd: salt --timeout=15 '*' cmd.run "if salt-call test.version |grep -q {{ SALT_VERSION }} ; then echo True; fi"
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 5, delay: 5}
skip_fail: false
- {%- endif %}
-{%- endfor %}
{%- endmacro %}
{%- macro MACRO_CHECK_SALT_VERSION_SERVICES_ON_CFG() %}
diff --git a/tcp_tests/tests/system/test_mcp_update.py b/tcp_tests/tests/system/test_mcp_update.py
index 51d29d0..6556e5f 100644
--- a/tcp_tests/tests/system/test_mcp_update.py
+++ b/tcp_tests/tests/system/test_mcp_update.py
@@ -109,6 +109,34 @@
salt_actions.enforce_state("I@jenkins:client", "jenkins.client")
+@pytest.mark.parametrize("_", [settings.ENV_NAME])
+@pytest.fixture
+def wa_for_stacklight_monitor_hostname_prod37068(reclass_actions,
+ salt_actions):
+ stacklight_monitor_hostname = salt_actions.cmd_run(
+ "I@salt:master",
+ ". /root/venv-reclass-tools/bin/activate; "
+ "reclass-tools get-key "
+ "parameters.dogtag.server.stacklight_monitor_hostname "
+ "/srv/salt/reclass/classes/cluster/{env_name}"
+ "/openstack/barbican.yml".format(
+ env_name=settings.ENV_NAME))
+
+ if stacklight_monitor_hostname != '':
+ reclass_actions.add_key(
+ "parameters.dogtag.server.stacklight_monitor_hostname",
+ "mon",
+ "cluster/*/openstack/barbican.yml"
+ )
+ salt_actions.cmd_run("cfg01*",
+ "cd /srv/salt/reclass; git add -u && "
+ "git commit --allow-empty "
+ "-m 'fix stacklight_monitor_hostname "
+ "for dogtag'")
+ else:
+ LOG.info("Skipping WA stacklight_monitor_hostname fix prod37068")
+
+
@pytest.fixture
def wa_for_galera_clustercheck_password_prod35705(reclass_actions,
salt_actions):
@@ -117,13 +145,13 @@
"_param:galera_clustercheck_password")[0]:
reclass_actions.add_key(
"parameters._param.galera_clustercheck_password",
- "a"*32,
+ "a" * 32,
"cluster/*/infra/secrets.yml")
salt_actions.run_state(tgt, "saltutil.refresh_pillar")
salt_actions.enforce_state(tgt, "galera")
salt_actions.enforce_state(tgt, "haproxy")
reclass_actions.commit(
- "[from TCP-QA] Add galera_clustercheck_password")
+ "[from TCP-QA] Add galera_clustercheck_password")
else:
LOG.info("Skipping WA for Galera Clustercheck Password")
@@ -154,12 +182,11 @@
@pytest.fixture
def wa_for_alerta_password_prod35958(reclass_actions,
salt_actions):
-
if not salt_actions.get_pillar("I@prometheus:alerta",
"_param:alerta_admin_api_key_generated")[0]:
reclass_actions.add_key(
"parameters._param.alerta_admin_api_key_generated",
- "a"*32,
+ "a" * 32,
"cluster/*/infra/secrets.yml")
reclass_actions.add_key(
"parameters._param.alerta_admin_key",
@@ -259,6 +286,7 @@
"wa_for_galera_clustercheck_password_prod35705",
"wa_for_alerta_password_prod35958",
"wa_redis_server_version_3_prod36960",
+ "wa_for_stacklight_monitor_hostname_prod37068",
"switch_to_staging_updates_mirantis_repo")
class TestUpdateMcpCluster(object):
"""
@@ -270,16 +298,17 @@
@pytest.mark.parametrize("_", [settings.ENV_NAME])
@pytest.mark.run_mcp_update
def test_update_drivetrain(self, salt_actions, drivetrain_actions,
- show_step, update_to_tag, _):
+ show_step, update_to_tag, reclass_actions, _):
"""Updating DriveTrain component to release/proposed/2019.2.0 version
Scenario:
1. Add workaround for PROD-32751
2. Run job git-mirror-downstream-mk-pipelines
3. Run job git-mirror-downstream-pipeline-library
- 4. Run job pre-upgrade-verify
- 5. If jobs are passed then start 'Deploy - upgrade MCP Drivetrain'
- 6. Wait till salt-master completes all async jobs
+ 4. Switch mcp-ci/pipeline-library to new version
+ 5. Run job pre-upgrade-verify
+ 6. If jobs are passed then start 'Deploy - upgrade MCP Drivetrain'
+ 7. Wait till salt-master completes all async jobs
Duration: ~70 min
"""
@@ -321,8 +350,35 @@
assert job_result == 'SUCCESS', job_description
- # ############### Start 'Pre-upgrade verify' job (since 2019.2.17)#####
+ # ############### Switch mcp-ci/pipeline-library to new version########
show_step(4)
+
+ # https://docs.mirantis.com/mcp/q4-18/mcp-operations-guide/
+ # update-upgrade/minor-update/preupdate-drivetrain-verify.html
+ actual_piplines_version = salt.get_single_pillar(
+ "I@salt:master",
+ "_param:jenkins_pipelines_branch")
+ LOG.info(actual_piplines_version)
+ if update_to_tag is not None:
+ if actual_piplines_version != "refs/tags/{}".format(
+ update_to_tag):
+ reclass_actions.add_key(
+ 'parameters._param.jenkins_pipelines_branch',
+ "refs/tags/{}".format(
+ update_to_tag),
+ 'cluster/*/infra/init.yml')
+ salt.run_state("I@jenkins:client and not I@salt:master",
+ "saltutil.refresh_pillar")
+ salt_actions.enforce_state("I@jenkins:client:lib",
+ "jenkins.client.lib")
+ salt_actions.cmd_run("cfg01*",
+ "cd /srv/salt/reclass; git add -u && "
+ "git commit --allow-empty "
+ "-m 'use workaround for PROD-37068 "
+ "missing'")
+
+ # ############### Start 'Pre-upgrade verify' job (since 2019.2.17)#####
+ show_step(5)
job_name = 'pre-upgrade-verify'
job_parameters = {}
@@ -337,7 +393,7 @@
assert job_result == 'SUCCESS', job_description
# ################### Start 'Deploy - upgrade MCP Drivetrain' job #####
- show_step(5)
+ show_step(6)
job_name = 'upgrade-mcp-release'
job_parameters = {
@@ -346,10 +402,10 @@
update_to_tag or 'release/proposed/2019.2.0',
'TARGET_MCP_VERSION': update_to_tag or '2019.2.0',
"DRIVE_TRAIN_PARAMS": {
- "OS_DIST_UPGRADE": True,
- "OS_UPGRADE": True,
- "BATCH_SIZE": 10
- }
+ "OS_DIST_UPGRADE": True,
+ "OS_UPGRADE": True,
+ "BATCH_SIZE": 10
+ }
}
job_result, job_description = dt.start_job_on_jenkins(
job_name=job_name,
@@ -359,8 +415,8 @@
assert job_result == 'SUCCESS', job_description
# ############### Wait till salt-master completes all async jobs #####
- show_step(5)
- salt.wait_jobs_completed(timeout=20*60)
+ show_step(7)
+ salt.wait_jobs_completed(timeout=20 * 60)
@pytest.mark.grab_versions
@pytest.mark.parametrize("_", [settings.ENV_NAME])
@@ -450,7 +506,7 @@
show_step(2)
if (update_mysql_to_version == '5.7'):
mysql_class = "cluster.{env_name}.openstack." \
- "database.mysql_version"\
+ "database.mysql_version" \
.format(env_name=settings.ENV_NAME)
reclass.add_class(
mysql_class,
@@ -459,17 +515,17 @@
mysql_class,
"cluster/*/openstack/database/slave.yml")
mysql_path = "cluster/{env_name}" \
- "/openstack/database/mysql_version.yml"\
+ "/openstack/database/mysql_version.yml" \
.format(env_name=settings.ENV_NAME)
galera_mysql_version = '5.7'
if update_mysql_to_version != '5.7':
galera_mysql_version = update_mysql_to_version
mysql_version = "parameters:\n" \
" _param:\n" \
- " galera_mysql_version: {}"\
+ " galera_mysql_version: {}" \
.format(galera_mysql_version)
if not os.path.isfile("/srv/salt/reclass/classes/{path}".format(
- path=mysql_path)):
+ path=mysql_path)):
reclass.create_yaml_with_context(mysql_version,
mysql_path)
else:
@@ -701,7 +757,7 @@
ceph_version = salt_actions.get_single_pillar(
"I@salt:master",
- "_param:_param:ceph_version")
+ "_param:ceph_version")
if ceph_version == "nautilus":
wa_cve_2021_20288_global_id_reclaim(reclass, salt)
else:
diff --git a/tcp_tests/tests/system/test_upgrade_pike_queens.py b/tcp_tests/tests/system/test_upgrade_pike_queens.py
index aa6997e..0b35add 100644
--- a/tcp_tests/tests/system/test_upgrade_pike_queens.py
+++ b/tcp_tests/tests/system/test_upgrade_pike_queens.py
@@ -321,4 +321,4 @@
job_result, job_description = dt.start_job_on_jenkins(
job_name=job_name,
job_parameters=job_parameters)
- assert job_result == 'SUCCESS', job_description
+ assert job_result in ['SUCCESS', 'UNSTABLE'], job_description