Fix missing 'apt_mk_version' parameter for cookiecutter-based model
Change-Id: Ia8ea723c461d5978590f611d504caa735a0a498b
Reviewed-on: https://review.gerrithub.io/379531
Reviewed-by: Dennis Dmitriev <dis.xcom@gmail.com>
Tested-by: Dennis Dmitriev <dis.xcom@gmail.com>
diff --git "a/tcp_tests/environment/environment_template/\173\173 cookiecutter._env_name \175\175/overrides.yml" "b/tcp_tests/environment/environment_template/\173\173 cookiecutter._env_name \175\175/overrides.yml"
index 5a2e28e..f35b469 100644
--- "a/tcp_tests/environment/environment_template/\173\173 cookiecutter._env_name \175\175/overrides.yml"
+++ "b/tcp_tests/environment/environment_template/\173\173 cookiecutter._env_name \175\175/overrides.yml"
@@ -8,6 +8,8 @@
# _param:
# foo: bar
parameters:
+ _param:
+ apt_mk_version: stable # This will be replaced with REPOSITORY_SUITE env variable
openssh:
server:
password_auth: true
diff --git a/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/salt.yaml b/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/salt.yaml
index 520afbf..fe37cfc 100644
--- a/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/salt.yaml
+++ b/tcp_tests/templates/cookied-mcp-ocata-dop-sl2/salt.yaml
@@ -15,124 +15,7 @@
{{ SHARED.MACRO_INSTALL_SALT_MASTER() }}
-- description: "[EXPERIMENTAL] Upload 'environment' to {{ HOSTNAME_CFG01 }}"
- upload:
- local_path: {{ config.salt_deploy.environment_template_dir }}
- remote_path: /tmp/environment/
- node_name: {{ HOSTNAME_CFG01 }}
- skip_fail: false
-
-- description: "[EXPERIMENTAL] Upload cookiecutter-templates context to {{ HOSTNAME_CFG01 }}"
- upload:
- local_path: {{ config.salt_deploy.templates_dir }}{{ LAB_CONFIG_NAME }}/
- local_filename: salt-context-cicd_oss.yaml
- remote_path: /tmp/environment/
- node_name: {{ HOSTNAME_CFG01 }}
-
-- description: "[EXPERIMENTAL] Upload environment inventory to {{ HOSTNAME_CFG01 }}"
- upload:
- local_path: {{ config.salt_deploy.templates_dir }}{{ LAB_CONFIG_NAME }}/
- local_filename: salt-context-environment.yaml
- remote_path: /tmp/environment/
- node_name: {{ HOSTNAME_CFG01 }}
-
-{#{ SHARED.MACRO_CLONE_RECLASS_MODELS() }#}
-- description: Create cluster model from cookiecutter templates
- cmd: |
- set -e;
- pip install cookiecutter
- export GIT_SSL_NO_VERIFY=true; git clone https://gerrit.mcp.mirantis.net/mk/cookiecutter-templates /tmp/cookiecutter-templates
- mkdir -p /srv/salt/reclass/classes/cluster/
- mkdir -p /srv/salt/reclass/classes/system/
- mkdir -p /srv/salt/reclass/classes/service/
- mkdir -p /srv/salt/reclass/nodes/_generated
-
- # Override some context parameters
- sed -i 's/cluster_name:.*/cluster_name: {{ LAB_CONFIG_NAME }}/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/cluster_domain:.*/cluster_domain: {{ DOMAIN_NAME }}/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/control_vlan:.*/control_vlan: \"2416\"/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/tenant_vlan:.*/tenant_vlan: \"2417\"/g' {{ CLUSTER_CONTEXT_PATH }}
-
- # Temporary workaround (with hardcoded address .90 -> .15) of bug https://mirantis.jira.com/browse/PROD-14377
- sed -i 's/salt_master_address:.*/salt_master_address: {{ SHARED.IPV4_NET_CONTROL_PREFIX }}.15/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/salt_master_management_address:.*/salt_master_management_address: {{ SHARED.IPV4_NET_ADMIN_PREFIX }}.15/g' {{ CLUSTER_CONTEXT_PATH }}
-
- # Replace firstly to an intermediate value to avoid intersection between
- # already replaced and replacing networks.
- # For example, if generated IPV4_NET_ADMIN_PREFIX=10.16.0 , then there is a risk of replacing twice:
- # 192.168.10 -> 10.16.0 (generated network for admin)
- # 10.16.0 -> <external network>
- # So let's replace constant networks to the keywords, and then keywords to the desired networks.
- sed -i 's/10\.167\.5/==IPV4_NET_ADMIN_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/10\.167\.4/==IPV4_NET_CONTROL_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/10\.167\.6/==IPV4_NET_TENANT_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/172\.17\.16\./==IPV4_NET_EXTERNAL_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
-
- sed -i 's/==IPV4_NET_ADMIN_PREFIX==/{{ SHARED.IPV4_NET_ADMIN_PREFIX }}/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/==IPV4_NET_CONTROL_PREFIX==/{{ SHARED.IPV4_NET_CONTROL_PREFIX }}/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/==IPV4_NET_TENANT_PREFIX==/{{ SHARED.IPV4_NET_TENANT_PREFIX }}/g' {{ CLUSTER_CONTEXT_PATH }}
- sed -i 's/==IPV4_NET_EXTERNAL_PREFIX==/{{ SHARED.IPV4_NET_EXTERNAL_PREFIX }}./g' {{ CLUSTER_CONTEXT_PATH }}
-
- for i in $(ls /tmp/cookiecutter-templates/cluster_product/); do
- python /tmp/cookiecutter-templates/generate.py \
- --template /tmp/cookiecutter-templates/cluster_product/$i \
- --config-file {{ CLUSTER_CONTEXT_PATH }} \
- --output-dir /srv/salt/reclass/classes/cluster/;
- done
-
- export GIT_SSL_NO_VERIFY=true; git clone https://gerrit.mcp.mirantis.net/salt-models/reclass-system /srv/salt/reclass/classes/system/
-
- # Create the cfg01 node and disable checkouting the model from remote repository
- cat << 'EOF' >> /srv/salt/reclass/nodes/_generated/{{ HOSTNAME_CFG01 }}.yml
- classes:
- - system.openssh.server.team.all
- - cluster.{{ LAB_CONFIG_NAME }}.infra.config
- - environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}.reclass_datasource_local
- - environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}
-
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 1}
- skip_fail: false
-
-- description: "[EXPERIMENTAL] Remove linux.network.interface object from the cluster/system models and use fixed 'environment' model instead"
- cmd: |
- set -e;
- apt-get -y install python-virtualenv python-pip build-essential python-dev libssl-dev;
- pip install git+https://github.com/dis-xcom/reclass-tools;
- reclass-tools del-key parameters.linux.network.interface /srv/salt/reclass/classes/cluster/;
- reclass-tools del-key parameters.linux.network.interface /srv/salt/reclass/classes/system/;
- reclass-tools del-key parameters.linux.network.interface /usr/share/salt-formulas/reclass/;
-
- #if ! reclass-tools get-key 'classes' /srv/salt/reclass/nodes/{{ HOSTNAME_CFG01 }}.yml | grep -q "environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}$"; then
- # reclass-tools add-key 'classes' 'environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}' /srv/salt/reclass/nodes/{{ HOSTNAME_CFG01 }}.yml --merge ;
- #fi;
-
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
- skip_fail: false
-
-- description: "Workaround for PROD-14756: all roles must use service.keepalived.cluster.single with the default 'VIP' instance"
- cmd: |
- set -e;
- find /srv/salt/reclass/classes/cluster/ -type f -exec sed -i 's/system.keepalived.*/service.keepalived.cluster.single/g' {} +
- find /srv/salt/reclass/classes/system/ -type f -exec sed -i 's/system.keepalived.*/service.keepalived.cluster.single/g' {} +
-
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
- skip_fail: false
-
-- description: "[EXPERIMENTAL] Create environment model for virtual environment"
- cmd: |
- set -e;
- ln -s '/tmp/environment/environment_template/{{ '{# interfaces #}' }}' '/tmp/environment/environment_template/{{ '{{ cookiecutter._env_name }}' }}/';
- ln -s '/tmp/environment/environment_template/{{ '{# roles #}' }}' '/tmp/environment/environment_template/{{ '{{ cookiecutter._env_name }}' }}/';
- reclass-tools render --template-dir /tmp/environment/environment_template/ \
- --output-dir /srv/salt/reclass/classes/environment/ \
- --context {{ ENVIRONMENT_CONTEXT_PATH }} \
- --env-name {{ ENVIRONMENT_MODEL_INVENTORY_NAME }}
- node_name: {{ HOSTNAME_CFG01 }}
- retry: {count: 1, delay: 5}
- skip_fail: false
+{{ SHARED.MACRO_GENERATE_COOKIECUTTER_MODELS() }}
{{ SHARED.MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='"linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "powerdns" "glusterfs" "xtrabackup" "maas"') }}
diff --git a/tcp_tests/templates/shared-salt.yaml b/tcp_tests/templates/shared-salt.yaml
index 9065138..a5680a6 100644
--- a/tcp_tests/templates/shared-salt.yaml
+++ b/tcp_tests/templates/shared-salt.yaml
@@ -82,6 +82,8 @@
{%- macro MACRO_CLONE_RECLASS_MODELS(IS_CONTRAIL_LAB=false) %}
{############################################################}
+{# Creates a 'cluster' model from cookiecutter-templates and 'environment' model from uploaded template #}
+
- description: Clone reclass models with submodules
cmd: |
set -e;
@@ -153,6 +155,157 @@
{%- endmacro %}
+{%- macro MACRO_GENERATE_COOKIECUTTER_MODELS(IS_CONTRAIL_LAB=false) %}
+{############################################################}
+- description: "[EXPERIMENTAL] Upload 'environment' to {{ HOSTNAME_CFG01 }}"
+ upload:
+ local_path: {{ config.salt_deploy.environment_template_dir }}
+ remote_path: /tmp/environment/
+ node_name: {{ HOSTNAME_CFG01 }}
+ skip_fail: false
+
+- description: "[EXPERIMENTAL] Upload cookiecutter-templates context to {{ HOSTNAME_CFG01 }}"
+ upload:
+ local_path: {{ config.salt_deploy.templates_dir }}{{ LAB_CONFIG_NAME }}/
+ local_filename: salt-context-cicd_oss.yaml
+ remote_path: /tmp/environment/
+ node_name: {{ HOSTNAME_CFG01 }}
+
+- description: "[EXPERIMENTAL] Upload environment inventory to {{ HOSTNAME_CFG01 }}"
+ upload:
+ local_path: {{ config.salt_deploy.templates_dir }}{{ LAB_CONFIG_NAME }}/
+ local_filename: salt-context-environment.yaml
+ remote_path: /tmp/environment/
+ node_name: {{ HOSTNAME_CFG01 }}
+
+- description: Create cluster model from cookiecutter templates
+ cmd: |
+ set -e;
+ pip install cookiecutter
+ export GIT_SSL_NO_VERIFY=true; git clone https://gerrit.mcp.mirantis.net/mk/cookiecutter-templates /tmp/cookiecutter-templates
+ mkdir -p /srv/salt/reclass/classes/cluster/
+ mkdir -p /srv/salt/reclass/classes/system/
+ mkdir -p /srv/salt/reclass/classes/service/
+ mkdir -p /srv/salt/reclass/nodes/_generated
+
+ # Override some context parameters
+ sed -i 's/cluster_name:.*/cluster_name: {{ LAB_CONFIG_NAME }}/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/cluster_domain:.*/cluster_domain: {{ DOMAIN_NAME }}/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/control_vlan:.*/control_vlan: \"2416\"/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/tenant_vlan:.*/tenant_vlan: \"2417\"/g' {{ CLUSTER_CONTEXT_PATH }}
+
+ # Temporary workaround (with hardcoded address .90 -> .15) of bug https://mirantis.jira.com/browse/PROD-14377
+ sed -i 's/salt_master_address:.*/salt_master_address: {{ SHARED.IPV4_NET_CONTROL_PREFIX }}.15/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/salt_master_management_address:.*/salt_master_management_address: {{ SHARED.IPV4_NET_ADMIN_PREFIX }}.15/g' {{ CLUSTER_CONTEXT_PATH }}
+
+ # Replace firstly to an intermediate value to avoid intersection between
+ # already replaced and replacing networks.
+ # For example, if generated IPV4_NET_ADMIN_PREFIX=10.16.0 , then there is a risk of replacing twice:
+ # 192.168.10 -> 10.16.0 (generated network for admin)
+ # 10.16.0 -> <external network>
+ # So let's replace constant networks to the keywords, and then keywords to the desired networks.
+ sed -i 's/10\.167\.5/==IPV4_NET_ADMIN_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/10\.167\.4/==IPV4_NET_CONTROL_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/10\.167\.6/==IPV4_NET_TENANT_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/172\.17\.16\./==IPV4_NET_EXTERNAL_PREFIX==/g' {{ CLUSTER_CONTEXT_PATH }}
+
+ sed -i 's/==IPV4_NET_ADMIN_PREFIX==/{{ SHARED.IPV4_NET_ADMIN_PREFIX }}/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/==IPV4_NET_CONTROL_PREFIX==/{{ SHARED.IPV4_NET_CONTROL_PREFIX }}/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/==IPV4_NET_TENANT_PREFIX==/{{ SHARED.IPV4_NET_TENANT_PREFIX }}/g' {{ CLUSTER_CONTEXT_PATH }}
+ sed -i 's/==IPV4_NET_EXTERNAL_PREFIX==/{{ SHARED.IPV4_NET_EXTERNAL_PREFIX }}./g' {{ CLUSTER_CONTEXT_PATH }}
+
+ for i in $(ls /tmp/cookiecutter-templates/cluster_product/); do
+ python /tmp/cookiecutter-templates/generate.py \
+ --template /tmp/cookiecutter-templates/cluster_product/$i \
+ --config-file {{ CLUSTER_CONTEXT_PATH }} \
+ --output-dir /srv/salt/reclass/classes/cluster/;
+ done
+
+ export GIT_SSL_NO_VERIFY=true; git clone https://gerrit.mcp.mirantis.net/salt-models/reclass-system /srv/salt/reclass/classes/system/
+
+ # Create the cfg01 node and disable checkouting the model from remote repository
+ cat << 'EOF' >> /srv/salt/reclass/nodes/_generated/{{ HOSTNAME_CFG01 }}.yml
+ classes:
+ - system.openssh.server.team.all
+ - cluster.{{ LAB_CONFIG_NAME }}.infra.config
+ - environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}.reclass_datasource_local
+ - environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}
+ EOF
+
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 1}
+ skip_fail: false
+
+- description: "[EXPERIMENTAL] Remove linux.network.interface object from the cluster/system models and use fixed 'environment' model instead"
+ cmd: |
+ set -e;
+ apt-get -y install python-virtualenv python-pip build-essential python-dev libssl-dev;
+ pip install git+https://github.com/dis-xcom/reclass-tools;
+ reclass-tools del-key parameters.linux.network.interface /srv/salt/reclass/classes/cluster/;
+ reclass-tools del-key parameters.linux.network.interface /srv/salt/reclass/classes/system/;
+ reclass-tools del-key parameters.linux.network.interface /usr/share/salt-formulas/reclass/;
+
+ #if ! reclass-tools get-key 'classes' /srv/salt/reclass/nodes/{{ HOSTNAME_CFG01 }}.yml | grep -q "environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}$"; then
+ # reclass-tools add-key 'classes' 'environment.{{ ENVIRONMENT_MODEL_INVENTORY_NAME }}' /srv/salt/reclass/nodes/{{ HOSTNAME_CFG01 }}.yml --merge ;
+ #fi;
+
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: "Workaround for PROD-14756: all roles must use service.keepalived.cluster.single with the default 'VIP' instance"
+ cmd: |
+ set -e;
+ find /srv/salt/reclass/classes/cluster/ -type f -exec sed -i 's/system.keepalived.*/service.keepalived.cluster.single/g' {} +
+ find /srv/salt/reclass/classes/system/ -type f -exec sed -i 's/system.keepalived.*/service.keepalived.cluster.single/g' {} +
+
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: "[EXPERIMENTAL] Create environment model for virtual environment"
+ cmd: |
+ set -e;
+ ln -s '/tmp/environment/environment_template/{{ '{# interfaces #}' }}' '/tmp/environment/environment_template/{{ '{{ cookiecutter._env_name }}' }}/';
+ ln -s '/tmp/environment/environment_template/{{ '{# roles #}' }}' '/tmp/environment/environment_template/{{ '{{ cookiecutter._env_name }}' }}/';
+ reclass-tools render --template-dir /tmp/environment/environment_template/ \
+ --output-dir /srv/salt/reclass/classes/environment/ \
+ --context {{ ENVIRONMENT_CONTEXT_PATH }} \
+ --env-name {{ ENVIRONMENT_MODEL_INVENTORY_NAME }}
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 5}
+ skip_fail: false
+
+- description: Modify generated model and reclass-system if necessary
+ cmd: |
+ set -e;
+ {%- if SALT_MODELS_SYSTEM_COMMIT != '' %}
+ pushd /srv/salt/reclass/classes/system/
+ git checkout {{ SALT_MODELS_SYSTEM_COMMIT }} && \
+ popd
+ {%- elif SALT_MODELS_SYSTEM_REF_CHANGE != '' -%}
+ pushd /srv/salt/reclass/classes/system/
+ {%- for item in SALT_MODELS_SYSTEM_REF_CHANGE.split(" ") %}
+ git fetch {{ SALT_MODELS_SYSTEM_REPOSITORY }} {{ item }} && git cherry-pick FETCH_HEAD
+ {%- endfor %}
+ popd
+ {%- endif %}
+
+ export REPLACE_DIRS="/srv/salt/reclass/classes/ /srv/salt/reclass/nodes/"
+ find ${REPLACE_DIRS} -type f -exec sed -i 's/apt_mk_version:.*/apt_mk_version: {{ REPOSITORY_SUITE }}/g' {} +
+
+ {%- if IS_CONTRAIL_LAB %}
+ # vSRX IPs for tcp-qa images have 172.16.10.90 hardcoded
+ find ${REPLACE_DIRS} -type f -exec sed -i 's/opencontrail_router01_address:.*/opencontrail_router01_address: 172.16.10.90/g' {} +
+ find ${REPLACE_DIRS} -type f -exec sed -i 's/infra_config_deploy_address: 1.*/infra_config_deploy_address: {{ IPV4_NET_ADMIN_PREFIX }}.15/g' {} +
+ {%- endif %}
+
+ node_name: {{ HOSTNAME_CFG01 }}
+ retry: {count: 1, delay: 1}
+ skip_fail: false
+{%- endmacro %}
+
+
{%- macro MACRO_CONFIGURE_RECLASS(FORMULA_SERVICES='') %}
{#######################################################}
- description: Configure reclass