tcp_tests/templates/virtual-mcp-ocata-cicd/salt.yaml

{% from 'virtual-mcp-ocata-cicd/underlay.yaml' import HOSTNAME_CFG01 with context %}
{% from 'virtual-mcp-ocata-cicd/underlay.yaml' import REPOSITORY_SUITE with context %}

{% set SALT_MODELS_REPOSITORY = os_env('SALT_MODELS_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/mcp-virtual-lab') %}
{% set SALT_MODELS_BRANCH = os_env('SALT_MODELS_BRANCH','master') %}
{% set SALT_MODELS_COMMIT = os_env('SALT_MODELS_COMMIT','master') %}

# Reference to a patch that should be applied to the model if required, for example: export SALT_MODELS_REF_CHANGE=refs/changes/19/7219/12
{% set SALT_MODELS_REF_CHANGE = os_env('SALT_MODELS_REF_CHANGE', '') %}

# Address pools for reclass cluster model are taken in the following order:
# 1. environment variables,
# 2. config.underlay.address_pools based on fuel-devops address pools
#    (see generated '.ini' file after underlay is created),
# 3. defaults
{% set address_pools = config.underlay.address_pools %}
{% set IPV4_NET_ADMIN = os_env('IPV4_NET_ADMIN', address_pools.get('admin-pool01', '192.168.10.0/24')) %}
{% set IPV4_NET_CONTROL = os_env('IPV4_NET_CONTROL', address_pools.get('private-pool01', '172.16.10.0/24')) %}
{% set IPV4_NET_TENANT = os_env('IPV4_NET_TENANT', address_pools.get('tenant-pool01', '10.1.0.0/24')) %}
{% set IPV4_NET_EXTERNAL = os_env('IPV4_NET_EXTERNAL', address_pools.get('external-pool01', '10.16.0.0/24')) %}

{% set IPV4_NET_ADMIN_PREFIX = '.'.join(IPV4_NET_ADMIN.split('.')[0:3]) %}
{% set IPV4_NET_CONTROL_PREFIX = '.'.join(IPV4_NET_CONTROL.split('.')[0:3]) %}
{% set IPV4_NET_TENANT_PREFIX = '.'.join(IPV4_NET_TENANT.split('.')[0:3]) %}
{% set IPV4_NET_EXTERNAL_PREFIX = '.'.join(IPV4_NET_EXTERNAL.split('.')[0:3]) %}

# Install salt to the config node

- description: Installing salt master on cfg01
  cmd:  eatmydata apt-get install -y reclass git salt-master
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Configure salt-master on cfg01
  cmd: |
    cat << 'EOF' >> /etc/salt/master.d/master.conf
    file_roots:
      base:
      - /usr/share/salt-formulas/env
    pillar_opts: False
    open_mode: True
    reclass: &reclass
      storage_type: yaml_fs
      inventory_base_uri: /srv/salt/reclass
    ext_pillar:
      - reclass: *reclass
    master_tops:
      reclass: *reclass
    EOF
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Configure GIT settings and certificates
  cmd: touch /root/.git_trusted_certs.pem;
    for server in git.tcpcloud.eu github.com; do
        openssl s_client -showcerts -connect $server:443 </dev/null
        | openssl x509 -outform PEM
        >> /root/.git_trusted_certs.pem;
    done;
    HOME=/root git config --global http.sslCAInfo /root/.git_trusted_certs.pem;
    HOME=/root git config --global user.email "tcp-qa@example.com";
    HOME=/root git config --global user.name "TCP QA";
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false


- description: Clone reclass models with submodules
  cmd: |
    ssh-keyscan -H github.com >> ~/.ssh/known_hosts;
    git clone -b {{ SALT_MODELS_BRANCH }} --recurse-submodules {{ SALT_MODELS_REPOSITORY }} /srv/salt/reclass;
    pushd /srv/salt/reclass && git checkout {{ SALT_MODELS_COMMIT }} && popd;
    {%- if SALT_MODELS_REF_CHANGE != '' %}
    pushd /srv/salt/reclass && git fetch {{ SALT_MODELS_REPOSITORY }} {{ SALT_MODELS_REF_CHANGE }} && git checkout FETCH_HEAD && popd;
    {%- endif %}
    mkdir -p /srv/salt/reclass/classes/service;

    # Replace firstly to an intermediate value to avoid intersection between
    # already replaced and replacing networks.
    # For example, if generated IPV4_NET_ADMIN_PREFIX=10.16.0 , then there is a risk of replacing twice:
    # 192.168.10 -> 10.16.0 (generated network for admin)
    # 10.16.0 -> <external network>
    # So let's replace constant networks to the keywords, and then keywords to the desired networks.
    find /srv/salt/reclass/ -type f -exec sed -i 's/192\.168\.10\./==IPV4_NET_ADMIN_PREFIX==/g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/172\.16\.10\./==IPV4_NET_CONTROL_PREFIX==/g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/10\.1\.0\./==IPV4_NET_TENANT_PREFIX==/g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/10\.16\.0\./==IPV4_NET_EXTERNAL_PREFIX==/g' {} +

    find /srv/salt/reclass/ -type f -exec sed -i 's/==IPV4_NET_ADMIN_PREFIX==/{{ IPV4_NET_ADMIN_PREFIX }}./g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/==IPV4_NET_CONTROL_PREFIX==/{{ IPV4_NET_CONTROL_PREFIX }}./g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/==IPV4_NET_TENANT_PREFIX==/{{ IPV4_NET_TENANT_PREFIX }}./g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/==IPV4_NET_EXTERNAL_PREFIX==/{{ IPV4_NET_EXTERNAL_PREFIX }}./g' {} +

    find /srv/salt/reclass/ -type f -exec sed -i 's/apt_mk_version:.*/apt_mk_version: {{ REPOSITORY_SUITE }}/g' {} +

    # Disable checkouting the model from remote repository
    cat << 'EOF' >> /srv/salt/reclass/nodes/{{ HOSTNAME_CFG01 }}.yml
    # local storage
      reclass:
        storage:
          data_source:
            engine: local
    EOF

    # Show the changes to the console
    cd /srv/salt/reclass/; git diff
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Configure reclass
  cmd: |
    FORMULA_PATH=${FORMULA_PATH:-/usr/share/salt-formulas};
    FORMULA_REPOSITORY=${FORMULA_REPOSITORY:-deb [arch=amd64] http://apt.mirantis.com/xenial {{ REPOSITORY_SUITE }} salt};
    FORMULA_GPG=${FORMULA_GPG:-http://apt.mirantis.com/public.gpg};
    which wget > /dev/null || (apt-get update; apt-get install -y wget);
    echo "${FORMULA_REPOSITORY}" > /etc/apt/sources.list.d/mcp_salt.list;
    wget -O - "${FORMULA_GPG}" | apt-key add -;
    apt-get clean; apt-get update;
    [ ! -d /srv/salt/reclass/classes/service ] && mkdir -p /srv/salt/reclass/classes/service;
    declare -a formula_services=("linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch" "docker" "kibana");
    echo -e "\nInstalling all required salt formulas\n";
    eatmydata apt-get install -y "${formula_services[@]/#/salt-formula-}";
    for formula_service in "${formula_services[@]}"; do
      echo -e "\nLink service metadata for formula ${formula_service} ...\n";
      [ ! -L "/srv/salt/reclass/classes/service/${formula_service}" ] && ln -s ${FORMULA_PATH}/reclass/service/${formula_service} /srv/salt/reclass/classes/service/${formula_service};
    done;
    [ ! -d /srv/salt/env ] && mkdir -p /srv/salt/env;
    [ ! -L /srv/salt/env/prd ] && ln -s ${FORMULA_PATH}/env /srv/salt/env/prd;
    [ ! -d /etc/reclass ] && mkdir /etc/reclass;

    cat << 'EOF' >> /etc/reclass/reclass-config.yml
    storage_type: yaml_fs
    pretty_print: True
    output: yaml
    inventory_base_uri: /srv/salt/reclass
    EOF
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: "*Workaround* remove all cfg01 nodes except {{ HOSTNAME_CFG01 }} to not depend on other clusters in 'reclass --top'"
  cmd: |
    # Remove all other nodes except {{ HOSTNAME_CFG01 }} to not rely on them for 'reclass --top'
    find /srv/salt/reclass/nodes/ -type f -not -name {{ HOSTNAME_CFG01 }}.yml -delete
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Show reclass-salt --top for cfg01 node
  cmd: reclass-salt --top
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Restart salt-master service
  cmd: systemctl restart salt-master;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

{%- for ssh in config.underlay.ssh %}
- description: Configure salt-minion on {{ ssh['node_name'] }}
  cmd: |
    [ ! -d /etc/salt/minion.d ] && mkdir -p /etc/salt/minion.d;
    cat << "EOF" >> /etc/salt/minion.d/minion.conf
    id: {{ ssh['node_name'] }}
    master: {{ config.salt.salt_master_host }}
    EOF
    eatmydata apt-get install -y salt-minion;
    echo "Check for system info and metadata availability ...";
    salt-call --no-color grains.items;
    salt-call --no-color pillar.items;
  node_name: {{ ssh['node_name'] }}
  retry: {count: 1, delay: 1}
  skip_fail: false
{%- endfor %}


- description: Accept salt keys from all the nodes
  cmd: salt-key -A -y
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: true

- description: Configure salt adoptors on cfg01
  cmd: |
    ln -s /usr/lib/python2.7/dist-packages/reclass/adapters/salt.py /usr/local/sbin/reclass-salt;
    chmod +x /usr/lib/python2.7/dist-packages/reclass/adapters/salt.py
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

# Prepare salt services and nodes settings
- description: Run 'linux' formula on cfg01
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@salt:master' state.sls linux;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Run 'openssh' formula on cfg01
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls openssh;
    salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' cmd.run "sed -i 's/PasswordAuthentication no/PasswordAuthentication
    yes/' /etc/ssh/sshd_config && service ssh reload";
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 5}
  skip_fail: false

- description: '*Workaround* of the bug https://mirantis.jira.com/browse/PROD-7962'
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    '*' cmd.run "echo '    StrictHostKeyChecking no' >> /root/.ssh/config"
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Run 'salt.master' formula on cfg01
  cmd: timeout 120 salt --timeout=120 --hard-crash --state-output=mixed --state-verbose=False -C 'I@salt:master' state.sls salt.master;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls reclass
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Refresh pillars on all minions
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' saltutil.refresh_pillar
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Show  reclass-salt --top for all generated nodes
  cmd: reclass-salt --top
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Sync all salt resources
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' saltutil.sync_all
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure linux on master
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@salt:master' state.sls 'linux.system'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure salt.minion on master
  cmd: salt --timeout=120 --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls salt.minion
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

- description: Run state 'salt' on master (for salt.api, etc)
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls salt
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# Bootstrap all nodes

- description: Configure linux on other nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@linux:system' state.sls 'linux'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 5}
  skip_fail: false

- description: Configure openssh on all nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@linux:system and not cfg01*' state.sls openssh;
    salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@linux:system and not cfg01*' cmd.run "sed -i 's/PasswordAuthentication no/PasswordAuthentication
    yes/' /etc/ssh/sshd_config && service ssh reload"
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure salt.minion on other nodes
  cmd: salt --timeout=120 --hard-crash --state-output=mixed --state-verbose=False -C 'I@linux:system and not cfg01*' state.sls salt.minion && sleep 10
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 15}
  skip_fail: false

- description: Check salt minion versions on slaves
  cmd: salt '*' test.version
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

- description: Check salt top states on nodes
  cmd: salt '*' state.show_top
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure ntp and rsyslog on nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@linux:system' state.sls ntp,rsyslog
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false

#- description: Hack gtw node
#  cmd: salt 'gtw*' cmd.run "ip addr del {{ IPV4_NET_CONTROL_PREFIX }}.110/24 dev ens4; ip addr flush dev ens4";
#  node_name: {{ HOSTNAME_CFG01 }}
#  retry: {count: 1, delay: 10}
#  skip_fail: false

#- description: Hack cmp01 node
#  cmd: salt 'cmp01*' cmd.run "ip addr del {{ IPV4_NET_CONTROL_PREFIX }}.105/24 dev ens4; ip addr flush dev ens4";
#  node_name: {{ HOSTNAME_CFG01 }}
#  retry: {count: 1, delay: 10}
#  skip_fail: false

#- description: Hack cmp02 node
#  cmd: salt 'cmp02*' cmd.run "ip addr del {{ IPV4_NET_CONTROL_PREFIX }}.106/24 dev ens4; ip addr flush dev ens4";
#  node_name: {{ HOSTNAME_CFG01 }}
#  retry: {count: 1, delay: 10}
#  skip_fail: false