tcp_tests/templates/virtual-mcp11-k8s-calico/salt.yaml

{% from 'virtual-mcp11-k8s-calico/underlay.yaml' import HOSTNAME_CFG01 with context %}
{% from 'virtual-mcp11-k8s-calico/underlay.yaml' import REPOSITORY_SUITE with context %}

{% set SALT_MODELS_REPOSITORY = os_env('SALT_MODELS_REPOSITORY','https://gerrit.mcp.mirantis.net/salt-models/mcp-virtual-lab') %}
{% set SALT_MODELS_BRANCH = os_env('SALT_MODELS_BRANCH','master') %}
{% set SALT_MODELS_COMMIT = os_env('SALT_MODELS_COMMIT','master') %}

# Address pools for reclass cluster model are taken in the following order:
# 1. environment variables,
# 2. config.underlay.address_pools based on fuel-devops address pools
#    (see generated '.ini' file after underlay is created),
# 3. defaults
{% set address_pools = config.underlay.address_pools %}
{% set IPV4_NET_ADMIN = os_env('IPV4_NET_ADMIN', address_pools.get('admin-pool01', '172.16.10.0/24')) %}
{% set IPV4_NET_CONTROL = os_env('IPV4_NET_CONTROL', address_pools.get('private-pool01', '192.168.10.0/24')) %}

{% set IPV4_NET_ADMIN_PREFIX = '.'.join(IPV4_NET_ADMIN.split('.')[0:3]) %}
{% set IPV4_NET_CONTROL_PREFIX = '.'.join(IPV4_NET_CONTROL.split('.')[0:3]) %}

# Set docker engine version if one exported as environment variables
{% set DOCKER_ENGINE = os_env('DOCKER_ENGINE', "") %}
{% set LAB_CONFIG_NAME = os_env('LAB_CONFIG_NAME', "virtual-mcp11-k8s-calico") %}

# Install salt to the config node
- description: Installing salt master on cfg01
  cmd:  apt-get install -y reclass git; apt-get install -y salt-master
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Configure salt-master on cfg01
  cmd: |
    cat << 'EOF' >> /etc/salt/master.d/master.conf
    file_roots:
      base:
      - /usr/share/salt-formulas/env
    pillar_opts: False
    open_mode: True
    reclass: &reclass
      storage_type: yaml_fs
      inventory_base_uri: /srv/salt/reclass
    ext_pillar:
      - reclass: *reclass
    master_tops:
      reclass: *reclass
    EOF
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Configure GIT settings and certificates
  cmd: touch /root/.git_trusted_certs.pem;
    for server in github.com; do
        openssl s_client -showcerts -connect $server:443 </dev/null
        | openssl x509 -outform PEM
        >> /root/.git_trusted_certs.pem;
    done;
    HOME=/root git config --global http.sslCAInfo /root/.git_trusted_certs.pem;
    HOME=/root git config --global user.email "tcp-qa@example.com";
    HOME=/root git config --global user.name "TCP QA";
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Clone reclass models with submodules
  cmd: |
    ssh-keyscan -H github.com >> ~/.ssh/known_hosts;
    git clone -b {{ SALT_MODELS_BRANCH }} --recurse-submodules {{ SALT_MODELS_REPOSITORY }} /srv/salt/reclass;
    pushd /srv/salt/reclass && git checkout {{ SALT_MODELS_COMMIT }} && popd;
    mkdir -p /srv/salt/reclass/classes/service;

    # Replace firstly to an intermediate value to avoid intersection between
    # already replaced and replacing networks.
    # For example, if generated IPV4_NET_ADMIN_PREFIX=10.16.0 , then there is a risk of replacing twice:
    # 192.168.10 -> 10.16.0 (generated network for admin)
    # 10.16.0 -> <external network>
    # So let's replace constant networks to the keywords, and then keywords to the desired networks.
    find /srv/salt/reclass/ -type f -exec sed -i 's/192\.168\.10\./==IPV4_NET_ADMIN_PREFIX==/g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/172\.16\.10\./==IPV4_NET_CONTROL_PREFIX==/g' {} +

    find /srv/salt/reclass/ -type f -exec sed -i 's/==IPV4_NET_ADMIN_PREFIX==/{{ IPV4_NET_ADMIN_PREFIX }}./g' {} +
    find /srv/salt/reclass/ -type f -exec sed -i 's/==IPV4_NET_CONTROL_PREFIX==/{{ IPV4_NET_CONTROL_PREFIX }}./g' {} +

    find /srv/salt/reclass/ -type f -exec sed -i 's/apt_mk_version:.*/apt_mk_version: {{ REPOSITORY_SUITE }}/g' {} +

    # Disable checkouting the model from remote repository
    cat << 'EOF' >> /srv/salt/reclass/nodes/{{ HOSTNAME_CFG01 }}.yml
    # local storage
    reclass:
      storage:
        data_source:
          engine: local
    EOF

    # Show the changes to the console
    cd /srv/salt/reclass/; git diff
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false


- description: Configure reclass
  cmd: |
    FORMULA_PATH=${FORMULA_PATH:-/usr/share/salt-formulas};
    FORMULA_REPOSITORY=${FORMULA_REPOSITORY:-deb [arch=amd64] http://apt-mk.mirantis.com/xenial {{ REPOSITORY_SUITE }} salt};
    FORMULA_GPG=${FORMULA_GPG:-http://apt-mk.mirantis.com/public.gpg};
    which wget > /dev/null || (apt-get update; apt-get install -y wget);
    echo "${FORMULA_REPOSITORY}" > /etc/apt/sources.list.d/mcp_salt.list;
    wget -O - "${FORMULA_GPG}" | apt-key add -;
    apt-get clean; apt-get update;
    [ ! -d /srv/salt/reclass/classes/service ] && mkdir -p /srv/salt/reclass/classes/service;
    declare -a formula_services=("linux" "reclass" "salt" "openssh" "ntp" "git" "nginx" "collectd" "sensu" "heka" "sphinx" "keystone" "mysql" "grafana" "haproxy" "rsyslog" "horizon" "prometheus" "telegraf" "elasticsearch");
    echo -e "\nInstalling all required salt formulas\n";
    eatmydata apt-get install -y "${formula_services[@]/#/salt-formula-}";
    for formula_service in "${formula_services[@]}"; do
      echo -e "\nLink service metadata for formula ${formula_service} ...\n";
      [ ! -L "/srv/salt/reclass/classes/service/${formula_service}" ] && ln -s ${FORMULA_PATH}/reclass/service/${formula_service} /srv/salt/reclass/classes/service/${formula_service};
    done;
    [ ! -d /srv/salt/env ] && mkdir -p /srv/salt/env;
    [ ! -L /srv/salt/env/prd ] && ln -s ${FORMULA_PATH}/env /srv/salt/env/prd;
    [ ! -d /etc/reclass ] && mkdir /etc/reclass;

    cat << 'EOF' >> /etc/reclass/reclass-config.yml
    storage_type: yaml_fs
    pretty_print: True
    output: yaml
    inventory_base_uri: /srv/salt/reclass
    EOF
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Restart salt-master service
  cmd: |
     systemctl restart salt-master;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

{% for ssh in config.underlay.ssh %}
- description: Configure salt-minion on {{ ssh['node_name'] }}
  cmd: |
    [ ! -d /etc/salt/minion.d ] && mkdir -p /etc/salt/minion.d;
    cat << "EOF" >> /etc/salt/minion.d/minion.conf
    id: {{ ssh['node_name'] }}
    master: {{ config.salt.salt_master_host }}
    EOF
    eatmydata apt-get install -y salt-minion;
    echo "Check for system info and metadata availability ...";
    salt-call --no-color grains.items;
    salt-call --no-color pillar.items;
  node_name: {{ ssh['node_name'] }}
  retry: {count: 1, delay: 1}
  skip_fail: false
{% endfor %}

- description: Accept salt keys from all the nodes
  cmd: salt-key -A -y
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: true

- description: Configure salt adoptors on cfg01
  cmd: |
    ln -s /usr/lib/python2.7/dist-packages/reclass/adapters/salt.py /usr/local/sbin/reclass-salt;
    chmod +x /usr/lib/python2.7/dist-packages/reclass/adapters/salt.py
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

# Prepare salt services and nodes settings
- description: Run 'linux' formula on cfg01
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls linux;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Run 'openssh' formula on cfg01
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls openssh;
    salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' cmd.run "sed -i 's/PasswordAuthentication no/PasswordAuthentication
    yes/' /etc/ssh/sshd_config && service ssh restart";
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 5}
  skip_fail: false

- description: '*Workaround* of the bug https://mirantis.jira.com/browse/PROD-7962'
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    '*' cmd.run "echo '    StrictHostKeyChecking no' >> /root/.ssh/config"
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: false

- description: Run 'salt.master' formula on cfg01
  cmd: timeout 120 salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls salt.master.service;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

- description: Run 'salt' formula on cfg01 with workaround proposed in PROD-10894
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls salt;
    salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' saltutil.sync_all
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 5, delay: 5}
  skip_fail: false

- description: Generate inventory for all the nodes to the /srv/salt/reclass/nodes/_generated
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@salt:master' state.sls reclass
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Refresh pillars on all minions
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' saltutil.refresh_pillar
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Sync all salt resources
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' saltutil.sync_all
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Show reclass-salt --top
  cmd: reclass-salt --top
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Execute salt.minion.cert
  cmd: salt-call --no-color state.sls salt.minion.cert -l info;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

# Change environment configuration before deploy

{% for image_prefix in "CALICOCTL CALICO CALICO_CNI HYPERKUBE NETCHECKER_AGENT NETCHECKER_SERVER".split() %}
- description: Change {{ image_prefix.lower() }} image to custom if necessary
  cmd: |
    image_link={{ os_env('CUSTOM_IMAGE_' + image_prefix, "") }}
    if [ -n "${image_link}" ]; then
        echo "{{ image_prefix.lower() }} image link changed to "${image_link}""
        salt -C 'I@salt:master' reclass.cluster_meta_set kubernetes_{{ image_prefix.lower() }}_image ${image_link}
    else
        echo "Custom {{ image_prefix.lower() }} image link is not set, skipped."
    fi
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false
{% endfor %}

- description: Change docker engine version to custom if necessary
  cmd: |
     FILE="/srv/salt/reclass/classes/cluster/{{ LAB_CONFIG_NAME }}/kubernetes/compute.yml"
     if [ -n "{{ DOCKER_ENGINE }}" ]; then
       echo "Change docker engine version to: {{ DOCKER_ENGINE }}"
       sed "s#docker-engine=.*#docker-engine={{ DOCKER_ENGINE }}#" -i.bak ${FILE}
     else
       echo "Custom docker engine version is not set, skipped."
     fi
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

# Bootstrap all nodes

- description: Configure linux on other nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '* and not
    cfg01* and not mon*' state.sls linux
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure linux on mon nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'mon*' state.sls linux
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

- description: Configure openssh on all nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '* and not
    cfg01*' state.sls openssh;salt --hard-crash --state-output=mixed --state-verbose=False
    -C '* and not cfg*' cmd.run "sed -i 's/PasswordAuthentication no/PasswordAuthentication
    yes/' /etc/ssh/sshd_config && service ssh restart"
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure salt.minion on other nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '* and not
    cfg01*' state.sls salt.minion
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 5}
  skip_fail: false

- description: Check salt minion versions on slaves
  cmd: salt '*' test.version
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Check salt top states on nodes
  cmd: salt '*' state.show_top
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure ntp and rsyslog on nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' state.sls ntp,rsyslog
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false