{% from 'cookied-bm-mcp-pike-k8s-contrail/underlay.yaml' import HOSTNAME_CFG01 with context %}
{% from 'cookied-bm-mcp-pike-k8s-contrail/underlay.yaml' import HOSTNAME_CTL01 with context %}

{%- macro MACRO_CHECK_SYSTEMCTL() %}
{#######################################}
- description: Check systemctl on compute
  cmd: |
    set -ex;
    salt 'cmp*' cmd.run "systemctl --version";
    salt 'cmp*' cmd.run "journalctl -u dbus";
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 1}
  skip_fail: true
{%- endmacro %}

- description:  Install keepalived on primary controller
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@keepalived:cluster and *01*' state.sls keepalived
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false

- description:  Install keepalived
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@keepalived:cluster' state.sls keepalived
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false

- description: Install haproxy
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@haproxy:proxy' state.sls haproxy
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Install etcd
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@etcd:server' state.sls etcd.server.service
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 30}
  skip_fail: false

- description: Install certs
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@etcd:server' state.sls salt.minion -b 1
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

# Install opencontrail database services
- description: Install opencontrail database services for 01
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:database and *01*' state.sls opencontrail.database
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 30}
  skip_fail: false

- description: Install opencontrail database services
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:database' state.sls opencontrail.database
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 30}
  skip_fail: false

# Install opencontrail control services
- description: Install opencontrail services for 01
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:control and *01*' state.sls opencontrail exclude=opencontrail.client
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 30}
  skip_fail: false

- description: Install opencontrail services
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:control' state.sls opencontrail exclude=opencontrail.client
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 30}
  skip_fail: false

- description: Install docker host
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@kubernetes:master' state.sls salt.minion.cert
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 30}
  skip_fail: false

- description: Install docker host
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@docker:host' state.sls docker.host
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 30}
  skip_fail: false

  #- description: Configure OpenContrail as an add-on for Kubernetes
  #  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
  #    -C 'I@kubernetes:master' state.sls kubernetes.master.kube-addons
  #  node_name: {{ HOSTNAME_CFG01 }}
  #  retry: {count: 1, delay: 5}
  #  skip_fail: false

- description: Install Kubernetes components
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@kubernetes:master' state.sls kubernetes.pool
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 30}
  skip_fail: false

- description: "Run k8s master at *01* to simplify namespaces creation"
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
     -C 'I@kubernetes:master and *01*' state.sls kubernetes.master exclude=kubernetes.master.setup,kubernetes.master.kube-addons
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 15}
  skip_fail: false

- description: Run k8s without master.setup
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
     -C 'I@kubernetes:master' state.sls kubernetes exclude=kubernetes.master.setup,kubernetes.master.kube-addons
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 15}
  skip_fail: false

- description: Check the etcd health
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
     -C 'I@etcd:server' cmd.run '. /var/lib/etcd/configenv && etcdctl cluster-health'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

{{ MACRO_CHECK_SYSTEMCTL() }}

- description: Run Kubernetes master setup
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
     -C 'I@kubernetes:master and *01*' state.sls kubernetes.master.setup
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Restart Kubelet
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@kubernetes:master' service.restart 'kubelet'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Waiting for contrail-containers up. opencontrail.client state should be run only after that
  cmd: |
    sleep 30;
    total_pods=`kubectl get pods --all-namespaces | awk '/opencontrail/ {print $3}' | cut -d "/" -f2`
    for i in `seq 1 10`; do
      ready_pods=`kubectl get pods --all-namespaces | awk '/opencontrail/ {print $3}' | cut -d "/" -f1`
      if [ "$ready_pods" == "$total_pods" ];then
        echo "containers are ready. Going to the next step"
        break
      elif [ "$i" -ne "10" ]; then
        echo "Opencontrail containers is not ready. $ready_pods from $total_pods is ready."
        sleep 60
        continue
      else
        echo "Failed to up contrail containers in 10 minutes"
        exit 1
      fi
    done
  node_name: {{ HOSTNAME_CTL01 }}
  retry: {count: 1, delay: 5}
  skip_fail: true

- description: Check all pods
  cmd: |
     salt 'ctl*' cmd.run "kubectl -o wide get pods --all-namespaces";
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

# Install contrail computes
- description: Set up the OpenContrail resources
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:database:id:1' state.sls opencontrail.client
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 60}
  skip_fail: false

- description: Apply opencontrail.client on contrail computes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:compute' state.sls opencontrail exclude=opencontrail.client
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

- description: Reboot contrail computes
  cmd: |
    salt --async -C 'I@opencontrail:compute' system.reboot;
    sleep 450;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 30}
  skip_fail: true

- description: Apply opencontrail.client on contrail computes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:compute' state.sls opencontrail.client
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

- description: Apply opencontrail.client on contrail computes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:compute' state.sls opencontrail
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Refresh pillars on cmp*
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'cmp*' saltutil.refresh_pillar
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Sync all on contrail computes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@opencontrail:compute' saltutil.sync_all
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Install docker host
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@kubernetes:master' state.sls salt.minion.cert
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 30}
  skip_fail: false

- description: Install Kubernetes components
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@kubernetes:pool and not I@kubernetes:master' state.sls kubernetes.pool
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 30}
  skip_fail: false

- description: Restart Kubelet
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@kubernetes:pool and not I@kubernetes:master' service.restart 'kubelet'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Configure OpenContrail as an add-on for Kubernetes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@kubernetes:master' state.sls kubernetes.master.kube-addons
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Renew hosts file on a whole cluster
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C '*' state.sls linux.network.host;
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Final check all pods
  cmd: |
     sleep 60;
     salt 'ctl*' cmd.run "kubectl -o wide get pods --all-namespaces";
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Check contrail status on all pods
  cmd: |
     pods=`kubectl get pods --all-namespaces | awk '/opencontrail/ {print $2}'`
     for i in $pods; do
       kubectl exec $i -c opencontrail-controller -n kube-system contrail-status;
       kubectl exec $i -c opencontrail-analytics -n kube-system contrail-status;
       kubectl exec $i -c opencontrail-analyticsdb -n kube-system contrail-status;
     done
  node_name: {{ HOSTNAME_CTL01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false