{% from 'virtual-mcp-ocata-cicd/underlay.yaml' import HOSTNAME_CFG01 with context %}

# Pattern that helps salt to select CICD nodes
{% set CICD_NODES_SELECTOR = os_env('CICD_NODES_SELECTOR','cid*') %}

# Install OSS: Operational Support System Tools

# Keepalived
#-----------
- description: Install keepalived on {{ CICD_NODES_SELECTOR }} nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@haproxy:proxy and {{ CICD_NODES_SELECTOR }}' state.sls keepalived
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false

- description: Install haproxy on {{ CICD_NODES_SELECTOR }} nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@haproxy:proxy and {{ CICD_NODES_SELECTOR }}' state.sls haproxy
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false

- description: Check the VIP
  cmd: |
    CICD_CONTROL_ADDRESS=`salt-call --out=newline_values_only pillar.get _param:cicd_control_address`;
    echo "_param:cicd_control_address (vip): ${CICD_CONTROL_ADDRESS}";
    salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@haproxy:proxy and {{ CICD_NODES_SELECTOR }}' cmd.run "ip a | grep ${CICD_CONTROL_ADDRESS}" | grep -B1 ${CICD_CONTROL_ADDRESS}
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

# Glusterfs
#-----------

- description: Prepare glusterfs service
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@glusterfs:server and {{ CICD_NODES_SELECTOR }}' state.sls glusterfs.server.service
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Setup glusterfs server
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@glusterfs:server and {{ CICD_NODES_SELECTOR }}' state.sls glusterfs.server.setup -b 1
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 2, delay: 5}
  skip_fail: false

- description: Setup glusterfs client
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@glusterfs:client and {{ CICD_NODES_SELECTOR }}' state.sls glusterfs.client
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Check the gluster status
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False
    -C 'I@glusterfs:client and {{ CICD_NODES_SELECTOR }}*' cmd.run 'gluster peer status; gluster volume status' -b 1
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

# Setup Docker Swarm
#-------------------

- description: "Workaround: create /var/lib/jenkins to get Jenkins slaves working"
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@jenkins:client' cmd.run 'mkdir -p /var/lib/jenkins'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false

- description: Prepare Docker host
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:host and {{ CICD_NODES_SELECTOR }}' state.sls docker.host
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Install Docker Swarm master
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ CICD_NODES_SELECTOR }}' state.sls docker.swarm
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Setup salt-minion on docker swarm master
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ CICD_NODES_SELECTOR }}' state.sls salt &&
    salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ CICD_NODES_SELECTOR }}' mine.flush &&
    salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ CICD_NODES_SELECTOR }}' mine.update
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 15}
  skip_fail: false

- description: Install Docker Swarm on other nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm and {{ CICD_NODES_SELECTOR }}' state.sls docker.swarm
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Show Docker Swarm nodes
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master and {{ CICD_NODES_SELECTOR }}' cmd.run 'docker node ls'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

# Configure OSS services
#-----------------------

- description: Setup devops portal
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@devops_portal:config:enabled' state.sls devops_portal.config
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Setup Rundeck server
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@rundeck:server' state.sls rundeck.server
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

# Deploy Docker services
#-----------------------

# Original comment from pipeline:  XXX: for some weird unknown reason, refresh_pillar is required to execute here

- description: "Workaround from the pipeline: XXX: for some weird unknown reason, refresh_pillar is required to execute here"
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@aptly:publisher' saltutil.refresh_pillar
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: "Workaround from the pipeline: We need /etc/aptly-publisher.yaml to be present before services are deployed. [dd: there were issues when /etc/aptly-publisher.yaml becomes a directory, so this step should be considered]"
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@aptly:publisher' state.sls aptly.publisher
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false

- description: Install Docker client
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@docker:swarm:role:master' state.sls docker.client
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 5}
  skip_fail: false

- description: "Workaround from the pipeline: sync all salt objects"
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' saltutil.sync_all
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 5}
  skip_fail: false


# Aptly
#------

- description: Install Aptly and check it's API
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@aptly:server' state.sls aptly &&
    timeout 90 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@aptly:server' cmd.run
      'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
       while true; do curl -sf http://${CICD_CONTROL_ADDRESS}:8084/api/version  && break; sleep 2; done'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# OpenLDAP
#---------

- description: Install LDAP and check it's availability
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@openldap:client' state.sls openldap &&
    timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@openldap:client' cmd.run
      'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
       while true; do curl -sf ldap://${CICD_CONTROL_ADDRESS} >/dev/null && break; sleep 2; done' &&
    sleep 20
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# Gerrit
#-------

- description: Install Gerrit and check it's availability
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@gerrit:client' state.sls gerrit &&
    timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@gerrit:client' cmd.run
      'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
       while true; do curl -sf http://${CICD_CONTROL_ADDRESS}:8080 >/dev/null && break; sleep 2; done'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# Jenkins
#--------


- description: Install Jenkins and check it's availability
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@jenkins:client' state.sls jenkins
  # TODO: Jenkins requires authenticate token. Remove -f from curl for now.
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@jenkins:client' state.sls jenkins &&
    timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@jenkins:client' cmd.run
      'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
       while true; do curl -s http://${CICD_CONTROL_ADDRESS}:8081 >/dev/null && break; sleep 2; done'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# Postgres
#---------

- description: Install PostgreSQL and create databases
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@postgresql:client' state.sls postgresql.client -b 1&&
    timeout 300 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@postgresql:client' cmd.run
      'while true; do if docker service logs postgresql_db | grep -q "ready to accept"; then break; else sleep 5; fi; done'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# Pushkin
#--------

- description: Check Pushkin API availability
  cmd: timeout 60 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@postgresql:client' cmd.run
    'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
     while true; do curl -s http://${CICD_CONTROL_ADDRESS}:8887 && break; sleep 2; done'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false

# Rundeck
#--------

- description: Install Rundeck and check availability
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@rundeck:client' state.sls rundeck.client &&
    timeout 30 salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@rundeck:client' cmd.run
      'export CICD_CONTROL_ADDRESS=$(salt-call --out=newline_values_only pillar.get _param:cicd_control_address);
       while true; do curl -sf http://${CICD_CONTROL_ADDRESS}:4440 && break; sleep 2; done'
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# Generate docs
#--------------

- description: Install sphinx (may fail depending on the model)
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@sphinx:server' state.sls sphinx
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: true

- description: Generate docs (may fail depending on the model)
  cmd: salt-run state.orchestrate sphinx.orch.generate_doc
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: true

- description: Install nginx
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False -C 'I@nginx:server' state.sls nginx
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 3, delay: 10}
  skip_fail: false

# Final checks
#-------------

- description: Check for system services in failed state
  cmd: salt --hard-crash --state-output=mixed --state-verbose=False '*' cmd.run "systemctl --failed | grep -E 'loaded[ \t]+failed' && echo 'Command execution failed'  || true"
  node_name: {{ HOSTNAME_CFG01 }}
  retry: {count: 1, delay: 10}
  skip_fail: false