config-drive/master_config.yaml

#cloud-config
output:
  all: '| tee -a /var/log/cloud-init-output.log /dev/tty0'
write_files:
  - owner: root:root
    path: /etc/cloud/master_environment
    permissions: '0644'
    content: |
      [ -f /etc/cloud/master_environment_override ] && . /etc/cloud/master_environment_override
      export SALT_MASTER_DEPLOY_IP=${SALT_MASTER_DEPLOY_IP:-"172.16.164.15"}
      export SALT_MASTER_MINION_ID=${SALT_MASTER_MINION_ID:-"cfg01.deploy-name.local"}

      export http_proxy=${http_proxy:-""}
      export https_proxy=${https_proxy:-""}

      export MCP_VERSION=${MCP_VERSION:-"stable"}

      export PIPELINES_FROM_ISO=${PIPELINES_FROM_ISO:-"true"}
      export MK_PIPELINES_URL=${MK_PIPELINES_URL:-"https://gerrit.mcp.mirantis.com/mk/mk-pipelines.git"}
      export PIPELINE_LIB_URL=${PIPELINE_LIB_URL:-"https://gerrit.mcp.mirantis.com/mcp-ci/pipeline-library.git"}

      export MCP_SALT_REPO_URL=${MCP_SALT_REPO_URL:-"http://mirror.mirantis.com/"}
      export MCP_SALT_REPO=${MCP_SALT_REPO:-"deb [arch=amd64] $MCP_SALT_REPO_URL/$MCP_VERSION/salt-formulas/xenial xenial main"}

      export ENABLE_MCP_SALT_REPO_UPDATES=${ENABLE_MCP_SALT_REPO_UPDATES:-"true"}
      export MCP_SALT_REPO_UPDATES=${MCP_SALT_REPO_UPDATES:-"deb [arch=amd64] $MCP_SALT_REPO_URL/update/$MCP_VERSION/salt-formulas/xenial xenial main"}
      export MCP_SALT_REPO_KEY=${MCP_SALT_REPO_KEY:-"${MCP_SALT_REPO_URL}/${MCP_VERSION}/salt-formulas/xenial/archive-salt-formulas.key"}

      export FORMULAS=${FORMULAS:-"salt-formula-*"}
      export SALT_OPTS=${SALT_OPTS:-"-l debug -t 30 --retcode-passthrough --no-color"}
      export CFG_BOOTSTRAP_DRIVE_URL=${CFG_BOOTSTRAP_DRIVE_URL:-""}

master_config:
  - &master_config |
    function _post_maas_cfg() {
      # In newer maas version, apache2 disabled, so API avaible on direct service port only
      if [[ ! $(systemctl is-active --quiet apache2) ]] ; then
        sed -i 's/API_SERVER=.*/API_SERVER=localhost:5240/g' /var/lib/maas/.maas_login.sh
      fi
      source /var/lib/maas/.maas_login.sh
      # disable backports for maas enlist pkg repo. Those operation enforce maas
      # to re-create sources.list and drop [source] fetch-definition from it.
      main_arch_id=$(maas ${PROFILE} package-repositories read | jq -r ".[] | select(.name==\"main_archive\") | .id")
      maas ${PROFILE} package-repository update ${main_arch_id} "disabled_pockets=backports" || true
      maas ${PROFILE} package-repository update ${main_arch_id} "disabled_components=multiverse" || true
      maas ${PROFILE} package-repository update ${main_arch_id} "arches=amd64" || true
      # Remove stale notifications, which appear during sources configuration.
      for i in $(maas ${PROFILE} notifications read | jq ".[]| .id"); do
        maas ${PROFILE} notification delete ${i} || true
      done
    }

    function process_formulas(){
      local RECLASS_ROOT=${RECLASS_ROOT:-/srv/salt/reclass/}
      local FORMULAS_PATH=${FORMULAS_PATH:-/usr/share/salt-formulas}

      curl -s ${MCP_SALT_REPO_KEY} | apt-key add -
      echo "${MCP_SALT_REPO}" > /etc/apt/sources.list.d/mcp_salt.list
      if [[ "${ENABLE_MCP_SALT_REPO_UPDATES}" == "true" ]] ; then
        echo "${MCP_SALT_REPO_UPDATES}" >> /etc/apt/sources.list.d/mcp_salt.list
      fi
      apt update && apt install -y "${FORMULAS}"

      [ ! -d ${RECLASS_ROOT}/classes/service ] && mkdir -p ${RECLASS_ROOT}/classes/service
      for formula_service in $(ls /usr/share/salt-formulas/reclass/service/); do
        #Since some salt formula names contain "-" and in symlinks they should contain "_" adding replacement
        formula_service=${formula_service//-/$"_"}
        if [ ! -L "${RECLASS_ROOT}/classes/service/${formula_service}" ]; then
          ln -sf ${FORMULAS_PATH}/reclass/service/${formula_service} ${RECLASS_ROOT}/classes/service/${formula_service}
        fi
      done
    }

    function enable_services(){
      local services="salt-api salt-master salt-minion"
      if [ ! -f /opt/postgresql_in_docker ]; then
        local services="${services} postgresql.service"
      fi
        for s in ${services} ; do
        systemctl enable ${s} || true
        systemctl restart ${s} || true
      done
    }

    function process_maas(){
      maas_cluster_enabled=$(salt-call --out=newline_values_only pillar.get maas:cluster:enabled | tr "[:upper:]" "[:lower:]" )
      _region=$(salt-call --out=newline_values_only pillar.get maas:region:enabled | tr "[:upper:]" "[:lower:]" )
      if ([ -f /opt/postgresql_in_docker ] && ([[ "${maas_cluster_enabled}" == "true" ]] || [[ "${_region}" == "true" ]])); then
        systemctl disable postgresql.service
        wait_for_postgresql
        salt-call ${SALT_OPTS} state.apply postgresql.client
      else
        postgres_enabled=$(salt-call --out=newline_values_only pillar.get postgresql:server:enabled | tr "[:upper:]" "[:lower:]")
        if [[ "${postgres_enabled}" == "true" ]]; then
          salt-call ${SALT_OPTS} state.apply postgresql.server
        fi
      fi

      if [[ "${maas_cluster_enabled}" == "true" ]]; then
        salt-call ${SALT_OPTS} state.apply maas.cluster
      else
        echo "WARNING: maas.cluster skipped!"
      fi
      if [[ "$_region" == "true" ]]; then
        salt-call ${SALT_OPTS} state.apply maas.region
      else
        echo "WARNING: maas.region skipped!"
      fi
      # Do not move it under first cluster-only check!
      if [[ "${maas_cluster_enabled}" == "true" ]]; then
        _post_maas_cfg
      fi
    }

    function wait_for_postgresql() {
      salt_string="salt-call --out=text pillar.get postgresql:client:server:server01:admin"
      local pg_port=$(${salt_string}:port | awk '{print $2}')
      local pg_host=$(${salt_string}:host | awk '{print $2}')
      local wait_time=0
      until [[ $(/usr/bin/pg_isready -h ${pg_host} -p ${pg_port} | awk '{ print $3 }' ) == 'accepting' ]]; do
        if [[ $wait_time -gt 20 ]]; then
          echo "ERROR: wait_for_postgresql failed" ; exit 1
        fi
        echo "Waiting for postgres at: ${pg_host}:${pg_port}"
        sleep $(( wait_time++  ));
      done
    }

    function wait_for_jenkins() {
      # Wait for jenkins to be functional
      local j_host=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:host)
      local j_host=${j_host:-"localhost"}
      local j_port=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:port)
      local j_port=${j_port:-"8081"}
      local j_prefix=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:url_prefix)
      local j_prefix=${j_prefix:-""}
      local wait_time=0
      until [[ $(curl -sL -w "%{http_code}" ${j_host}:${j_port}${j_prefix} -o /dev/null) == 200 ]]; do
        if [[ $wait_time -gt 20 ]]; then
          echo "ERROR: wait_for_jenkins failed" ; exit 1
        fi
        sleep $(( wait_time++ ))
      done
    }

    function process_swarm() {
      local _swarm=$(salt-call --out=newline_values_only pillar.get docker:swarm:advertise_addr)
      if [[ "${_swarm}" != "" ]]; then
        salt-call ${SALT_OPTS} state.apply docker.swarm
      fi
      _docker_enabled=$(salt-call --out=newline_values_only pillar.get docker:client:enabled | tr "[:upper:]" "[:lower:]")
      if [[ "${_docker_enabled}" == "true" ]]; then
        salt-call ${SALT_OPTS} state.apply docker.client.images
        salt-call ${SALT_OPTS} state.apply docker.client
      fi
    }

    function process_jenkins() {
      # INFO: jenkins is in docker in 2019.x releases
      if [ -f /opt/jenkins_in_docker ]; then
        rm -v /opt/jenkins_in_docker
        export JENKINS_HOME=/srv/volumes/jenkins
        _jenkins_enabled=$(salt-call --out=newline_values_only pillar.get jenkins:client:enabled | tr "[:upper:]" "[:lower:]")
        if [[ "${_jenkins_enabled}" == "true" ]]; then
          wait_for_jenkins
          salt-call ${SALT_OPTS} state.apply jenkins.client
        fi
      else
        export JENKINS_HOME=/var/lib/jenkins
        systemctl enable jenkins
        systemctl start jenkins
        wait_for_jenkins
        _jenkins_enabled=$(salt-call --out=newline_values_only pillar.get jenkins:client:enabled | tr "[:upper:]" "[:lower:]")
        if [[ "${_jenkins_enabled}" == "true" ]]; then
          salt-call ${SALT_OPTS} state.apply jenkins.client
        fi
        systemctl stop jenkins
        find ${JENKINS_HOME}/jenkins.model.JenkinsLocationConfiguration.xml -type f -print0 | xargs -0 sed -i -e "s/10.167.4.15/$SALT_MASTER_DEPLOY_IP/g"
      fi

      ssh-keyscan cfg01 > ${JENKINS_HOME}/.ssh/known_hosts && chmod a+r ${JENKINS_HOME}/.ssh/known_hosts || true
    }

    function wait_for_salt(){
      local wait_time=0
      until $(salt-call --timeout=30 test.ping  &> /dev/null ); do
        if [[ $wait_time -gt 15 ]]; then
          echo "ERROR: wait_for_salt failed" ; exit 1
        fi
        sleep $(( wait_time++ ))
      done
    }

    function process_salt_base(){
      # PROD-21179| PROD-21792 : To describe such trick's around salt.XX state ordering
      salt-call ${SALT_OPTS} state.apply salt.master
      # Wait for salt-master to wake up after restart
      sleep 5
      wait_for_salt
      # Run salt.minion.ca to prepare CA certificate before salt.minion.cert is used
      salt-call ${SALT_OPTS} state.apply salt.minion.ca
      # Add sleep for completion of postponed restart of salt-minion. PROD-25986
      sleep 15
      salt-call ${SALT_OPTS} state.apply salt.minion
      # Wait for salt-minion to wake up after restart
      wait_for_salt
      salt-call ${SALT_OPTS} state.apply salt
      salt-call ${SALT_OPTS} state.apply reclass
    }
    #== Body ==================================================================#

    . /etc/cloud/master_environment
    printenv | sort -u
    # WA for https://gerrit.mcp.mirantis.com/#/c/34514/
    echo "INFO: stopping orphaned maas"
    systemctl stop maas-rackd.service maas-dhcpd.service maas-dhcpd6.service || true
    systemctl disable maas-rackd.service maas-dhcpd.service maas-dhcpd6.service || true

    echo "Preparing metadata model"
    if [[ -n "${CFG_BOOTSTRAP_DRIVE_URL}" ]]; then
      echo "CFG_BOOTSTRAP_DRIVE_URL detected,downloading..."
      _tname="cfg01_${RANDOM}.iso"
      _wget_opts="--progress=dot:mega --waitretry=15 --retry-connrefused"
      wget ${_wget_opts} -O /${_tname} "${CFG_BOOTSTRAP_DRIVE_URL}"
      mount -o loop /${_tname} /mnt
    else
      mount $(blkid -t TYPE=iso9660 -o device) /mnt
    fi
    rsync -av /mnt/mcp/model/ /srv/salt/reclass/
    chown -R root:root /srv/salt/reclass/ || true
    find /srv/salt/reclass/classes/ -type d -exec chmod 0755 {} \;
    find /srv/salt/reclass/classes/ -type f -exec chmod 0644 {} \;

    echo "Configuring salt"
    envsubst < /root/minion.conf > /etc/salt/minion.d/minion.conf
    if [ -f /mnt/mcp/gpg/salt_master_pillar.asc ]; then
      mkdir -p /etc/salt/gpgkeys
      chmod 0700 /etc/salt/gpgkeys
      GNUPGHOME=/etc/salt/gpgkeys gpg --import /mnt/mcp/gpg/salt_master_pillar.asc
    fi
    enable_services

    # Wait for salt-master and salt-minion to wake up after restart
    wait_for_salt

    while true; do
    salt-key | grep -w "$SALT_MASTER_MINION_ID" && break
      sleep 5
    done

    echo "updating local git repos"
    if [[ "$PIPELINES_FROM_ISO" == "true" ]] ; then
      rsync -av /mnt/mcp/mk-pipelines/ /home/repo/mk/mk-pipelines/
      rsync -av /mnt/mcp/pipeline-library/ /home/repo/mcp-ci/pipeline-library/
      umount /mnt || true
      chown -R git:www-data /home/repo/mk/mk-pipelines /home/repo/mcp-ci/pipeline-library
    else
      umount /mnt || true
      git clone --mirror "${MK_PIPELINES_URL}" /home/repo/mk/mk-pipelines/
      git clone --mirror "${PIPELINE_LIB_URL}" /home/repo/mcp-ci/pipeline-library/
      chown -R git:www-data /home/repo/mk/mk-pipelines /home/repo/mcp-ci/pipeline-library
    fi

    process_formulas

    salt-call saltutil.refresh_pillar
    salt-call saltutil.sync_all
    if ! $(reclass -n ${SALT_MASTER_MINION_ID} > /dev/null ) ; then
      echo "ERROR: Reclass render failed!"
      exit 1
    fi

    salt-call ${SALT_OPTS} state.apply linux.system.repo,linux.network,linux.system,linux,openssh
    process_salt_base

    systemctl stop apache2 || true
    systemctl disable apache2 || true
    chmod 0 /etc/init.d/apache2

    _kfg_enabled=$(salt-call --out=newline_values_only pillar.get kubernetes:master:enabled | tr "[:upper:]" "[:lower:]")
    if [ "${_kfg_enabled}" != "true" ]; then
      _nginx_enabled=$(salt-call --out=newline_values_only pillar.get nginx:server:enabled | tr "[:upper:]" "[:lower:]")
      if [[ "${_nginx_enabled}" == "true" ]]; then
        salt-call ${SALT_OPTS} state.apply nginx.server
      fi
      if [ -f /opt/jenkins_in_docker ] || [ -f /opt/postgresql_in_docker ]; then
        process_swarm
      fi
      if [ -f /opt/jenkins_in_docker ] && [ ! -f /opt/postgresql_in_docker ]; then
        docker stack rm postgresql || true
      fi
    else
      salt-call ${SALT_OPTS} state.apply etcd.server.service
      salt-call ${SALT_OPTS} state.apply kubernetes.master.kube-addons
      _images=$(salt-call --out=newline_values_only pillar.get kubernetes:pool:images)
      if [ -n "${_images}" ]; then
        salt-call ${SALT_OPTS} state.apply kubernetes.pool.images
      fi
      salt-call ${SALT_OPTS} state.apply kubernetes exclude=kubernetes.master.setup,kubernetes.client
      salt-call ${SALT_OPTS} state.apply kubernetes.master
      salt-call ${SALT_OPTS} state.apply nginx.server
      _ldap_enabled=$(salt-call --out=newline_values_only pillar.get openldap:client:enabled | tr "[:upper:]" "[:lower:]")
      if [ "${_ldap_enabled}" == "true" ]; then
        salt-call ${SALT_OPTS} state.apply openldap.client
      fi
      _gerrit_enabled=$(salt-call --out=newline_values_only pillar.get gerrit:client:enabled | tr "[:upper:]" "[:lower:]")
      if [ "${_gerrit_enabled}" == "true" ]; then
        salt-call ${SALT_OPTS} state.apply gerrit.client
      fi
    fi

    process_jenkins
    process_maas

    stop_services="salt-api salt-master salt-minion maas-rackd.service maas-regiond.service postgresql.service"
    for s in ${stop_services} ; do
      systemctl stop ${s} || true
      sleep 1
    done
    # Set bootstrap-done flag for future
    mkdir -p /var/log/mcp/
    touch /var/log/mcp/.bootstrap_done
    sync
    shutdown -r now
runcmd:
   - [bash, -cex, *master_config]