Fix wait for salt warm-up check
* Fix wait_time condition checks
* add maas wa from https://gerrit.mcp.mirantis.com/#/c/34514/
Change-Id: I6b2181c5d48b2524493e4ca809f08709e671fc24
Prod-related: PROD-27016 (PROD:27016)
Prod-related: (PROD:26907)
diff --git a/config-drive/master_config.yaml b/config-drive/master_config.yaml
index e8baa4a..ba0e9b0 100644
--- a/config-drive/master_config.yaml
+++ b/config-drive/master_config.yaml
@@ -114,11 +114,13 @@
function wait_for_postgresql() {
salt_string="salt-call --out=text pillar.get postgresql:client:server:server01:admin"
- pg_port=$(${salt_string}:port | awk '{print $2}')
- pg_host=$(${salt_string}:host | awk '{print $2}')
- wait_time=0
- until [[ $(/usr/bin/pg_isready -h ${pg_host} -p ${pg_port} | awk '{ print $3 }' ) == 'accepting' ]] || [ $wait_time -eq 20 ];
- do
+ local pg_port=$(${salt_string}:port | awk '{print $2}')
+ local pg_host=$(${salt_string}:host | awk '{print $2}')
+ local wait_time=0
+ until [[ $(/usr/bin/pg_isready -h ${pg_host} -p ${pg_port} | awk '{ print $3 }' ) == 'accepting' ]]; do
+ if [[ $wait_time -gt 20 ]]; then
+ echo "ERROR: wait_for_postgresql failed" ; exit 1
+ fi
echo "Waiting for postgres at: ${pg_host}:${pg_port}"
sleep $(( wait_time++ ));
done
@@ -126,20 +128,23 @@
function wait_for_jenkins() {
# Wait for jenkins to be functional
- j_host=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:host)
- j_host=${j_host:-"localhost"}
- j_port=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:port)
- j_port=${j_port:-"8081"}
- j_prefix=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:url_prefix)
- j_prefix=${j_prefix:-""}
- wait_time=0
- until [[ $(curl -sL -w "%{http_code}" ${j_host}:${j_port}${j_prefix} -o /dev/null) == 200 ]] || [ $wait_time -eq 20 ]; do
+ local j_host=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:host)
+ local j_host=${j_host:-"localhost"}
+ local j_port=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:port)
+ local j_port=${j_port:-"8081"}
+ local j_prefix=$(salt-call --out=newline_values_only pillar.get jenkins:client:master:url_prefix)
+ local j_prefix=${j_prefix:-""}
+ local wait_time=0
+ until [[ $(curl -sL -w "%{http_code}" ${j_host}:${j_port}${j_prefix} -o /dev/null) == 200 ]]; do
+ if [[ $wait_time -gt 20 ]]; then
+ echo "ERROR: wait_for_jenkins failed" ; exit 1
+ fi
sleep $(( wait_time++ ))
done
}
function process_swarm() {
- _swarm=$(salt-call --out=newline_values_only pillar.get docker:swarm:advertise_addr)
+ local _swarm=$(salt-call --out=newline_values_only pillar.get docker:swarm:advertise_addr)
if [[ "${_swarm}" != "" ]]; then
salt-call ${SALT_OPTS} state.apply docker.swarm
fi
@@ -176,20 +181,29 @@
ssh-keyscan cfg01 > ${JENKINS_HOME}/.ssh/known_hosts && chmod a+r ${JENKINS_HOME}/.ssh/known_hosts || true
}
+ function wait_for_salt(){
+ local wait_time=0
+ until $(salt-call --timeout=30 test.ping &> /dev/null ); do
+ if [[ $wait_time -gt 15 ]]; then
+ echo "ERROR: wait_for_salt failed" ; exit 1
+ fi
+ sleep $(( wait_time++ ))
+ done
+ }
+
function process_salt_base(){
# PROD-21179| PROD-21792 : To describe such trick's around salt.XX state ordering
salt-call ${SALT_OPTS} state.apply salt.master
# Wait for salt-master to wake up after restart
sleep 5
- salt-call --timeout=120 test.ping
+ wait_for_salt
# Run salt.minion.ca to prepare CA certificate before salt.minion.cert is used
salt-call ${SALT_OPTS} state.apply salt.minion.ca
# Add sleep for completion of postponed restart of salt-minion. PROD-25986
sleep 15
salt-call ${SALT_OPTS} state.apply salt.minion
# Wait for salt-minion to wake up after restart
- sleep 5
- salt-call --timeout=120 test.ping
+ wait_for_salt
salt-call ${SALT_OPTS} state.apply salt
salt-call ${SALT_OPTS} state.apply reclass
}
@@ -197,6 +211,10 @@
. /etc/cloud/master_environment
printenv | sort -u
+ # WA for https://gerrit.mcp.mirantis.com/#/c/34514/
+ echo "INFO: stopping orphaned maas"
+ systemctl stop maas-rackd.service maas-dhcpd.service maas-dhcpd6.service || true
+ systemctl disable maas-rackd.service maas-dhcpd.service maas-dhcpd6.service || true
echo "Preparing metadata model"
if [[ -n "${CFG_BOOTSTRAP_DRIVE_URL}" ]]; then
@@ -216,14 +234,14 @@
echo "Configuring salt"
envsubst < /root/minion.conf > /etc/salt/minion.d/minion.conf
if [ -f /mnt/mcp/gpg/salt_master_pillar.asc ]; then
- mkdir /etc/salt/gpgkeys
+ mkdir -p /etc/salt/gpgkeys
chmod 0700 /etc/salt/gpgkeys
GNUPGHOME=/etc/salt/gpgkeys gpg --import /mnt/mcp/gpg/salt_master_pillar.asc
fi
enable_services
# Wait for salt-master and salt-minion to wake up after restart
- salt-call --timeout=120 test.ping
+ wait_for_salt
while true; do
salt-key | grep -w "$SALT_MASTER_MINION_ID" && break