Fix wait for salt warm-up check
* Fix wait_time condition checks
* add maas wa from https://gerrit.mcp.mirantis.com/#/c/34514/
Change-Id: I6b2181c5d48b2524493e4ca809f08709e671fc24
(cherry picked from commit f25764781f29c5c5a4f4a46c54a8ce84fb7381ed)
Prod-related: PROD-27016 (PROD:27016)
Prod-related: (PROD:26907)
diff --git a/config-drive/master_config.yaml b/config-drive/master_config.yaml
index 7d975fd..ccafbc3 100644
--- a/config-drive/master_config.yaml
+++ b/config-drive/master_config.yaml
@@ -127,11 +127,13 @@
function wait_for_postgresql() {
salt_string="salt-call --out=text pillar.get postgresql:client:server:server01:admin"
- pg_port=$(${salt_string}:port | awk '{print $2}')
- pg_host=$(${salt_string}:host | awk '{print $2}')
- wait_time=0
- until [[ $(/usr/bin/pg_isready -h ${pg_host} -p ${pg_port} | awk '{ print $3 }' ) == 'accepting' ]] || [ $wait_time -eq 20 ];
- do
+ local pg_port=$(${salt_string}:port | awk '{print $2}')
+ local pg_host=$(${salt_string}:host | awk '{print $2}')
+ local wait_time=0
+ until [[ $(/usr/bin/pg_isready -h ${pg_host} -p ${pg_port} | awk '{ print $3 }' ) == 'accepting' ]]; do
+ if [[ $wait_time -gt 20 ]]; then
+ echo "ERROR: wait_for_postgresql failed" ; exit 1
+ fi
echo "Waiting for postgres at: ${pg_host}:${pg_port}"
sleep $(( wait_time++ ));
done
@@ -142,13 +144,16 @@
jport=$(salt-call --out=text pillar.get jenkins:master:http:port | awk '{print $2}')
jport=${jport:-8081}
wait_time=0
- until [[ $(curl -sL -w "%{http_code}" localhost:$jport -o /dev/null) == 200 ]] || [ $wait_time -eq 20 ]; do
+ until [[ $(curl -sL -w "%{http_code}" localhost:$jport -o /dev/null) == 200 ]] ; do
+ if [[ $wait_time -gt 20 ]]; then
+ echo "ERROR: wait_for_jenkins failed" ; exit 1
+ fi
sleep $(( wait_time++ ))
done
}
function process_swarm() {
- _swarm=$(salt-call --out=text pillar.get docker:swarm:advertise_addr | awk '{print $2}')
+ local _swarm=$(salt-call --out=text pillar.get docker:swarm:advertise_addr | awk '{print $2}')
if [[ "${_swarm}" != "" ]]; then
salt-call ${SALT_OPTS} state.sls docker.swarm
fi
@@ -190,9 +195,9 @@
ssh-keyscan cfg01 > ${JENKINS_HOME}/.ssh/known_hosts && chmod a+r ${JENKINS_HOME}/.ssh/known_hosts || true
}
- failsafe_ssh_key(){
+ function failsafe_ssh_key(){
if [ -f /mnt/root_auth_keys ]; then
- echo "Installing failsafe public ssh key from /mnt/root_auth_keys to /root/.ssh/authorized_keys"
+ echo 'Installing failsafe public ssh key from /mnt/root_auth_keys to /root/.ssh/authorized_keys'
install -m 0700 -d /root/.ssh
cat /mnt/root_auth_keys >> /root/.ssh/authorized_keys
chmod 600 /root/.ssh/authorized_keys
@@ -202,27 +207,41 @@
fi
}
+ function wait_for_salt(){
+ local wait_time=0
+ until $(salt-call --timeout=30 test.ping &> /dev/null ); do
+ if [[ $wait_time -gt 15 ]]; then
+ echo "ERROR: wait_for_salt failed" ; exit 1
+ fi
+ sleep $(( wait_time++ ))
+ done
+ }
+
function process_salt_base(){
# PROD-21179| PROD-21792 : To describe such trick's around salt.XX state ordering
salt-call ${SALT_OPTS} state.sls salt.master
# Wait for salt-master to wake up after restart
sleep 5
- salt-call --timeout=120 test.ping
+ wait_for_salt
# Run salt.minion.ca to prepare CA certificate before salt.minion.cert is used
salt-call ${SALT_OPTS} state.sls salt.minion.ca
# Add sleep for completion of postponed restart of salt-minion. PROD-25986
sleep 15
salt-call ${SALT_OPTS} state.sls salt.minion
# Wait for salt-minion to wake up after restart
- sleep 5
- salt-call --timeout=120 test.ping
- salt-call ${SALT_OPTS} state.sls salt
- salt-call ${SALT_OPTS} state.sls reclass
+ wait_for_salt
+ salt-call ${SALT_OPTS} state.apply salt
+ salt-call ${SALT_OPTS} state.apply reclass
}
#== Body ==================================================================#
. /etc/cloud/master_environment
printenv | sort -u
+ # WA for https://gerrit.mcp.mirantis.com/#/c/34514/
+ echo "INFO: stopping orphaned maas"
+ systemctl stop maas-rackd.service maas-dhcpd.service maas-dhcpd6.service || true
+ systemctl disable maas-rackd.service maas-dhcpd.service maas-dhcpd6.service || true
+
process_network
echo "Preparing metadata model"
@@ -246,14 +265,14 @@
echo "Configuring salt"
envsubst < /root/minion.conf > /etc/salt/minion.d/minion.conf
if [ -f /mnt/gpg/salt_master_pillar.asc ]; then
- mkdir /etc/salt/gpgkeys
- chmod 700 /etc/salt/gpgkeys
+ mkdir -p /etc/salt/gpgkeys
+ chmod 0700 /etc/salt/gpgkeys
GNUPGHOME=/etc/salt/gpgkeys gpg --import /mnt/gpg/salt_master_pillar.asc
fi
enable_services
# Wait for salt-master and salt-minion to wake up after restart
- salt-call --timeout=120 test.ping
+ wait_for_salt
while true; do
salt-key | grep "$SALT_MASTER_MINION_ID" && break