blob: af85717b24af8fa18acdec03fc247f4622341559 [file] [log] [blame]
Alex Savatieievbadc4762019-09-30 13:46:37 -05001#!/bin/bash
Alex Savatieieveaf0a992019-10-02 17:51:54 -05002silent=false
3cleaning=false
Alex Savatieievd0ae84f2019-10-23 13:36:37 -05004all_computes=false
5fill_mode=false
Alex Savatieiev814307f2020-04-13 12:47:39 -05006zone=nova
7use_fqdn=false
Ievgeniia Zadorozhnabbe0f442024-01-22 23:59:05 +01008vm_name_prefix=cvp_test_vm_
Alex Savatieieveaf0a992019-10-02 17:51:54 -05009
10tmp_out=$(mktemp)
11trap "rm -f ${tmp_out}" EXIT
12
13declare errors=()
14
15function show_help {
16 printf "Compute check/filling script\n\t-h, -?\tShow this help\n"
17 printf "\t-d\tCleaning of earlier created VMs\n"
18 printf "\t-q\tSilent mode\n"
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050019 printf "\t-a\tEnumeratre all computes\n"
20 printf "\t-f\tFill mode\n"
Alex Savatieiev814307f2020-04-13 12:47:39 -050021 printf "\t-z <zone>\tAvailability zone to use on create\n"
22 printf "\t-n\tUse compute's FQDN when setting zone hint\n"
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050023 printf "\nUsage: cmp_check.sh (-a | <compute_hostname>) (-f [<vm_count>|def:1])\n"
Alex Savatieieveaf0a992019-10-02 17:51:54 -050024 printf "\t<compute_hostname> is a host shortname\n"
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050025 printf "\t<vm_count> is optional. Defaults to 1\n"
26 printf "Examples:\n"
27 printf "\tFill all computes with 3 VMs:\n\t\t'bash cmp_check.sh -fa 3'\n\n"
28 printf "\tFill specific compute with 5 VMs:\n\t\t'bash cmp_check.sh -f cmp001 5'\n\n"
29 printf "\tCheck all computes:\n\t\t'bash cmp_check.sh -a'\n\n"
30 printf "\tCheck specific compute:\n\t\t'bash cmp_check.sh cmp001'\n\n"
31 printf "\tClean all computes:\n\t\t'bash cmp_check.sh -da'\n\n"
Alex Savatieieveaf0a992019-10-02 17:51:54 -050032}
33
34OPTIND=1 # Reset in case getopts has been used previously in the shell.
Alex Savatieiev814307f2020-04-13 12:47:39 -050035while getopts "h?:qdafz:n" opt; do
Alex Savatieieveaf0a992019-10-02 17:51:54 -050036 case "$opt" in
37 h|\?)
38 show_help
39 exit 0
40 ;;
41 q) silent=true
42 ;;
43 d) cleaning=true
44 ;;
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050045 a) all_computes=true
46 ;;
47 f) fill_mode=true
48 ;;
Alex Savatieiev814307f2020-04-13 12:47:39 -050049 z) zone=${OPTARG}
50 printf "# Using availability zone of '${zone}'\n"
51 ;;
52 n) use_fqdn=true
53 printf "# Using FQDN as a compute host name\n"
54 ;;
Alex Savatieieveaf0a992019-10-02 17:51:54 -050055 esac
56done
57
58shift $((OPTIND-1))
59[ "${1:-}" = "--" ] && shift
60
61# Check and create cmp_name var
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050062if [[ -z ${1+x} ]] && [[ ! ${all_computes} == "true" ]]; then
Alex Savatieieveaf0a992019-10-02 17:51:54 -050063 show_help
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050064 printf "\nERROR: No compute host specified\n"
Alex Savatieieveaf0a992019-10-02 17:51:54 -050065 exit 1
66fi
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050067if [[ ${all_computes} == "true" ]]; then
68 cmp_name=all
69 # if enumerate mode is set, vmcount source is ${1}
70 if [[ -z ${1+x} ]] || [[ ! ${fill_mode} == true ]]; then
71 vmcount=1
72 else
73 vmcount=${1}
74 fi
Alex Savatieieveaf0a992019-10-02 17:51:54 -050075else
Alex Savatieievd0ae84f2019-10-23 13:36:37 -050076 cmp_name=${1}
77 # in single compute mode, vmcount source is ${2}
78 # in check mode count is always 1
79 if [[ -z ${2+x} ]] || [[ ! ${fill_mode} == true ]]; then
80 vmcount=1
81 else
82 vmcount=${2}
83 fi
Alex Savatieieveaf0a992019-10-02 17:51:54 -050084fi
Alex Savatieieveaf0a992019-10-02 17:51:54 -050085
86function cmp_stats() {
87 cmpid=$(openstack hypervisor list --matching ${1} -f value -c ID)
Alex Savatieiev814307f2020-04-13 12:47:39 -050088 vars=( $(openstack hypervisor show ${cmpid} -f shell -c state -c running_vms -c vcpus -c vcpus_used -c memory_mb -c memory_mb_used) )
89 [ ! 0 -eq $? ] && errors+=("${1}: $(cat ${vars[@]})")
90 if [ ! $state == '"up"' ]; then
91 echo "# Hypervisor fail, state is '${state}'"
92 errors
93 exit 1
94 else
95 declare ${vars[@]}
96 printf "${1}: vms=%s vcpus=%s/%s ram=%s/%s\n" ${running_vms} ${vcpus_used} ${vcpus} ${memory_mb_used} ${memory_mb}
97 fi
Alex Savatieieveaf0a992019-10-02 17:51:54 -050098}
Alex Savatieievbadc4762019-09-30 13:46:37 -050099
100function waitfor () {
101 counter=0
102 while [ ${counter} -lt 6 ]; do
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500103 ids=( $(openstack server list --name ${1} --status ${2} -f value -c ID) )
Alex Savatieievbadc4762019-09-30 13:46:37 -0500104 if [ ${#ids[@]} -eq 0 ]; then
105 sleep 5
106 counter=$((counter + 1))
107 else
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500108 [ ! "$silent" = true ] && printf "# '${1}' reached status ${2}\n"
Alex Savatieievbadc4762019-09-30 13:46:37 -0500109 break
110 fi
111 done
112}
113
114function getid() {
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500115 openstack server list -c ID -c Name -f value | grep "${1}" | cut -d' ' -f1
Alex Savatieievbadc4762019-09-30 13:46:37 -0500116}
117
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500118function get_all_cmp() {
Alex Savatieiev814307f2020-04-13 12:47:39 -0500119 if [ $use_fqdn == true ]; then
120 openstack hypervisor list -f value -c "Hypervisor Hostname" -c State | grep "up" | sort | cut -d' ' -f1
121 else
122 openstack hypervisor list -f value -c "Hypervisor Hostname" -c State | grep "up" | sort | cut -d'.' -f1
123 fi
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500124}
125
Alex Savatieievbadc4762019-09-30 13:46:37 -0500126function vm_create() {
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500127 [ ! "$silent" = true ] && set -x
Ievgeniia Zadorozhna78ada712024-02-23 17:23:38 +0100128 openstack server create --nic net-id=${fixed_net_left_id} --image ${cirros61_id} --flavor ${flavor_tiny_id} --key-name ${keypair_id} --security-group ${secgroup_all_id} --availability-zone ${zone}:${1} ${2} 2>${tmp_out} >/dev/null
Alex83075ac2022-02-16 13:39:50 -0600129 #openstack server create --nic net-id=${fixed_net_left_id} --image ${ubuntu16_id} --flavor ${flavor_high_id} --key-name ${keypair_id} --security-group ${secgroup_all_id} --availability-zone ${zone}:${1} ${2} 2>${tmp_out} >/dev/null
130 #openstack server create --nic net-id=${fixed_net_right_id} --image ${ubuntu16_id} --flavor ${flavor_high_id} --key-name ${keypair_id} --security-group ${secgroup_all_id} --availability-zone ${zone}:${1} ${2} 2>${tmp_out} >/dev/null
Alexdb7786b2022-02-21 17:58:29 -0600131 #openstack server create --nic net-id=${fixed_net_left_id} --image ${ubuntu20_id} --flavor ${flavor_high_id} --key-name ${keypair_id} --security-group ${secgroup_all_id} --availability-zone ${zone}:${1} ${2} 2>${tmp_out} >/dev/null
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500132 [ ! 0 -eq $? ] && errors+=("${1}/${2}: $(cat ${tmp_out})")
Alex Savatieievbadc4762019-09-30 13:46:37 -0500133 set +x
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500134 [ ! "$silent" = true ] && cat ${tmp_out}
Alex Savatieievbadc4762019-09-30 13:46:37 -0500135}
136
137function vm_action() {
Alex6892c8c2021-01-28 16:07:29 -0600138 openstack server ${1} ${2} 2>${tmp_out} >/dev/null
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500139 if [ ! 0 -eq $? ]; then
140 errors+=("${cmp_name}: $(cat ${tmp_out})")
141 fi
Alex Savatieievbadc4762019-09-30 13:46:37 -0500142}
143
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500144function errors {
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500145 echo "==== Errors"
146 for i in "${!errors[@]}"; do
147 printf "#%s\n" "${errors[$i]}"
148 done
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500149}
150
151function join_by { local IFS="$1"; shift; echo "$*"; }
152
Alex Savatieievbadc4762019-09-30 13:46:37 -0500153
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500154function clean_cmp() {
155 # #### Cleaning mode
156 if [ $cleaning = true ]; then
Ievgeniia Zadorozhnabbe0f442024-01-22 23:59:05 +0100157 vmname=${vm_name_prefix}${1}
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500158 vmid=( $(getid ${vmname}) )
159 if [ ${#vmid[@]} -ne 0 ]; then
160 [ ! "$silent" = true ] && echo "# ${1}: cleaning ${#vmid[@]} VMs"
161 vm_action delete "$(join_by ' ' "${vmid[@]}")"
162 else
163 [ ! "$silent" = true ] && echo "# ${1}: ...no VMs found"
164 fi
165 fi
166}
167
168function check_cmp_node() {
169 cmp_stats ${1}
170 vm_create ${1} ${2}
171 waitfor ${2} ACTIVE
172 vmid=$(getid ${2})
173
174 cmp_stats ${1}
175
176 vm_action pause ${vmid}
177 waitfor ${2} PAUSED
178 vm_action unpause ${vmid}
179 waitfor ${2} ACTIVE
180
Alex Savatieiev814307f2020-04-13 12:47:39 -0500181 [ ! "$silent" = true ] && echo "# ... deleting created VMs"
182 clean_cmp ${1}
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500183 cmp_stats ${1}
184}
Alex Savatieievbadc4762019-09-30 13:46:37 -0500185
Alex Savatieievbadc4762019-09-30 13:46:37 -0500186if [ ! -f cvp.manifest ]; then
187 echo "ERROR: No cvp.manifest file detected. Consider running prepare.sh"
188 exit 1
189else
190 source cvp.manifest
191fi
192
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500193[ ! "$silent" = true ] && echo "# Sourcing cvprc"
Alex Savatieievbadc4762019-09-30 13:46:37 -0500194source cvprc
195
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500196# #### Checking for CMP existence
197if [[ ! ${cmp_name} == "all" ]]; then
Alex Savatieiev814307f2020-04-13 12:47:39 -0500198 echo "# Inspecting '${zone}:${cmp_name}'"
199 cmp_fqdn=$(openstack host list --zone ${zone} -f value -c 'Host Name' -c 'Zone' | grep ${cmp_name} | cut -d' ' -f1 2>${tmp_out})
200 [ ! 0 -eq $? ] && errors+=("${cmp_name}\@${zone}: $(cat ${tmp_out})")
201 if [[ -z ${cmp_fqdn} ]]; then
202 echo "ERROR: ${cmp_name} not found in ${zone}"
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500203 errors
204 exit 1
205 fi
Alex Savatieiev814307f2020-04-13 12:47:39 -0500206 printf "# Found ${cmp_fqdn} in '${zone}' using given name of ${cmp_name}\n"
207 vars=( $(openstack hypervisor show ${cmp_fqdn} -f shell -c id -c state -c hypervisor_hostname) )
208 [ ! 0 -eq $? ] && errors+=("${cmp_name}: $(cat ${tmp_out})")
209 declare ${vars[@]}
210 # check that such node exists
211 if [ -z ${id+x} ]; then
212 # no id
213 echo "ERROR: ${cmp_name} not found among hypervisors"
214 errors
215 exit 1
216 else
217 echo "# ${id}, ${hypervisor_hostname}, status '${state}'"
218 if [ ! ${state} == '"up"' ]; then
219 echo "ERROR: ${hypervisor_hostname} is '${state}'"
220 exit 1
221 else
222 unset id
223 unset hypervisor_hostname
224 unset state
225 fi
226 fi
227
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500228fi
229
230if [[ ${cmp_name} == all ]]; then
Alex Savatieiev814307f2020-04-13 12:47:39 -0500231 echo "# Gathering compute count with state 'up'"
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500232 cmp_nodes=( $(get_all_cmp) )
233else
Alex Savatieiev814307f2020-04-13 12:47:39 -0500234 if [ $use_fqdn == true ]; then
235 cmp_nodes=( ${cmp_fqdn} )
236 else
237 cmp_nodes=( ${cmp_name} )
238 fi
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500239fi
240
241
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500242# #### Cleaning mode
243if [ $cleaning = true ]; then
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500244 # get all computes
245 if [[ ${cmp_name} == all ]]; then
246 echo "# Cleaning ${#cmp_nodes[@]} computes"
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500247 else
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500248 echo "# Cleaning ${cmp_name}"
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500249 fi
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500250
251 # clean them
252 for node in ${cmp_nodes[@]}; do
Alex Savatieiev814307f2020-04-13 12:47:39 -0500253 cname=$(echo ${node} | cut -d'.' -f1)
254 clean_cmp ${cname}
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500255 done
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500256 echo "# Done cleaning"
257 errors
258 exit 0
Alex Savatieievbadc4762019-09-30 13:46:37 -0500259fi
260
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500261# ###
262if [[ ! ${fill_mode} = true ]]; then
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500263 # ### CMP Checking mode
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500264 if [[ ${cmp_name} == all ]]; then
265 echo "# Checking ${#cmp_nodes[@]} computes"
266 fi
267 # check node
268 for node in ${cmp_nodes[@]}; do
269 echo "# ${node}: checking"
Alex Savatieiev814307f2020-04-13 12:47:39 -0500270 cname=$(echo ${node} | cut -d'.' -f1)
Ievgeniia Zadorozhnabbe0f442024-01-22 23:59:05 +0100271 check_cmp_node ${node} ${vm_name_prefix}${cname}
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500272 echo "# ${node}: done"
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500273 done
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500274 errors
275 exit 0
276else
277 # ### CMP fillling mode
278 if [[ ${cmp_name} == all ]]; then
279 echo "# Filling ${#cmp_nodes[@]} computes"
280 fi
281
282 for node in ${cmp_nodes[@]}; do
283 echo "# ${node}: filling"
284 counter=1
285 while [[ $counter -lt ${vmcount}+1 ]]; do
Alex Savatieiev814307f2020-04-13 12:47:39 -0500286 cname=$(echo ${node} | cut -d'.' -f1)
Ievgeniia Zadorozhnabbe0f442024-01-22 23:59:05 +0100287 vmname_c=${vm_name_prefix}${cname}_$(printf "%02d" ${counter})
Alex Savatieievd0ae84f2019-10-23 13:36:37 -0500288 [ ! "$silent" = true ] && echo "# ${node}: creating ${vmname_c}"
289 vm_create ${node} ${vmname_c}
290 cmp_stats ${node}
291 ((counter++))
292 done
293 printf "# ${node}: done\n"
294 done
Alex Savatieieveaf0a992019-10-02 17:51:54 -0500295fi
296
297errors
Alex Savatieievbadc4762019-09-30 13:46:37 -0500298