Added RefApp Density testing to the toolset full pod
Added RefApp Density automated testing to the toolset
full pod:
* added openstack-refapp source code archive
* extended the dockerfile to install openstack-refapp
* added refapp-prepare.sh script to do some pre-steps
* added run-refapp-density.sh script to run the tests
* refactored old code in the dockerfile to improve
Related-PROD: PROD-36856
Change-Id: Ib6c66b14d8d7f057aaebeafe2ab1eb6ff3c7b06b
diff --git a/k8s/docker-mos-toolset-full b/k8s/docker-mos-toolset-full
index d2e4ac3..0ce01e3 100644
--- a/k8s/docker-mos-toolset-full
+++ b/k8s/docker-mos-toolset-full
@@ -1,6 +1,6 @@
FROM ubuntu:20.04
-MAINTAINER Alex Savatieiev (a.savex@gmail.com)
+LABEL maintainer="qa-ps@mirantis.com"
ADD ./src/si-tests.tgz /opt/si-tests/
ADD ./src/mos-checker.tgz /opt/cfg-checker/
@@ -12,18 +12,23 @@
./configure && \
make && \
make install && \
- cd /opt && \
- rm -rf /opt/fio
+ rm -rf /opt/fio && \
+ mkdir /opt/density && \
+ rm -rf /var/lib/apt/lists/*
RUN export TZ="America/Chicago" && \
ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
- apt-get install -y python3-pip python3-venv vim git iperf3 iperf mtr htop iputils-ping traceroute tcpdump wget iproute2 curl screen qemu-utils
+ apt-get update && \
+ apt-get install -y python3-pip python3-venv vim git iperf3 iperf mtr htop iputils-ping traceroute tcpdump wget iproute2 curl screen qemu-utils jq && \
+ rm -rf /var/lib/apt/lists/*
-RUN curl https://baltocdn.com/helm/signing.asc | apt-key add - && \
+RUN apt-get update && \
+ curl https://baltocdn.com/helm/signing.asc | apt-key add - && \
apt-get install apt-transport-https --yes && \
echo "deb https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list && \
apt-get update && \
- apt-get install -y helm
+ apt-get install -y helm && \
+ rm -rf /var/lib/apt/lists/*
RUN pip3 install --no-cache-dir python-openstackclient python-neutronclient python-heatclient pyghmi python-octaviaclient tempestparser python-ironicclient aodhclient gnocchiclient python-barbicanclient python-glanceclient
@@ -45,12 +50,13 @@
python3 -m venv .sivenv && \
. .sivenv/bin/activate && \
pip3 install --no-cache-dir -r si_tests/requirements.txt && \
- deactivate && \
- cd /opt
+ deactivate
RUN mkdir /opt/packages/ && \
cd /opt/packages && \
- apt-get download $(apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances iperf3 iperf fio | grep "^\w" | sort -u) || true
+ apt-get update && \
+ apt-get download $(apt-cache depends --recurse --no-recommends --no-suggests --no-conflicts --no-breaks --no-replaces --no-enhances iperf3 iperf fio | grep "^\w" | sort -u) || true && \
+ rm -rf /var/lib/apt/lists/*
RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && \
install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && \
@@ -61,8 +67,17 @@
python3 -m venv .venv && \
. .venv/bin/activate && \
pip3 install --no-cache-dir -r requirements.txt && \
+ deactivate
+
+ADD ./src/openstack-refapp.tgz /opt/density/openstack-refapp/
+
+RUN cd /opt/density/openstack-refapp/ && \
+ python3 -m venv .refapp-venv && \
+ . .refapp-venv/bin/activate && \
+ pip3 install --no-cache-dir -r requirements.txt && \
deactivate && \
- cd /opt
+ cp /opt/res-files/scripts/prepare-refapp.sh /opt/density/openstack-refapp/ && \
+ cp /opt/res-files/scripts/run-refapp-density.sh /opt/density/openstack-refapp/
ENTRYPOINT ["sleep infinite"]
diff --git a/k8s/src/openstack-refapp.tgz b/k8s/src/openstack-refapp.tgz
new file mode 100644
index 0000000..019b24e
--- /dev/null
+++ b/k8s/src/openstack-refapp.tgz
Binary files differ
diff --git a/scripts/prepare-refapp.sh b/scripts/prepare-refapp.sh
new file mode 100644
index 0000000..e7ff161
--- /dev/null
+++ b/scripts/prepare-refapp.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+FLAVOR='cvp.high'
+IMAGE='cvp.ubuntu.2004'
+EXT_NET='public'
+DNS_SERVERS='[]' # example of the custom value: DNS_SERVERS='["192.168.129.5", "192.168.129.254"]'
+APP_DOCKER_IMAGE='mirantis.azurecr.io/openstack/openstack-refapp:0.1.2' # you can replace with some local copy for offline cloud
+
+REFAPP_PATH=/opt/density/openstack-refapp/
+
+# Check the private CVP key, convert to the public one
+cvp_private_key=/opt/cmp-check/cvp_testkey
+if [ -e ${cvp_private_key} ]; then
+ ssh-keygen -y -f ${cvp_private_key} > ${REFAPP_PATH}/cvp_testkey.pub
+else
+ echo "The key file at ${cvp_private_key} does not exist. Please run \"cd /opt/cmp-check && bash /opt/cmp-check/prepare.sh -w $(pwd) && cd ${REFAPP_PATH}\" to create a key and other test resources."
+ exit 1
+fi
+
+TOP_FILE=${REFAPP_PATH}/heat-templates/top.yaml
+
+# Replace the default public key with the CVP one
+cvp_key=$(cat ${REFAPP_PATH}/cvp_testkey.pub)
+escaped_cvp_key=$(echo "$cvp_key" | sed -e 's/[\/&]/\\&/g')
+sed -i "/^ *cluster_public_key:/,/default:/ s/default:.*/default: '$escaped_cvp_key'/" $TOP_FILE
+
+# Replace the flavor
+sed -i "/^ *database_flavor:/,/default:/ s/default:.*/default: '$FLAVOR'/" $TOP_FILE
+
+# Replace the image
+sed -i "/^ *database_image:/,/default:/ s/default:.*/default: '$IMAGE'/" $TOP_FILE
+
+# Replace the external net
+sed -i "/^ *public_network_id:/,/default:/ s/default:.*/default: '$EXT_NET'/" $TOP_FILE
+
+# Replace the DNS servers list
+sed -i "/^ *dns_nameservers:/,/default:/ s/default:.*/default: $DNS_SERVERS/" $TOP_FILE
+
+# Replace the docker image for RefApp application if the cloud has no Internet access
+ESC_APP_DOCKER_IMAGE=$(sed 's/[\/&]/\\&/g' <<< "$APP_DOCKER_IMAGE")
+sed -i "/^\s*app_docker_image:/,/default:/ s/default:.*/default: '$ESC_APP_DOCKER_IMAGE'/" $TOP_FILE
diff --git a/scripts/run-refapp-density.sh b/scripts/run-refapp-density.sh
new file mode 100644
index 0000000..731657d
--- /dev/null
+++ b/scripts/run-refapp-density.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+
+
+# Set variables
+HEAT_STACK_PREFIX=refapp
+ITERATIONS_START=1
+ITERATIONS_END=10
+LOG_FILE="./run-refapp-density-${ITERATIONS_START}-${ITERATIONS_END}.log"
+DATA_FILE="./run-refapp-density-${ITERATIONS_START}-${ITERATIONS_END}.csv"
+
+# source CVP project credentials
+source /opt/cmp-check/cvprc
+
+start_test() {
+ # Start test
+ #export OS_CLOUD=openstack
+ TEST_START_DATE=$(date +%s)
+ echo "Test started at $(date)" >> ${LOG_FILE}
+ echo "DATE;ITERATION_NUM;STACK_CREATION_DATE_DELTA;APP_REQUEST_TIME_ROOT;APP_REQUEST_TIME_POST_RECORD;APP_REQUEST_TIME_GET_RECORDS;APP_REQUEST_TIME_GET_RECORD" >> ${DATA_FILE}
+ for ITERATION_NUM in `seq ${ITERATIONS_START} ${ITERATIONS_END}`
+ do
+ STACK_CREATION_START_DATE=$(date +%s)
+ openstack stack create -t heat-templates/top.yaml ${HEAT_STACK_PREFIX}-${ITERATION_NUM} --wait | tee -a ${LOG_FILE}
+ STACK_STATUS=$(openstack stack show ${HEAT_STACK_PREFIX}-${ITERATION_NUM} -f value -c stack_status)
+ echo "$(date) ${HEAT_STACK_PREFIX}-${ITERATION_NUM} stack status is ${STACK_STATUS}" >> ${LOG_FILE}
+ STACK_CREATION_END_DATE=$(date +%s)
+ STACK_CREATION_DATE_DELTA=$((${STACK_CREATION_END_DATE} - ${STACK_CREATION_START_DATE}))
+ if [ ${STACK_STATUS} == "CREATE_COMPLETE" ]
+ then
+ APP_URL=$(openstack stack show ${HEAT_STACK_PREFIX}-${ITERATION_NUM} -f value -c outputs | jq -cr '.[] | select(.output_key | contains("app_url")) | .output_value')
+ # TODO implement awaiting for successful first curl
+ sleep 30
+ # Make sure that the app is already available
+ curl -m 60 -s -o /dev/null ${APP_URL}
+ EXIT_CODE=$(echo $?)
+ if [ ${EXIT_CODE} != 0 ]
+ then
+ echo "curl command can't reach ${APP_URL} from ${HEAT_STACK_PREFIX}-${ITERATION_NUM} stack. Exit code ${EXIT_CODE}" >> ${LOG_FILE}
+ fi
+
+ UNIQ_LB_MEMBERS_COUNT=0
+ RETRY_NUMBER=0
+ while (( ${UNIQ_LB_MEMBERS_COUNT} != 3 && ${RETRY_NUMBER} <= 600 ))
+ do
+ UNIQ_LB_MEMBERS_COUNT=$(for LB_MEMBER in {1..3}; do curl -m 10 -s ${APP_URL} | jq -r .host; done | sort | uniq | wc -l)
+ RETRY_NUMBER=$((${RETRY_NUMBER} + 1))
+ sleep 1
+ done
+ if [ ${UNIQ_LB_MEMBERS_COUNT} != 3 ]
+ then
+ echo "$(date) There are only ${UNIQ_LB_MEMBERS_COUNT} LB members instead of 3 for ${APP_URL} LB endpoint from ${HEAT_STACK_PREFIX}-${ITERATION_NUM} stack." >> ${LOG_FILE}
+ fi
+
+ APP_REQUEST_TIME_ROOT=$(curl -m 10 -s -w %{time_total} -o /dev/null ${APP_URL})
+ EXIT_CODE=$(echo $?)
+ if [ ${EXIT_CODE} != 0 ]
+ then
+ echo "curl command can't reach ${APP_URL} from ${HEAT_STACK_PREFIX}-${ITERATION_NUM} stack. Exit code ${EXIT_CODE}" >> ${LOG_FILE}
+ fi
+ APP_REQUEST_TIME_POST_RECORD=$(curl -m 10 -s -w %{time_total} -X POST -H "Content-Type: application/json" --data '{"record": {"data": "spam"}}' -o /dev/null ${APP_URL}/records)
+ EXIT_CODE=$(echo $?)
+ if [ ${EXIT_CODE} != 0 ]
+ then
+ echo "curl command can't reach ${APP_URL}/records with POST requst from ${HEAT_STACK_PREFIX}-${ITERATION_NUM} stack. Exit code ${EXIT_CODE}" >> ${LOG_FILE}
+ fi
+ APP_REQUEST_TIME_GET_RECORDS=$(curl -m 10 -s -w %{time_total} -o /dev/null ${APP_URL}/records)
+ EXIT_CODE=$(echo $?)
+ if [ ${EXIT_CODE} != 0 ]
+ then
+ echo "curl command can't reach ${APP_URL}/records from ${HEAT_STACK_PREFIX}-${ITERATION_NUM} stack. Exit code ${EXIT_CODE}" >> ${LOG_FILE}
+ fi
+ APP_REQUEST_TIME_GET_RECORD=$(curl -m 10 -s -w %{time_total} -o /dev/null ${APP_URL}/record/1)
+ EXIT_CODE=$(echo $?)
+ if [ ${EXIT_CODE} != 0 ]
+ then
+ echo "curl command can't reach ${APP_URL}/records/1 from ${HEAT_STACK_PREFIX}-${ITERATION_NUM} stack. Exit code ${EXIT_CODE}" >> ${LOG_FILE}
+ fi
+ # Write the results to CSV
+ echo "$(date);ITERATION-${ITERATION_NUM};${STACK_CREATION_DATE_DELTA};${APP_REQUEST_TIME_ROOT};${APP_REQUEST_TIME_POST_RECORD};${APP_REQUEST_TIME_GET_RECORDS};${APP_REQUEST_TIME_GET_RECORD}" >> ${DATA_FILE}
+ else
+ echo "$(date);ITERATION-${ITERATION_NUM};${STACK_STATUS}" >> ${DATA_FILE}
+ fi
+ done
+ TEST_END_DATE=$(date +%s)
+ TEST_DURATION=$((TEST_END_DATE - TEST_START_DATE))
+ echo "$(date) Full test duration is ${TEST_DURATION} seconds" >> ${LOG_FILE}
+}
+
+main() {
+ start_test
+}
+
+main
\ No newline at end of file