update scripts, prepare for fixing MOL-180
diff --git a/scripts/config.sh b/scripts/config.sh
new file mode 100644
index 0000000..2283eee
--- /dev/null
+++ b/scripts/config.sh
@@ -0,0 +1,12 @@
+FLAVOR_NAME="disk_io_perf.256"
+SERV_GROUP="disk_io_perf.aa"
+KEYPAIR_NAME="disk_io_perf"
+IMAGE_NAME="disk_io_perf"
+KEY_FILE_NAME="${KEYPAIR_NAME}.pem"
+IMAGE_URL="https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img"
+IMAGE_USER="ubuntu"
+NETWORK_ZONE_NAME="net04"
+FL_NETWORK_ZONE_NAME="net04_ext"
+VM_COUNT="x1"
+TESTER_TYPE="iozone"
+RUNNER="ssh"
diff --git a/scripts/data_extractor.py b/scripts/data_extractor.py
index d655f17..d3ba1ea 100644
--- a/scripts/data_extractor.py
+++ b/scripts/data_extractor.py
@@ -2,6 +2,7 @@
import json
import sqlite3
import contextlib
+from utils import ssize_to_kb
def connect(url):
@@ -130,23 +131,6 @@
json_to_db(json_data, conn)
-def ssize_to_kb(ssize):
- try:
- smap = dict(k=1, K=1, M=1024, m=1024, G=1024**2, g=1024**2)
- for ext, coef in smap.items():
- if ssize.endswith(ext):
- return int(ssize[:-1]) * coef
-
- if int(ssize) % 1024 != 0:
- raise ValueError()
-
- return int(ssize) / 1024
-
- except (ValueError, TypeError, AttributeError):
- tmpl = "Unknow size format {0!r} (or size not multiples 1024)"
- raise ValueError(tmpl.format(ssize))
-
-
def load_slice(cursor, build_id, y_param, **params):
params_id = {}
for param in list(params) + [y_param]:
diff --git a/scripts/prepare.sh b/scripts/prepare.sh
new file mode 100644
index 0000000..3d85072
--- /dev/null
+++ b/scripts/prepare.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+set -e
+
+my_dir="$(dirname "$0")"
+source "$my_dir/config.sh"
+
+# settings
+FL_RAM=256
+FL_HDD=50
+FL_CPU=1
+
+
+function lookup_for_objects() {
+ set +e
+
+ echo -n "Looking for image $IMAGE_NAME ... "
+ export img_id=$(nova image-list | grep " $IMAGE_NAME " | awk '{print $2}')
+ if [ ! -z "$img_id" ] ; then
+ echo " Found"
+ else
+ echo " Not Found"
+ fi
+
+ echo -n "Looking for flavor $FLAVOR_NAME ... "
+ export flavor_id=$(nova flavor-list | grep " $FLAVOR_NAME " | awk '{print $2}')
+ if [ ! -z "$flavor_id" ] ; then
+ echo " Found"
+ else
+ echo " Not Found"
+ fi
+
+ echo -n "Looking for server-group $SERV_GROUP ... "
+ export group_id=$(nova server-group-list | grep " $SERV_GROUP " | awk '{print $2}' )
+ if [ ! -z "$group_id" ] ; then
+ echo " Found"
+ else
+ echo " Not Found"
+ fi
+
+ echo -n "Looking for keypair $KEYPAIR_NAME ... "
+ export keypair_id=$(nova keypair-list | grep " $KEYPAIR_NAME " | awk '{print $2}' )
+ if [ ! -z "$keypair_id" ] ; then
+ echo " Found"
+ else
+ echo " Not Found"
+ fi
+
+ set -e
+}
+
+function clean() {
+ lookup_for_objects
+
+ if [ ! -z "$img_id" ] ; then
+ echo "Deleting $IMAGE_NAME image"
+ nova image-delete "$img_id" >/dev/null
+ fi
+
+ if [ ! -z "$flavor_id" ] ; then
+ echo "Deleting $FLAVOR_NAME flavor"
+ nova flavor-delete "$flavor_id" >/dev/null
+ fi
+
+ if [ ! -z "$group_id" ] ; then
+ echo "Deleting server-group $SERV_GROUP"
+ nova server-group-delete "$group_id" >/dev/null
+ fi
+
+ if [ ! -z "$keypair_id" ] ; then
+ echo "deleting keypair $KEYPAIR_NAME"
+ nova keypair-delete "$KEYPAIR_NAME" >/dev/null
+ fi
+
+ if [ -f "$KEY_FILE_NAME" ] ; then
+ echo "deleting keypair file $KEY_FILE_NAME"
+ rm -f "$KEY_FILE_NAME"
+ fi
+}
+
+function prepare() {
+ lookup_for_objects
+
+ if [ -z "$img_id" ] ; then
+ echo "Creating $IMAGE_NAME image"
+ opts="--disk-format qcow2 --container-format bare --is-public true"
+ glance image-create --name "$IMAGE_NAME" $opts --copy-from "$IMAGE_URL" >/dev/null
+ echo "Image created, but may need a time to became active"
+ fi
+
+ if [ -z "$flavor_id" ] ; then
+ echo "Creating flavor $FLAVOR_NAME"
+ nova flavor-create "$FLAVOR_NAME" "$FLAVOR_NAME" "$FL_RAM" "$FL_HDD" "$FL_CPU" >/dev/null
+ fi
+
+ if [ -z "$group_id" ] ; then
+ echo "Creating server group $SERV_GROUP"
+ nova server-group-create --policy anti-affinity "$SERV_GROUP" >/dev/null
+ fi
+
+ if [ -z "$keypair_id" ] ; then
+ echo "Creating server group $SERV_GROUP. Key would be stored into $KEY_FILE_NAME"
+ nova keypair-add "$KEYPAIR_NAME" > "$KEY_FILE_NAME"
+ fi
+
+ echo "Adding rules for ping and ssh"
+ set +e
+ nova secgroup-add-rule default icmp -1 -1 0.0.0.0/0 >/dev/null
+ nova secgroup-add-rule default tcp 22 22 0.0.0.0/0 >/dev/null
+ set -e
+}
+
+if [ "$1" = "--clear" ] ; then
+ clean
+else
+ prepare
+fi
diff --git a/scripts/run.sh b/scripts/run.sh
index 11c90ee..368410c 100644
--- a/scripts/run.sh
+++ b/scripts/run.sh
@@ -1,85 +1,109 @@
#!/bin/bash
set -e
-type="iozone"
+# don't forget to change it in prepare.sh as well
+my_dir="$(dirname "$0")"
+source "$my_dir/config.sh"
-bsizes="1k 4k 64k 256k 1m"
-ops="randwrite"
-osync="s" # a
-num_times="3"
-concurrences="32"
-
-for concurrence in $concurrences; do
- for bsize in $bsizes ; do
- for op in $ops ; do
- for sync in $osync ; do
- for counter in $(seq 1 $num_times) ; do
- # if [[ "$ops" == "write" && "$osync" == "s" ]] ; then
- # continue
- # fi
-
- if [[ "$sync" == "s" ]] ; then
- ssync="-s"
- factor="x500"
- else
- if [[ "$bsize" == "1k" || "$bsize" == "4k" ]] ; then
- continue
- fi
-
- ssync=
- factor="r2"
- fi
+BLOCK_SIZES="1k 4k 64k 256k 1m"
+OPERATIONS="randwrite write randread read"
+SYNC_TYPES="s a d"
+REPEAT_COUNT="3"
+CONCURRENCES="1 8 64"
+IODEPTHS="16"
- io_opts="--type $type -a $op --iodepth 16 --blocksize $bsize --iosize $factor $ssync --concurrency $concurrence"
+SYNC_FACTOR="x500"
+DIRECT_FACTOR="x500"
+ASYNC_FACTOR="r2"
- # aff_group=$(nova server-group-list | grep ' ceph ' | awk '{print $2}')
- # extra_opts="user=ubuntu,keypair_name=ceph,img_name=ubuntu,flavor_name=ceph.512"
- # extra_opts="${extra_opts},network_zone_name=net04,flt_ip_pool=net04_ext,key_file=ceph.pem"
- # extra_opts="${extra_opts},aff_group=${aff_group},count=x1"
- echo $io_opts
- # python run_test.py --runner ssh -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="$extra_opts"
+function get_file_size_opts() {
+ SYNC_TYPE="$1"
+ if [ "$SYNC_TYPE" = "s" ] ; then
+ echo "--iosize $SYNC_FACTOR -s"
+ elif [ "$SYNC_TYPE" = "d" ] ; then
+ echo "--iosize $DIRECT_FACTOR -d"
+ else
+ echo "--iosize $ASYNC_FACTOR"
+ fi
+}
- # io_opts="--type $type -a $op --iodepth 16 --blocksize $bsize --iosize $factor $ssync"
- # python run_test.py --runner rally -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="--deployment $1"
- done
- done
- done
- done
-done
+function echo_combinations() {
+ for IODEPTH in $IODEPTHS ; do
+ for CONCURRENCE in $CONCURRENCES ; do
+ for BSIZE in $BLOCK_SIZES ; do
+ for OPERATION in $OPERATIONS ; do
+ for SYNC_TYPE in $SYNC_TYPES ; do
-# bsizes="4k 64k 256k 1m"
-# ops="randread read"
+ # filter out too slow options
+ if [ "$BLOCKSIZE" = "1k" -o "$BLOCK_SIZE" = "4k" ] ; then
+ if [ "$SYNC_TYPE" = "a" ] ; then
+ continue
+ fi
+ fi
-# for bsize in $bsizes ; do
-# for op in $ops ; do
-# for xxx in $three_times ; do
-# io_opts="--type $type -a $op --iodepth 16 --blocksize $bsize --iosize r2"
-# python run_rally_test.py -l -o "$io_opts" -t io-scenario $type --rally-extra-opts="--deployment $1"
-# done
-# done
-# done
+ # filter out sync reads
+ if [ "$OPERATION" = "read" -o "$OPERATION" = "randread" ] ; then
+ if [ "$SYNC_TYPE" = "s" ] ; then
+ continue
+ fi
+ fi
-# bsizes="1k 4k"
-# ops="randwrite write"
-# three_times="1 2 3"
+ FILE_SIZE_AND_SYNC=$(get_file_size_opts "$SYNC_TYPE")
-# for bsize in $bsizes ; do
-# for op in $ops ; do
-# for xxx in $three_times ; do
-# factor="r2"
-# io_opts="--type $type -a $op --iodepth 16 --blocksize $bsize --iosize $factor"
-# python run_rally_test.py -l -o "$io_opts" -t io-scenario $type --rally-extra-opts="--deployment $1"
-# done
-# done
-# done
-# ops="randread read"
+ IO_OPTS="--type $TESTER_TYPE "
+ IO_OPTS="$IO_OPTS -a $OPERATION "
+ IO_OPTS="$IO_OPTS --iodepth $IODEPTH "
+ IO_OPTS="$IO_OPTS --blocksize $BSIZE "
+ IO_OPTS="$IO_OPTS $FILE_SIZE_AND_SYNC "
+ IO_OPTS="$IO_OPTS --concurrency $CONCURRENCE"
-# for op in $ops ; do
-# for xxx in $three_times ; do
-# io_opts="--type $type -a $op --iodepth 16 --blocksize 1k --iosize r2"
-# python run_rally_test.py -l -o "$io_opts" -t io-scenario $type --rally-extra-opts="--deployment $1"
-# done
-# done
+ for COUNTER in $(seq 1 $REPEAT_COUNT) ; do
+ echo $IO_OPTS
+ done
+ done
+ done
+ done
+ done
+ done
+}
+
+
+function run_test() {
+ OPTION_FILE="$1"
+
+ if [ ! -f "$OPTION_FILE" ] ; then
+ echo "Path to file with io.py options list should be passed"
+ exit 1
+ fi
+
+ if [ "$RUNNER" = "ssh" ] ; then
+ GROUP_ID=$(nova server-group-list | grep " $SERV_GROUP " | awk '{print $2}' )
+ EXTRA_OPTS="user=$IMAGE_USER"
+ EXTRA_OPTS="${EXTRA_OPTS},keypair_name=$KEYPAIR_NAME"
+ EXTRA_OPTS="${EXTRA_OPTS},img_name=$IMAGE_NAME"
+ EXTRA_OPTS="${EXTRA_OPTS},flavor_name=$FLAVOR_NAME"
+ EXTRA_OPTS="${EXTRA_OPTS},network_zone_name=$NETWORK_ZONE_NAME"
+ EXTRA_OPTS="${EXTRA_OPTS},flt_ip_pool=$FL_NETWORK_ZONE_NAME"
+ EXTRA_OPTS="${EXTRA_OPTS},key_file=$KEY_FILE_NAME"
+ EXTRA_OPTS="${EXTRA_OPTS},aff_group=$GROUP_ID"
+ EXTRA_OPTS="${EXTRA_OPTS},count=$VM_COUNT"
+ else
+ echo "Unsupported runner $RUNNER"
+ exit 1
+ fi
+
+ RUN_TEST_OPTS="-t io-scenario -l --runner $RUNNER"
+ set -x
+ python run_test.py $RUN_TEST_OPTS --runner-extra-opts="$EXTRA_OPTS" -f "$OPTION_FILE" $TESTER_TYPE
+ set +x
+}
+
+if [ "$1" = '--prepare-opts' ] ; then
+ echo_combinations
+else
+ run_test $1
+fi
+
diff --git a/scripts/run_2.sh b/scripts/run_2.sh
deleted file mode 100644
index cdf29dc..0000000
--- a/scripts/run_2.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/bash
-set -x
-
-function prepare() {
- nova image-list | grep ' ubuntu ' >/dev/null
- if [ $? -ne 0 ] ; then
- url="https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img"
- glance image-create --name 'ubuntu' --disk-format qcow2 --container-format bare --is-public true --copy-from $url
- fi
-
- nova flavor-list | grep ' ceph.512 ' >/dev/null
- if [ $? -ne 0 ] ; then
- nova flavor-create ceph.512 ceph.512 512 50 1
- fi
-
- nova server-group-list | grep ' ceph ' >/dev/null
- if [ $? -ne 0 ] ; then
- nova server-group-create --policy anti-affinity ceph
- fi
-
- nova keypair-list | grep ' ceph ' >/dev/null
- if [ $? -ne 0 ] ; then
- nova keypair-add ceph > ceph.pem
- fi
-
- nova secgroup-add-rule default icmp -1 -1 0.0.0.0/0
- nova secgroup-add-rule default tcp 22 22 0.0.0.0/0
-}
-
-
-function run_test() {
- set -e
-
- iodepts="1"
- for iodepth in $iodepts; do
- extra_opts="user=ubuntu,keypair_name=ceph,img_name=ubuntu,flavor_name=ceph.512"
- extra_opts="${extra_opts},network_zone_name=net04,flt_ip_pool=net04_ext,key_file=ceph.pem"
- extra_opts="${extra_opts},aff_group=${aff_group},count=x1"
-
- io_opts="--type $type -a write --iodepth 16 --blocksize 1m --iosize x20"
- python run_test.py --runner ssh -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="$extra_opts"
- done
-
- # io_opts="--type $type -a write --iodepth 16 --blocksize 1m --iosize x20"
- # python run_test.py --runner rally -l -o "$io_opts" -t io-scenario $type --runner-extra-opts="--deployment perf-1"
-}
-
-type="iozone"
-prepare
-# run_test
-# nova server-group-list | grep ' ceph ' | awk '{print $2}'
-# aff_group="0077d59c-bf5b-4326-8940-027e77d655ee"
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
deleted file mode 100755
index f80e5be..0000000
--- a/scripts/run_tests.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-
-set -x
-set -e
-
-CMD1="--type iozone -a write --iodepth 8 --blocksize 4k --iosize 40M -s"
-CMD2="--type fio -a write --iodepth 8 --blocksize 4k --iosize 4M -s"
-
-python run_rally_test.py -l -o "$CMD1" -t io-scenario iozone 2>&1 | tee ceph_results.txt
-python run_rally_test.py -l -o "$CMD2" -t io-scenario fio 2>&1 | tee -a ceph_results.txt
-