fix config file examples, houskeeping, etc
diff --git a/FIXME b/FIXME
deleted file mode 100644
index 45def18..0000000
--- a/FIXME
+++ /dev/null
@@ -1,18 +0,0 @@
-Проверить работу вольюмов
-Чего такого с port fw
-python 2.6
-Почему тайминги некорректные
-Копировать в папку оригинальный конфиг
-реюз вольюмс сделать
-
-assumption_check.py
- почти все криво
-
-charts.py
- 1) генерировать картинки с фиксированными именами
-
-report.py
- украсить
-
-rest_api.py
- переписать на prest
diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 0000000..776d6cd
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1,23 @@
+Installation
+------------
+
+Only Redhat/Ubuntu/Debian distros supported
+
+For small installation (test inly)
+
+ # ./install.sh
+
+For full installation (test + html report)
+
+ # ./insall.sh --full
+
+
+Manual installation:
+
+Install : pip, python-openssl python-novaclient python-cinderclient
+python-keystoneclient python-glanceclient python-faulthandler,
+python-scipy python-numpy python-matplotlib python-psutil
+
+Then run
+
+ # pip install -r requirements.txt
diff --git a/README.md b/README.md
index 80469a8..96eeda7 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,4 @@
Block storage devices tests tool. See wiki for details
+
+Look into config-example for examples of config file.
+Copy example in same folder asn replace ${VAR} with appropriate value
diff --git a/TODO b/TODO.ru
similarity index 81%
rename from TODO
rename to TODO.ru
index 5848068..a54daa2 100644
--- a/TODO
+++ b/TODO.ru
@@ -39,3 +39,23 @@
расчет смешанных IOPS
+Проверить работу вольюмов
+Чего такого с port fw
+python 2.6
+Почему тайминги некорректные
+Копировать в папку оригинальный конфиг
+реюз вольюмс сделать
+
+assumption_check.py
+ почти все криво
+
+charts.py
+ 1) генерировать картинки с фиксированными именами
+
+report.py
+ украсить
+
+rest_api.py
+ переписать на prest
+
+
diff --git a/ceph_raw_example.yaml b/ceph_raw_example.yaml
deleted file mode 100644
index 646945f..0000000
--- a/ceph_raw_example.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-clouds:
- ceph: ssh://USER@ANY_CEPH_NODE::KEY_FILE
-
-discover: ceph
-
-explicit_nodes:
- "ssh://TEST_USER@TESTNODE1::KEY_FILE": testnode
- "ssh://TEST_USER@TESTNODE2::KEY_FILE": testnode
- "ssh://TEST_USER@TESTNODE3::KEY_FILE": testnode
-
-sensors:
- roles_mapping:
- testnode: system-cpu, block-io, net-io
- ceph-osd: system-cpu, block-io, net-io
-
-tests:
- - io:
- cfg: ceph
- params:
- FILENAME: /dev/rbd/rbd/RBD_DEV_NAME_OR_LINK
- NUM_ROUNDS: 7
-
-internal:
- var_dir_root: /tmp/perf_tests
-
-logging:
- extra_logs: 1
diff --git a/configs-examples/ceph_fuel.yaml b/configs-examples/ceph_fuel.yaml
index 7018e86..0e1fb7b 100644
--- a/configs-examples/ceph_fuel.yaml
+++ b/configs-examples/ceph_fuel.yaml
@@ -1,43 +1,24 @@
+include: default.yaml
+lab_name: ${LAB_NAME}
+
clouds:
fuel:
- url: http://FUEL_WEB_IP:8000/
- creds: USER:PASSWD@TENANT #admin:admin@admin
- ssh_creds: root:FUEL_SSH_PASSWD
- openstack_env: OPENSTACK_ENV_NAME
+ url: http://${FUEL_WEBUI_IP}:8000/
+
+ # fuel UI creds
+ creds: ${USER}:${PASSWD}@${TENANT} #admin:admin@admin
+
+ # fuel master ssh passwd
+ ssh_creds: root:${FUEL_SSH_PASSWD}
+
+ # openstack environment
+ openstack_env: ${OPENSTACK_ENV_NAME}
discover: fuel_openrc_only
-internal:
- var_dir_root: /tmp/perf_tests
-
-logging:
- extra_logs: 1
-
-vm_configs:
- wally_1024:
- image:
- name: wally_ubuntu
- url: https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
- creds: "ssh://ubuntu@{ip}::{private_key_path}"
-
- flavor:
- name: wally_1024
- hdd_size: 50
- ram_size: 1024
- cpu_count: 1
-
- vol_sz: 30
- keypair_name: wally_vm_key
- name_templ: wally-{group}-{id}
- aa_group_name: wally-aa-{0}
- security_group: wally_ssh_to_everyone
-
tests:
- start_test_nodes:
openstack:
- # creds: /tmp/perf_tests/personalistic_sharron/test_openrc
- # creds: ENV
- creds: clouds
count: x1
cfg_name: wally_1024
network_zone_name: net04
@@ -48,4 +29,4 @@
cfg: ceph
params:
FILENAME: /dev/vdb
- NUM_ROUNDS: 7
+ TEST_FILE_SIZE: 100G
diff --git a/configs-examples/ceph_raw.yaml b/configs-examples/ceph_raw.yaml
index 646945f..9520a2f 100644
--- a/configs-examples/ceph_raw.yaml
+++ b/configs-examples/ceph_raw.yaml
@@ -1,12 +1,14 @@
+include: default.yaml
+
clouds:
- ceph: ssh://USER@ANY_CEPH_NODE::KEY_FILE
+ ceph: ssh://${USER}@${ANY_CEPH_NODE}::${KEY_FILE}
discover: ceph
explicit_nodes:
- "ssh://TEST_USER@TESTNODE1::KEY_FILE": testnode
- "ssh://TEST_USER@TESTNODE2::KEY_FILE": testnode
- "ssh://TEST_USER@TESTNODE3::KEY_FILE": testnode
+ "ssh://${TEST_USER}@${TESTNODE1}::${KEY_FILE}": testnode
+ "ssh://${TEST_USER}@${TESTNODE2}::${KEY_FILE}": testnode
+ "ssh://${TEST_USER}@${TESTNODE3}::${KEY_FILE}": testnode
sensors:
roles_mapping:
@@ -17,11 +19,5 @@
- io:
cfg: ceph
params:
- FILENAME: /dev/rbd/rbd/RBD_DEV_NAME_OR_LINK
- NUM_ROUNDS: 7
-
-internal:
- var_dir_root: /tmp/perf_tests
-
-logging:
- extra_logs: 1
+ FILENAME: /dev/rbd/rbd/${RBD_DEV_NAME_OR_LINK}
+ FILESIZE: ${FILESIZE}
diff --git a/configs-examples/db_raw.yaml b/configs-examples/db_raw.yaml
index 10104be..9d74c17 100644
--- a/configs-examples/db_raw.yaml
+++ b/configs-examples/db_raw.yaml
@@ -1,5 +1,7 @@
+include: default.yaml
+
explicit_nodes:
- "ssh://root@192.168.59.104::/home/yportnova/.ssh/id_rsa": testnode
+ "ssh://root@${NODE_IP}::${KEY_NAME}": testnode
tests:
- pgbench:
@@ -7,9 +9,3 @@
transactions_per_client: 2
- mysql:
warehouses: 20
-
-internal:
- var_dir_root: /tmp/perf_tests
-
-logging:
- extra_logs: 1
diff --git a/configs-examples/default.yaml b/configs-examples/default.yaml
new file mode 100644
index 0000000..8750bc3
--- /dev/null
+++ b/configs-examples/default.yaml
@@ -0,0 +1,27 @@
+settings:
+ collect_info: false
+ suspend_unused_vms: false
+ results_storage: /var/wally_results
+ log_level: DEBUG
+
+vm_configs:
+ keypair_file_private: wally_vm_key.pem
+ keypair_file_public: wally_vm_key.pub
+ keypair_name: wally_vm_key
+
+ wally_1024:
+ image:
+ name: wally_ubuntu
+ url: https://cloud-images.ubuntu.com/trusty/current/trusty-server-cloudimg-amd64-disk1.img
+ creds: "ssh://ubuntu@{ip}::{private_key_path}"
+
+ flavor:
+ name: wally_1024
+ hdd_size: 100
+ ram_size: 1024
+ cpu_count: 2
+
+ vol_sz: 100
+ name_templ: wally-{group}-{id}
+ aa_group_name: wally-aa-{0}
+ security_group: wally_ssh_to_everyone
diff --git a/configs-examples/hdd.yaml b/configs-examples/hdd.yaml
index dcb5f9e..d38904a 100644
--- a/configs-examples/hdd.yaml
+++ b/configs-examples/hdd.yaml
@@ -1,17 +1,11 @@
+include: default.yaml
+
explicit_nodes:
local: testnode
-internal:
- var_dir_root: /tmp/perf_tests
-
-testnode_log_root: /tmp/perf_tests_rem
-
tests:
- io:
cfg: hdd
params:
- FILENAME: FILE_ON_USB_HDD
- NUM_ROUNDS: 7
-
-logging:
- extra_logs: 1
\ No newline at end of file
+ FILENAME: ${FILEPATH}
+ FILESIZE: ${FILESIZE}
diff --git a/configs-examples/local_ceph.yaml b/configs-examples/local_ceph.yaml
index 613894d..519f094 100644
--- a/configs-examples/local_ceph.yaml
+++ b/configs-examples/local_ceph.yaml
@@ -1,3 +1,5 @@
+include: default.yaml
+
clouds:
ceph: local
@@ -6,9 +8,6 @@
explicit_nodes:
local: testnode
-internal:
- var_dir_root: /tmp/perf_tests
-
sensors:
roles_mapping:
testnode: system-cpu, block-io
@@ -17,8 +16,5 @@
- io:
cfg: ceph
params:
- FILENAME: CEPH_RBD_DEVICE
- NUM_ROUNDS: 7
-
-logging:
- extra_logs: 1
+ FILENAME: ${CEPH_RBD_DEVICE}
+ FILESIZE: ${FILESIZE}
diff --git a/install.sh b/install.sh
new file mode 100755
index 0000000..1fd97b4
--- /dev/null
+++ b/install.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+FULL="$1"
+
+function install_apt() {
+ apt-get install -y python-openssl python-novaclient python-cinderclient \
+ python-keystoneclient python-glanceclient python-faulthandler \
+ python-pip
+
+ if [ "$FULL" == "--full" ] ; then
+ apt-get install -y python-scipy python-numpy python-matplotlib python-psutil
+ fi
+}
+
+
+function install_yum() {
+ yum -y install pyOpenSSL python-novaclient python-cinderclient \
+ python-keystoneclient python-glanceclient
+ python-pip
+
+ if [ "$FULL" == "--full" ] ; then
+ yum -y install scipy numpy python-matplotlib python-psutil
+ fi
+}
+
+if which apt-get >/dev/null; then
+ install_apt
+else
+ if which yum >/dev/null; then
+ install_yum
+ else
+ echo "Error: Neither apt-get, not yum installed. Can't install deps"
+ exit 1
+ fi
+fi
+
+pip install -r requirements.txt
+
+if [ "$FULL" == "--full" ] ; then
+ pip install oktest iso8601==0.1.10
+fi
diff --git a/requirements-full.txt b/requirements-full.txt
deleted file mode 100644
index 6b4d8be..0000000
--- a/requirements-full.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-oktest
-faulthandler
-python-glanceclient
-iso8601==0.1.10
-scipy
-numpy
-mathplotlib
-psutil
diff --git a/requirements-web.txt b/requirements-web.txt
deleted file mode 100644
index 83a13ad..0000000
--- a/requirements-web.txt
+++ /dev/null
@@ -1,36 +0,0 @@
-argparse==1.2.1
-Babel==1.3
-decorator==3.4.0
-Flask-Bootstrap==3.3.0.1
-Flask-SQLAlchemy==2.0
-Flask==0.10.1
-futures==2.2.0
-GChartWrapper==0.8
-iso8601==0.1.10
-itsdangerous==0.24
-Jinja2==2.7.3
-MarkupSafe==0.23
-netaddr==0.7.13
-oslo.config==1.6.1
-oslo.i18n==1.4.0
-oslo.serialization==1.3.0
-oslo.utils==1.3.0
-paramiko==1.15.2
-pbr==0.10.7
-prettytable==0.7.2
-pyOpenSSL==0.14
-python-cinderclient==1.1.1
-python-glanceclient==0.15.0
-python-keystoneclient==1.1.0
-python-novaclient==2.20.0
-requests==2.2.1
-simplejson==3.6.5
-six==1.9.0
-sqlalchemy-migrate==0.9.4
-SQLAlchemy==0.9.8
-sqlparse==0.1.14
-stevedore==1.2.0
-Tempita==0.5.2
-warlock==1.1.0
-Werkzeug==0.10.1
-wsgiref==0.1.2
diff --git a/requirements.txt b/requirements.txt
index 680a364..b7da062 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,14 +5,8 @@
petname
prest
prettytable
-
-pyOpenSSL
-python-novaclient
-python-cinderclient
-python-keystoneclient
-
PyYAML
requests
simplejson
-sshtunnel
+sshtunnel==0.0.3
texttable
diff --git a/run.py b/run.py
deleted file mode 100644
index 0fe4373..0000000
--- a/run.py
+++ /dev/null
@@ -1,4 +0,0 @@
-import sys
-from wally.run_test import main
-
-exit(main(sys.argv))
diff --git a/wally/run_test.py b/wally/run_test.py
index 487fd49..edf58fa 100755
--- a/wally/run_test.py
+++ b/wally/run_test.py
@@ -322,6 +322,7 @@
def get_OS_credentials(cfg, ctx):
creds = None
os_creds = None
+ force_insecure = False
if 'openstack' in cfg.clouds:
os_cfg = cfg.clouds['openstack']
@@ -340,6 +341,9 @@
os_cfg['OS_AUTH_URL'].strip(),
os_cfg.get('OS_INSECURE', False))
+ elif 'OS_INSECURE' in os_cfg:
+ force_insecure = os_cfg.get('OS_INSECURE', False)
+
if os_creds is None and 'fuel' in cfg.clouds and \
'openstack_env' in cfg.clouds['fuel'] and \
ctx.fuel_openstack_creds is not None:
@@ -352,6 +356,13 @@
if creds is None:
creds = os_creds
+ if force_insecure and not creds.insecure:
+ creds = start_vms.OSCreds(creds.name,
+ creds.passwd,
+ creds.tenant,
+ creds.auth_url,
+ True)
+
logger.debug(("OS_CREDS: user={0.name} tenant={0.tenant}" +
"auth_url={0.auth_url} insecure={0.insecure}").format(creds))
diff --git a/wally/suits/io/fio.py b/wally/suits/io/fio.py
index 777ae5f..c009c3d 100644
--- a/wally/suits/io/fio.py
+++ b/wally/suits/io/fio.py
@@ -442,8 +442,10 @@
cmd = "sudo " + cmd
zero_md5 = '0f343b0931126a20f133d67c2b018a3b'
- offsets = [random.randrange(size - 1024) for _ in range(num_blocks)]
- offsets.append(size - 1024)
+ bsize = size * (1024 ** 2)
+ offsets = [random.randrange(bsize - 1024) for _ in range(num_blocks)]
+ offsets.append(bsize - 1024)
+ offsets.append(0)
for offset in offsets:
data = rossh(cmd.format(fname, offset), nolog=True)
@@ -574,29 +576,33 @@
list(pool.map(fc, self.config.nodes))
def pre_run_th(self, node, files, force):
- # fill files with pseudo-random data
- rossh = run_on_node(node)
- rossh.connection = node.connection
-
try:
- cmd = 'mkdir -p "{0}"'.format(self.config.remote_dir)
- if self.use_sudo:
- cmd = "sudo " + cmd
- cmd += " ; sudo chown {0} {1}".format(node.get_user(),
- self.config.remote_dir)
- rossh(cmd, nolog=True)
+ # fill files with pseudo-random data
+ rossh = run_on_node(node)
+ rossh.connection = node.connection
- assert self.config.remote_dir != "" and self.config.remote_dir != "/"
- rossh("rm -rf {0}/*".format(self.config.remote_dir), nolog=True)
+ try:
+ cmd = 'mkdir -p "{0}"'.format(self.config.remote_dir)
+ if self.use_sudo:
+ cmd = "sudo " + cmd
+ cmd += " ; sudo chown {0} {1}".format(node.get_user(),
+ self.config.remote_dir)
+ rossh(cmd, nolog=True)
- except Exception as exc:
- msg = "Failed to create folder {0} on remote {1}. Error: {2!s}"
- msg = msg.format(self.config.remote_dir, node.get_conn_id(), exc)
- logger.exception(msg)
- raise StopTestError(msg, exc)
+ assert self.config.remote_dir != "" and self.config.remote_dir != "/"
+ rossh("rm -rf {0}/*".format(self.config.remote_dir), nolog=True)
- self.install_utils(node, rossh)
- self.prefill_test_files(rossh, files, force)
+ except Exception as exc:
+ msg = "Failed to create folder {0} on remote {1}. Error: {2!s}"
+ msg = msg.format(self.config.remote_dir, node.get_conn_id(), exc)
+ logger.exception(msg)
+ raise StopTestError(msg, exc)
+
+ self.install_utils(node, rossh)
+ self.prefill_test_files(rossh, files, force)
+ except:
+ logger.exception("XXXX")
+ raise
def show_test_execution_time(self):
if len(self.fio_configs) > 1: