v2.0 started
diff --git a/AUTHORS b/AUTHORS
new file mode 100644
index 0000000..10c090c
--- /dev/null
+++ b/AUTHORS
@@ -0,0 +1,10 @@
+Alyona Kiseleva <akiselyova@mirantis.com>
+Dmitry Yatsushkevich <dyatsushkevich@mirantis.com>
+Kostiantyn Danylov aka koder <koder.mail@gmail.com>
+Peter Lomakin <plomakin@mirantis.com>
+Ved-vampir <akiselyova@mirantis.com>
+Yulia Portnova <yportnova@mirantis.com>
+gstepanov <gstepanov@mirantis.com>
+koder aka kdanilov <kdanilov@mirantis.com>
+stgleb <glebstepanov1992@gmail.com>
+yportnova <yportnova@mirantis.com>
diff --git a/ChangeLog b/ChangeLog
new file mode 100644
index 0000000..1a16c91
--- /dev/null
+++ b/ChangeLog
@@ -0,0 +1,269 @@
+CHANGES
+=======
+
+* add deviation to reports
+* Improvements in Dockerfile
+* Dockerfile updates
+* fix config file examples, houskeeping, etc
+* Create docker.md
+* Dockerfile to build tool with dependecies
+* fix problem with old paramiko
+* add insecure openstack connection support2
+* add insecure openstack connection support
+* 2.0 ready
+* fixes
+* temporary commit
+* fix file prefill check
+* fix build comparisio
+* add lat/bw limits, fio binaryes, fix bugs, fix latency calculations, etc
+* v2 is comming
+* tempo commit
+* tempo commit
+* tempo commit
+* tempo commit
+* update TODO
+* fix fio iops/lat/bw file name patterns
+* add mixed load test, other fixes
+* New cpu load option, bottlenecks
+* a lot of changes
+* fix modules and 2.6 compatibility
+
+v0.2
+----
+
+* fix muptiply tests in one run, update report generation
+* a lot of chenges
+* Fixed getting os_auth_url
+* fix tasks
+* a lot of changes
+* a lot of changes
+* a lot of changes
+* small db tests fixes
+* pgbench fixes
+* Mysql tests
+* release preparation
+* fixes in ceph tests, add linearity plot
+* improve OS preparation, change config structure
+* Create README.md
+* report changes
+* fixes
+* fixes
+* a lot of fixes
+* fix sftp exception types
+* fix ceph test ram usage
+* add local sensor datastore, make IO tests granular
+* add during test webiu
+* improve bg. process monitoring
+* Added tpcc-mysql suite
+* changed for new data format
+* sensors correction, postprocessing
+* fixes, fixes, fixes
+* fixes, fixes, fixes
+* fixes, fixes, fixes
+* temporary dir creation fixed
+* itest.py code reorganization, move test node logs to /var/wally folder to survive test node reboot
+* improve paramiko integration, replace .mesage for exceptions with str(exc)
+* fixing issues with bad ssg connection
+* now can read openstack creds from arbitrary openrc file
+* not needed exception removed
+* run tests on nodes in offline mode
+* improvement of protocol refactoring
+* fix bugs and 2.6 compatibility
+* fix python 2.6 compatibility
+* update report
+* fix reports
+* update usb hdd config
+* fix requirements.txt, improve test end time representation
+* start adding unit tests, rework config compiler
+* html report changed
+* more improvements and fixes and new bugs
+* fixing, improve sersors installation code
+* very large refactoring
+* fixing bugs
+* report page table of images added
+* per vm bw on chart
+* fix sensors deploy code
+* lot of fixes
+* Lab info added to report layout
+* Fixed charts
+* remove personal config
+* rename bw_mean to bw, other fixes
+* pre-release bug fixes
+* pre-release updates, bug fixes
+* pre-release updates, bug fixes
+* fix pre-demo bugs
+* change table output for easier postprocessing
+* Added get_openrc to cluster
+* add postprocessing, table difference generation and scipy approximation
+* discovering fixed
+* removed trash
+* Copying openrc from controller node, adding to global config and external ip to config yaml added
+* wqrefactor postprocessing code
+* fixed requirements
+* report.html template
+* Improved io charts
+* pep8 for MOL 305
+* MOL 305, added permanent journal that allows to remove virtual machines that has remained from previous tests that was crashed
+* add mt results example
+* fix multythreaded bugs in disk_test_agent, add linearity data and script
+* MOL 314 config file path added as cmd line parameter
+* MOL 302, build metainfo added to cmd line args
+* MOL 319, connector code integrated in run_test.py
+* single node tests
+* get_greds method in fuel_rest_api.Cluster
+* Fixed pgbench tests
+* postprocessing added (plot and dev)
+* large commit. refactoring, fio code totally reworker, huge improvenent in test time and results, etc
+* Connector for adding/cleaning ip forwarding rules added
+* changes to io.py applied to script, tasks for fio added
+* final refactoring
+* make openstack vm spawn code works
+* large commit. new code, with sensors, line count dropped, etc
+* small refactoring
+* code review
+* run_test refactoring and small fixes
+* code review
+* updates to node complete test, add io_py_result_processor.py lib
+* fixes, add complete test suite
+* automatic test running and sending to webapp
+* Automated test lanucher has been added. Run a bunch of test that create vm, configure it, run test and clean allocated resources
+* some unknown changes
+* update single-node test scripts
+* large refactoring of new code
+* remove old results db
+* single-node OS tests addede
+* add list sensors cmd line options
+* fixed rebase error
+* discovering
+* config.py changed
+* config file name from cmd args added
+* packer option added
+* protocol added
+* returned class changed
+* Node class
+* ceph discovering
+* config moved to yaml, redundant config removed
+* non storage funtions removed to data_processing.py, doc strings added to storage_api
+* average calculation removed
+* percent removed, name changed
+* refactor get_latency
+* nodes discovering
+* fix memory info code
+* graphana integration
+* continue work on new sensor model
+* lost import
+* ps_mem truncation
+* perprocess ram usage added
+* perprocess cpu usage added
+* io_sensor: Add get_latency function
+* system RAM usage sensor added
+* system cpu usage sensor added
+* new sensor manage code
+* update reporing tool
+* removed rally
+* fake run_test
+* a lot of fixes and improvements
+* meta info pep8
+* meta info added
+* pep8 added, lab_id and build_id added as a parameters to storage_api.load_data, additional params added to params combination
+* Created db_manage script to create,drop,upgrade,downgrade,migrate db
+* pgbench run.sh: receive num clients and transactions through params
+* Two scripts, pgbench tests
+* lab info save added
+* pg scripts
+* json returned to api endpoints
+* MOL 186, migration from file storage to SQL Databse
+* add local runner, fix some bugs
+* move files around
+* code review
+* storage code removed from web app file
+* ability to cache images on hard drive added
+* MOL 159, sending test results added, rest api fixed, mergind builds on server changed
+* saving images
+* implements MOL-180: make run_test.py compatible with FUEL master
+* fix small bugs
+* update scripts, prepare for fixing MOL-180
+* fix MOL-177: Fix logging
+* implement MOL-175: automatically select amount of vm
+* implement MOL-169 - Decrease env preparation time (parallelize requests to openstack, ssh connections, etc)
+* Implement MOL-168 - Allow to run set of tests on single vm
+* Changed title, sync/async displaying
+* tests finally can works over pure novaclient/ssh
+* merge of builds added
+* final
+* link changed to build name
+* small fix
+* build name added
+* h1 changed to h2
+* table width smalled
+* layout changed, jquery added, comparision fixed
+* kbps style changed
+* small changes to code layout and html
+* ref to index and dates to builds added
+* fixes in test code. done. ready to swith from rally
+* templates for aggregating added
+* aggregating data from lab nodes added
+* Fixed y axe
+* images in table and particular builds added
+* large refactoring, ready to move away from rally
+* keystone client added
+* collecting data from nodes added
+* Added missing init.py
+* fixed displaying charts
+* fixed displaying charts
+* logger added
+* modified x scale. splitted sync/async
+* config added test path
+* test path changed
+* post fixed
+* fake test added
+* requests added to requirements.txt
+* test add
+* lab description added
+* POST for receiving test results fixed
+* build type master added
+* build names added to test_result.json
+* 500 fix
+* test results added
+* test results added
+* we need MORE MONKEYPATCHING
+* add data extractor
+* add resutls
+* updates to tests
+* lab info page added
+* separate tables and images
+* tables added to webapp
+* added GChartWrapper to requirements
+* server host fixed
+* fixed data generator
+* fixed charts
+* data gen fixed
+* empty test error fixed
+* one more
+* path fixed
+* misc
+* hardcoded path removed
+* web app added
+* fix itertools.production usage
+* storage api changed
+* charts
+* small fix
+* storage get recent bg fixed
+* add terstdata generator
+* add all-on-one shell scripts
+* factory function added
+* factory function for storages added
+* recent builds method added to storage api
+* now can run multiply tests on same vm
+* fix results output
+* fix storage code style, fixes in rally run code
+* local storage and google docs finished
+* storage api added
+* add directory reqursive copying
+* fix erro message
+* add fio support. It's ALIVE
+* add rally patch-and-run script and iozone scenario
+* fix
+* creating worskeet added
+* report maker adder
+* Initial commit
diff --git a/scripts/fio_tests_configs/io_task_test.cfg b/scripts/fio_tests_configs/io_task_test.cfg
index bfb2d96..3df6067 100644
--- a/scripts/fio_tests_configs/io_task_test.cfg
+++ b/scripts/fio_tests_configs/io_task_test.cfg
@@ -1,4 +1,4 @@
-[defaults]
+[global]
group_reporting
wait_for_previous
ramp_time=0
@@ -9,14 +9,14 @@
time_based
RUNTIME=10
-[writetest * {NUM_ROUNDS}]
+[writetest]
blocksize=4k
rw=randwrite
direct=1
runtime={RUNTIME}
numjobs=1
-[readtest * {NUM_ROUNDS}]
+[readtest]
numjobs=4
blocksize=4k
rw=randread
diff --git a/wally/sensors/sensors/pscpu_sensors.py b/wally/sensors/sensors/pscpu_sensors.py
index cffdb71..ccbcefc 100644
--- a/wally/sensors/sensors/pscpu_sensors.py
+++ b/wally/sensors/sensors/pscpu_sensors.py
@@ -24,7 +24,7 @@
def pid_stat(pid):
- """ Return total cpu usage time from process"""
+ """Return total cpu usage time from process"""
# read /proc/pid/stat
with open(os.path.join('/proc/', pid, 'stat'), 'r') as pidfile:
proctimes = pidfile.readline().split()
diff --git a/wally/sensors/sensors/psram_sensors.py b/wally/sensors/sensors/psram_sensors.py
index faac87d..34f729a 100644
--- a/wally/sensors/sensors/psram_sensors.py
+++ b/wally/sensors/sensors/psram_sensors.py
@@ -11,7 +11,7 @@
# Note shared is always a subset of rss (trs is not always)
def get_mem_stats(pid):
- """ Return memory data of pid in format (private, shared) """
+ """Return memory data of pid in format (private, shared)"""
fname = '/proc/{0}/{1}'.format(pid, "smaps")
lines = open(fname).readlines()
@@ -69,7 +69,7 @@
def get_ram_size():
- """ Return RAM size in Kb"""
+ """Return RAM size in Kb"""
with open("/proc/meminfo") as proc:
mem_total = proc.readline().split()
return mem_total[1]
diff --git a/wally/sensors/sensors/utils.py b/wally/sensors/sensors/utils.py
index ad08676..cff3201 100644
--- a/wally/sensors/sensors/utils.py
+++ b/wally/sensors/sensors/utils.py
@@ -20,7 +20,7 @@
def get_pid_list(disallowed_prefixes, allowed_prefixes):
- """ Return pid list from list of pids and names """
+ """Return pid list from list of pids and names"""
# exceptions
but = disallowed_prefixes if disallowed_prefixes is not None else []
if allowed_prefixes is None:
@@ -42,7 +42,7 @@
def get_pid_name(pid):
- """ Return name by pid """
+ """Return name by pid"""
try:
with open(os.path.join('/proc/', pid, 'cmdline'), 'r') as pidfile:
try:
diff --git a/wally/statistic.py b/wally/statistic.py
index 06d525e..e2021e1 100644
--- a/wally/statistic.py
+++ b/wally/statistic.py
@@ -16,8 +16,11 @@
def med_dev(vals):
+ if len(vals) == 1:
+ return vals[0], 0.0
+
med = sum(vals) / len(vals)
- dev = ((sum(abs(med - i) ** 2.0 for i in vals) / len(vals)) ** 0.5)
+ dev = ((sum(abs(med - i) ** 2.0 for i in vals) / (len(vals) - 1)) ** 0.5)
return med, dev