fixes
diff --git a/configs-examples/default.yaml b/configs-examples/default.yaml
index c1ee515..cc78e31 100644
--- a/configs-examples/default.yaml
+++ b/configs-examples/default.yaml
@@ -8,14 +8,16 @@
#openstack:
# skip_preparation: false
# openrc: /home/koder/workspace/scale_openrc
+# insecure: true
+# openrc: ENV
# openrc:
-# user: USER
-# passwd: PASSWD
-# tenant: KEY_FILE
-# auth_url: URL
-# SOME_OTHER_OPTS: OPTIONAL
+# OS_USERNAME: USER
+# OS_PASSWORD: PASSWD
+# OS_TENANT_NAME: KEY_FILE
+# OS_AUTH_URL: URL
+# OS_INSECURE: OPTIONAL
# vms:
-# - "USERNAME[:PASSWD]@VM_NAME_PREFIX[::KEY_FILE]"
+# - "USERNAME@VM_NAME_PREFIX"
#
#ceph:
# cluster: ceph << Optional
@@ -63,7 +65,7 @@
security_group: wally_ssh_to_everyone
-ceph: nodeep-scrub, noscrub
+ceph_opts: nodeep-scrub, noscrub
#----------------------------------------- STEPS ------------------------------------------------------------------
# discover: a,b,c,... - comma separated list of clusters to discover. May be ommited
# List may contains - ceph, openstack, fuel
@@ -121,7 +123,7 @@
- io:
load: hdd
params:
- FILENAME: /dev/vdb
+ FILENAME: /opt/test.bin
FILESIZE: AUTO
openstack_ceph: OS_1_to_1 + ceph_vdb
diff --git a/configs-examples/local_block_device.yml b/configs-examples/local_block_device.yml
new file mode 100644
index 0000000..e9faa54
--- /dev/null
+++ b/configs-examples/local_block_device.yml
@@ -0,0 +1,13 @@
+include: default.yaml
+run_sensors: false
+results_storage: {STORAGE_FOLDER}
+
+nodes:
+ localhost: testnode
+
+tests:
+ - fio:
+ load: hdd
+ params:
+ FILENAME: {STORAGE_DEV_OR_FILE_NAME}
+ FILESIZE: {STORAGE_OR_FILE_SIZE}
diff --git a/requirements_extra.txt b/requirements_extra.txt
index e69de29..e01095f 100644
--- a/requirements_extra.txt
+++ b/requirements_extra.txt
@@ -0,0 +1,5 @@
+python3-env
+gcc
+python3-dev
+libssl-dev
+python3-tk
diff --git a/v2_plans.md b/v2_plans.md
index 32d61e7..3a49f6d 100644
--- a/v2_plans.md
+++ b/v2_plans.md
@@ -1,5 +1,22 @@
TODO today:
-----------
+* QDIOTimeHeatmap поломано получаение девайса по роли (aka ceph-storage)
+* ceph-osd ноды должны верифицироваться как storage также (в фильтрации девайсов)
+* дефолтный ключ в конфиг для всех новых нод
+* Проверять что файл ключа для нод существует. Есл иего нет валли ведет себя некорректно
+* Сторадж - все json - назвать js, все yaml - yml, всем без расширения дать расширения
+* Download boostrap & font css files and store them in report folder. Better to keep all in git repo
+* Add cluster config and tasks configs with compiled loads to report
+* add fiorbd default_qd and ceph profiles
+* Ceph profile - define QD from OSD count
+* Add cleanup cli, which get test output folder and clean everything
+* Add os cleanup cli, which removes all vm, flavours, images, keys from it
+* Add cli option to remove test file after test completed.
+* Make a python module, set all deps, clib/fio compilation, add UT for installation
+* openstack VM reuse - allow to use full sshuri with name prefix instead of vm name
+* fiorbd???? should be quite easy
+* allow to provide key for reused OS vms
+* openstack vm reuse - маркать виртуалки (wally_test_XXX) и находить их по этому имени и автоматически переиспользовать
* Согласовать имя секции для 'fio' - в default.yaml используется 'io', в конфигах - 'fio'
* Добавить UT
* Сделать полноценные тесты постобработки с генерацией псевдослучаныйх данных с определенным распределением
diff --git a/wally/main.py b/wally/main.py
index 497bdac..36a2690 100644
--- a/wally/main.py
+++ b/wally/main.py
@@ -159,9 +159,9 @@
ipython_parser = subparsers.add_parser('ipython', help=ipython_help)
ipython_parser.add_argument("storage_dir", help="Storage path")
# ---------------------------------------------------------------------
- jupyter_help = 'run ipython in prepared environment'
- jupyter_parser = subparsers.add_parser('jupyter', help=jupyter_help)
- jupyter_parser.add_argument("storage_dir", help="Storage path")
+ # jupyter_help = 'run jupyter in prepared environment'
+ # jupyter_parser = subparsers.add_parser('jupyter', help=jupyter_help)
+ # jupyter_parser.add_argument("storage_dir", help="Storage path")
# ---------------------------------------------------------------------
test_parser = subparsers.add_parser('test', help='run tests')
diff --git a/wally/openstack.py b/wally/openstack.py
index 16b9db6..575edc7 100644
--- a/wally/openstack.py
+++ b/wally/openstack.py
@@ -49,22 +49,25 @@
if 'openstack' in cfg.clouds:
os_cfg = cfg.clouds['openstack']
if 'OPENRC' in os_cfg:
- logger.info("Using OS credentials from " + os_cfg['OPENRC'])
- creds_tuple = get_creds_openrc(os_cfg['OPENRC'])
- os_creds = OSCreds(*creds_tuple)
- elif 'ENV' in os_cfg:
- logger.info("Using OS credentials from shell environment")
- os_creds = get_openstack_credentials()
- elif 'OS_TENANT_NAME' in os_cfg:
- logger.info("Using predefined credentials")
- os_creds = OSCreds(os_cfg['OS_USERNAME'].strip(),
- os_cfg['OS_PASSWORD'].strip(),
- os_cfg['OS_TENANT_NAME'].strip(),
- os_cfg['OS_AUTH_URL'].strip(),
- os_cfg.get('OS_INSECURE', False))
+ sett = os_cfg['OPENRC']
+ if isinstance(sett, str):
+ if 'ENV' == sett:
+ logger.info("Using OS credentials from shell environment")
+ os_creds = get_openstack_credentials()
+ else:
+ logger.info("Using OS credentials from " + os_cfg['OPENRC'])
+ creds_tuple = get_creds_openrc(sett)
+ os_creds = OSCreds(*creds_tuple)
+ else:
+ logger.info("Using predefined credentials")
+ os_creds = OSCreds(sett['OS_USERNAME'].strip(),
+ sett['OS_PASSWORD'].strip(),
+ sett['OS_TENANT_NAME'].strip(),
+ sett['OS_AUTH_URL'].strip(),
+ sett.get('OS_INSECURE', False))
- elif 'OS_INSECURE' in os_cfg:
- force_insecure = os_cfg.get('OS_INSECURE', False)
+ if 'insecure' in os_cfg:
+ force_insecure = os_cfg.get('insecure', False)
if os_creds is None and 'fuel' in cfg.clouds and 'openstack_env' in cfg.clouds['fuel'] and \
ctx.fuel_openstack_creds is not None:
diff --git a/wally/plot.py b/wally/plot.py
index f809d19..b1ee60f 100644
--- a/wally/plot.py
+++ b/wally/plot.py
@@ -121,6 +121,7 @@
# limit and label x spine
pp.ax.set_xlim(extra_x_space, len(iosums) + extra_x_space)
+ pp.ax.set_ylim(bottom=0)
pp.ax.set_xticks(xpos)
pp.ax.set_xticklabels(["{0}*{1}={2}".format(iosum.qd, iosum.nodes_count, iosum.qd * iosum.nodes_count)
for iosum in iosums],
diff --git a/wally/report.py b/wally/report.py
index f42c7c9..46c5599 100644
--- a/wally/report.py
+++ b/wally/report.py
@@ -812,26 +812,6 @@
# IO time and QD
class QDIOTimeHeatmap(JobReporter):
def get_divs(self, suite: SuiteConfig, job: JobConfig) -> Iterator[Tuple[str, str, HTMLBlock]]:
-
- # journal_devs = None
- # storage_devs = None
- # test_nodes_devs = ['rbd0']
- #
- # for node in self.rstorage.load_nodes():
- # if node.roles.intersection(STORAGE_ROLES):
- # cjd = set(node.params['ceph_journal_devs'])
- # if journal_devs is None:
- # journal_devs = cjd
- # else:
- # assert journal_devs == cjd, "{!r} != {!r}".format(journal_devs, cjd)
- #
- # csd = set(node.params['ceph_storage_devs'])
- # if storage_devs is None:
- # storage_devs = csd
- # else:
- # assert storage_devs == csd, "{!r} != {!r}".format(storage_devs, csd)
- #
-
trange = (job.reliable_info_range[0] // 1000, job.reliable_info_range[1] // 1000)
test_nc = len(list(find_nodes_by_roles(self.rstorage.storage, ['testnode'])))
@@ -844,8 +824,6 @@
yield Menu1st.engineering_per_job, job.summary, HTMLBlock(html.H3(html.center(caption)))
# QD heatmap
- # nodes = find_nodes_by_roles(self.rstorage.storage, roles)
-
ioq2d = find_sensors_to_2d(self.rstorage, trange, dev_role=dev_role, sensor='block-io', metric='io_queue')
ds = DataSource(suite.storage_id, job.storage_id, AGG_TAG, 'block-io', dev_role,
@@ -1020,7 +998,9 @@
nodes = ctx.rstorage.load_nodes()
update_storage_selector(ctx.rstorage, ctx.devs_locator, nodes)
- job_reporters_cls = [StatInfo, LoadToolResults, Resources, ClusterLoad, CPULoadPlot, QDIOTimeHeatmap]
+ # role2ds = roles_for_sensors(ctx.rstorage)
+
+ job_reporters_cls = [StatInfo, LoadToolResults, Resources, ClusterLoad, CPULoadPlot] #, QDIOTimeHeatmap]
# job_reporters_cls = [QDIOTimeHeatmap]
job_reporters = [rcls(ctx.rstorage, DefStyleProfile, DefColorProfile)
for rcls in job_reporters_cls] # type: ignore
diff --git a/wally/resources.py b/wally/resources.py
index a08993d..4074efb 100644
--- a/wally/resources.py
+++ b/wally/resources.py
@@ -122,7 +122,7 @@
cores_per_node = {}
for node in rstorage.load_nodes():
- cores_per_node[node.node_id] = sum(cores for _, cores in node.hw_info.cpus)
+ cores_per_node[node.node_id] = 48 # sum(cores for _, cores in node.hw_info.cpus)
for name in cpu_metrics:
cpu_ts[name] = sum_sensors(rstorage, time_range, node_id=nodes, sensor='system-cpu', metric=name)
diff --git a/wally/suits/io/test.cfg b/wally/suits/io/test.cfg
index 2977d1a..0280fd4 100644
--- a/wally/suits/io/test.cfg
+++ b/wally/suits/io/test.cfg
@@ -1,5 +1,8 @@
[test1]
-ioengine=libaio
+#ioengine=libaio
+ioengine=rbd
+rbdname=foo
+pool=rbd
buffered=0
group_reporting=1
iodepth=16
@@ -9,11 +12,9 @@
time_based=1
wait_for_previous=1
randrepeat=0
-filename=/dev/rbd1
size=4G
-ramp_time=10
-runtime=300
-numjobs=16
+ramp_time=0
+runtime=10
direct=1
blocksize=60k
-rw=randwrite
+rw=randread