Ceph report fixes and Ceph bench beta 0.1
- Ceph stats collection
- Updated Ceph results averages calculations
Fixes:
- Fixed huge PG dump copying >30MB jsons
- Fixes for the fio-runner constants
Related-PROD: PROD-36669
Change-Id: Id8e250f626dfdaecc12ad005b61d03a21c9e6c4e
diff --git a/cfg_checker/modules/ceph/info.py b/cfg_checker/modules/ceph/info.py
index 56e250e..f6d5758 100644
--- a/cfg_checker/modules/ceph/info.py
+++ b/cfg_checker/modules/ceph/info.py
@@ -1,4 +1,8 @@
+import base64
import json
+import os
+import tarfile
+import io
from time import sleep
@@ -59,6 +63,10 @@
# TODO: Consider filtering out or prepare data for the table
_osd = _d.pop("osd_name")
_node_name = _d.pop("node_name")
+ # Additional check for empty data
+ if not _d:
+ self.ceph_info['ceph_health']['latest'][_n] = {}
+ continue
_date = sorted(_d.keys(), reverse=True)[0]
self.ceph_info['ceph_health']['date'] = _date
self.ceph_info['ceph_health']['latest'][_n] = _d[_date]
@@ -238,28 +246,65 @@
self.cluster_info = {}
self.ceph_version = self.get_ceph_cluster_config()
- def _safe_tools_cmd(self, cmd, expect_output=True):
+ def _safe_tools_cmd(self, cmd_str, expect_output=True):
_r = self.master.exec_cmd_on_target_pod(
self.pod_name,
self.ceph_ns,
- cmd
+ cmd_str
)
if expect_output and not _r:
- logger_cli.debug("... got empty output for '{}'".format(cmd))
+ logger_cli.debug("... got empty output for '{}'".format(cmd_str))
elif not expect_output and _r:
logger_cli.warning(
"WARNING: Unexpected output for '{}':\n"
- "===== Start\n{}\n===== End".format(cmd, _r)
+ "===== Start\n{}\n===== End".format(cmd_str, _r)
)
return _r
- def _safe_get_cmd_output_as_json(self, cmd):
- _buf = self._safe_tools_cmd(cmd)
+ def _safe_tools_cmd_zipped_output(self, cmd_str):
+ # temp file
+ _tmp_path = "/tmp"
+ _filename = "checker_cmd_output"
+ _tar_path = os.path.join(_tmp_path, "checker_cmd.tgz")
+ _path = os.path.join(_tmp_path, _filename)
+
+ # Run original cmd with redirect
+ _cmd = [cmd_str, "-o", _path]
+ self._safe_tools_cmd(" ".join(_cmd), expect_output=False)
+ # zip it and base64 encode
+ _cmd = ["tar", "-zcvf", _tar_path, _path]
+ self._safe_tools_cmd(" ".join(_cmd))
+ _b64 = self._safe_tools_cmd("base64 " + _tar_path)
+ # decode and decompress
+ _io = io.BytesIO(base64.standard_b64decode(_b64))
+ _json = ""
+ with tarfile.open(fileobj=_io) as _tar:
+ _tar_item = _tar.extractfile(_tar.getmembers()[0])
+ _json = _tar_item.read()
+ # cleanup
+ self._safe_tools_cmd("rm -f " + _path)
+ self._safe_tools_cmd("rm -f " + _tar_path)
+ return _json
+
+ def _safe_get_cmd_output_as_json(self, cmd, zipped=False):
+ if zipped:
+ _buf = self._safe_tools_cmd_zipped_output(cmd)
+ else:
+ _buf = self._safe_tools_cmd(cmd)
try:
return json.loads(_buf)
- except ValueError:
+ except ValueError as e:
+ _out = ""
+ if len(_buf) > 512:
+ _out = _buf[:512]
+ _out += "..."
+ else:
+ _out = _buf
logger_cli.error(
- "\nERROR: failed to parse json: '{}'".format(_buf)
+ "\nERROR: failed to parse json: '{}'. Data: '{}'".format(
+ e,
+ _out
+ )
)
return _buf
@@ -365,7 +410,10 @@
return self._safe_get_cmd_output_as_json("ceph df -f json")
def get_ceph_pg_dump(self):
- return self._safe_get_cmd_output_as_json("ceph pg dump -f json")
+ return self._safe_get_cmd_output_as_json(
+ "ceph pg dump -f json",
+ zipped=True
+ )
def get_ceph_osd_df(self):
return self._safe_get_cmd_output_as_json("ceph osd df -f json")