cfg_checker/modules/ceph/bench.py

import csv
import os
import json

from datetime import datetime, timedelta
from time import sleep

from cfg_checker.common import logger_cli
from cfg_checker.helpers.console_utils import Progress
# from cfg_checker.common.exception import InvalidReturnException
# from cfg_checker.common.exception import ConfigException
# from cfg_checker.common.exception import KubeException

from cfg_checker.nodes import KubeNodes
from cfg_checker.agent.fio_runner import _get_seconds, _datetime_fmt


class CephBench(object):
    _agent_template = "cfgagent-template.yaml"

    def __init__(self, config):
        self.env_config = config
        return


class SaltCephBench(CephBench):
    def __init__(
        self,
        config
    ):
        logger_cli.error("ERROR: Not impelented for Salt environment!")

        # self.master = SaltNodes(config)
        super(SaltCephBench, self).__init__(config)
        return


class KubeCephBench(CephBench):
    def __init__(self, config):
        self.agent_count = config.bench_agent_count
        self.master = KubeNodes(config)
        super(KubeCephBench, self).__init__(config)
        self.storage_class = config.bench_storage_class
        self.agent_pods = []
        self.services = []
        self.api_urls = []
        self.mode = config.bench_mode
        if config.bench_mode == "tasks":
            self.load_tasks(config.bench_task_file)

        self.cleanup_list = []

    def load_tasks(self, taskfile):
        # Load csv file
        logger_cli.info("-> Loading taskfile '{}'".format(taskfile))
        self.tasks = []
        with open(taskfile) as f:
            _reader = csv.reader(f, delimiter=',')
            # load packages
            for row in _reader:
                self.tasks.append({
                    "readwrite": row[0],
                    "rwmixread": row[1],
                    "bs": row[2],
                    "iodepth": row[3],
                    "size": row[4]
                })

    def add_for_deletion(self, obj, typ):
        _d = [typ, obj.metadata.namespace, obj.metadata.name]
        self.cleanup_list.append(_d)
        return

    def prepare_agents(self, options):
        logger_cli.info("# Preparing {} agents".format(self.agent_count))
        for idx in range(self.agent_count):
            # create pvc/pv and pod
            logger_cli.info("-> creating agent '{:02}'".format(idx))
            _agent, _pv, _pvc = self.master.prepare_benchmark_agent(
                idx,
                os.path.split(options["filename"])[0],
                self.storage_class,
                options['size'] + 'i',
                self._agent_template
            )
            # save it to lists
            self.agent_pods.append(_agent)
            self.add_for_deletion(_pv, "pv")
            self.add_for_deletion(_pvc, "pvc")
            self.add_for_deletion(_agent, "pod")

            # expose it
            _svc = self.master.expose_benchmark_agent(_agent)
            self.add_for_deletion(_svc, "svc")
            # Save service
            self.services.append(_svc)

        # Build urls for agents
        for svc in self.services:
            self.api_urls.append(
                "http://{}:{}/api/".format(
                    svc.spec.cluster_ip,
                    8765
                )
            )
        logger_cli.info("-> Done creating agents")
        return

    def _poke_agent(self, url, body, action="GET"):
        _datafile = "/tmp/data"
        _data = [
            "-d",
            "@" + _datafile
        ]
        _cmd = [
            "curl",
            "-s",
            "-H",
            "'Content-Type: application/json'",
            "-X",
            action,
            url
        ]
        if body:
            _cmd += _data
            _ret = self.master.prepare_json_in_pod(
                self.agent_pods[0].metadata.name,
                self.master._namespace,
                body,
                _datafile
            )
        _ret = self.master.exec_cmd_on_target_pod(
            self.agent_pods[0].metadata.name,
            self.master._namespace,
            " ".join(_cmd)
        )
        try:
            return json.loads(_ret)
        except TypeError as e:
            logger_cli.error(
                "ERROR: Status not decoded: {}\n{}".format(e, _ret)
            )
        except json.decoder.JSONDecodeError as e:
            logger_cli.error(
                "ERROR: Status not decoded: {}\n{}".format(e, _ret)
            )

        return None

    def run_benchmark(self, options):
        def get_status():
            return [self._poke_agent(_u + "fio", {}) for _u in self.api_urls]
        logger_cli.info("# Starting '{}' benchmark".format(self.mode))
        logger_cli.info("# Checking agents")
        # make sure agents idle
        _tt = []
        _rr = []
        for _s in get_status():
            if _s is None:
                logger_cli.error("ERROR: Agent status not available")
                return False
            _h = _s["healthcheck"]["hostname"]
            _t = _s['status']
            _r = _s["healthcheck"]["ready"]
            if _t not in ["idle", "finished"]:
                logger_cli.error("Agent status invalid {}:{}".format(_h, _t))
                _tt += [False]
            else:
                _tt += [True]
            if not _r:
                logger_cli.error("Agent is not ready {}".format(_h))
                _rr += [False]
            else:
                _rr += [True]
        if not any(_tt) or not any(_rr):
            return False

        # Make sure that Ceph is at low load
        # TODO: Ceph status check

        # Do benchmark according to mode
        if self.mode == "tasks":
            # TODO: Implement 'tasks' mode
            # take next task

            # update options

            # init time to schedule

            # send next task to agent
            pass
            # wait for agents to finish
        elif self.mode == "single":
            logger_cli.info("# Running benchmark")
            # init time to schedule
            _time = datetime.now() + timedelta(seconds=10)
            _str_time = _time.strftime(_datetime_fmt)
            options["scheduled_to"] = _str_time
            logger_cli.info(
                "-> next benchmark scheduled to '{}'".format(_str_time)
            )
            # send single to agent
            _task = {
                "module": "fio",
                "action": "do_singlerun",
                "options": options
            }
            for _u in self.api_urls:
                logger_cli.info("-> sending task to '{}'".format(_u))
                _ret = self._poke_agent(_u, _task, action="POST")
                if 'error' in _ret:
                    logger_cli.error(
                        "ERROR: Agent returned: '{}'".format(_ret['error'])
                    )

        _runtime = _get_seconds(options["runtime"])
        _ramptime = _get_seconds(options["ramp_time"])
        _timeout = _runtime + _ramptime + 5
        _end = datetime.now() + timedelta(seconds=_timeout)
        while True:
            # Print status
            _sts = get_status()
            _str = ""
            for _s in _sts:
                _str += "{}: {} ({}); ".format(
                    _s["healthcheck"]["hostname"],
                    _s["status"],
                    _s["progress"]
                )
            # recalc how much is left
            diff = (_end - datetime.now()).total_seconds()
            logger_cli.debug("... [{:.2f}s]: {}".format(diff, _str))
            # In case end_datetime was in past to begin with
            if diff < 0:
                break
            logger_cli.info("-> Sleeping for {:.2f}s".format(diff/2))
            sleep(diff/2)
            if diff <= 0.1:
                break

        logger_cli.info("-> Done")
        return True

    def cleanup(self):
        self.cleanup_list.reverse()

        for _res in self.cleanup_list:
            self.master.cleanup_resource_by_name(_res[0], _res[2], ns=_res[1])

        # Wait for resource to be cleaned
        _timeout = 120
        _total = len(self.cleanup_list)
        logger_cli.info("-> Wait until {} resources cleaned".format(_total))
        _p = Progress(_total)
        while True:
            _g = self.master.get_resource_phase_by_name
            _l = [_g(r[0], r[2], ns=r[1]) for r in self.cleanup_list]
            _l = [item for item in _l if item]
            _idx = _total - len(_l)
            if len(_l) > 0:
                _p.write_progress(_idx)
            else:
                _p.end()
                logger_cli.info("# Done cleaning up")
                break
            if _timeout > 0:
                _timeout -= 1
            else:
                _p.end()
                logger_cli.info("# Timed out waiting after 120s.")
                break

        return

    # Create report
    def create_report(self, filename):

        return