persistance/storage_api.py

import datetime
import math
from urlparse import urlparse

from flask import json
from persistance.keystone import KeystoneAuth
from sqlalchemy import sql
from persistance.models import *


#class displays measurement. Moved from storage_api_v_1 to avoid circular imports.
class Measurement(object):
    def __init__(self):
        self.build = ""
        self.build_type = 0  # GA/Master/Other
        self.md5 = ""
        self.name = ""
        self.date = None
        self.results = {
            "": (float, float)
        }

    def __str__(self):
        return self.build + " " + self.build_type + " " + \
            self.md5 + " " + str(self.results)


def mean(l):
    n = len(l)

    return sum(l) / n


def stdev(l):
    m = mean(l)
    return math.sqrt(sum(map(lambda x: (x - m) ** 2, l)))



def total_lab_info(data):
    # <koder>: give 'd' meaningful name
    d = {}
    d['nodes_count'] = len(data['nodes'])
    d['total_memory'] = 0
    d['total_disk'] = 0
    d['processor_count'] = 0

    for node in data['nodes']:
        d['total_memory'] += node['memory']['total']
        d['processor_count'] += len(node['processors'])

        for disk in node['disks']:
            d['total_disk'] += disk['size']

    to_gb = lambda x: x / (1024 ** 3)
    d['total_memory'] = format(to_gb(d['total_memory']), ',d')
    d['total_disk'] = format(to_gb(d['total_disk']), ',d')
    return d


def collect_lab_data(url, cred):
    u = urlparse(url)
    keystone = KeystoneAuth(root_url=url, creds=cred, admin_node_ip=u.hostname)
    lab_info = keystone.do(method='get', path="/api/nodes")
    fuel_version = keystone.do(method='get', path="/api/version/")

    nodes = []
    result = {}

    for node in lab_info:
        # <koder>: give p,i,d,... vars meaningful names
        d = {}
        d['name'] = node['name']
        p = []
        i = []
        disks = []
        devices = []

        for processor in node['meta']['cpu']['spec']:
             p.append(processor)

        for iface in node['meta']['interfaces']:
            i.append(iface)

        m = node['meta']['memory'].copy()

        for disk in node['meta']['disks']:
            disks.append(disk)

        d['memory'] = m
        d['disks'] = disks
        d['devices'] = devices
        d['interfaces'] = i
        d['processors'] = p

        nodes.append(d)

    result['nodes'] = nodes
    # result['name'] = 'Perf-1 Env'
    result['fuel_version'] = fuel_version['release']

    return result


def get_build_info(build_name):
    session = db.session()
    result = session.query(Result, Build).join(Build).filter(Build.name == build_name).first()
    lab = session.query(Lab).filter(Lab.id == result[0].lab_id).first()
    return eval(lab.lab_general_info)


def get_build_detailed_info(build_name):
    data = get_build_info(build_name)
    return total_lab_info(data)


def process_build_data(build):
    for item in build.items():
        if type(item[1]) is list:
            m = mean(item[1])
            s = stdev(item[1])
            build[item[0]] = [m, s]


#filling Param table with initial parameters.
def add_io_params(session):
    param1 = Param(name="operation", type='{"write", "randwrite", "read", "randread"}', descr="type of write operation")
    param2 = Param(name="sync", type='{"a", "s"}', descr="Write mode synchronous/asynchronous")
    param3 = Param(name="block size", type='{"1k", "2k", "4k", "8k", "16k", "32k", "64k", "128k", "256k"}')

    session.add(param1)
    session.add(param2)
    session.add(param3)

    session.commit()


#function which adds particular build to database.
def add_build(session, build_id, build_name, build_type, md5):
    build = Build(type=build_type, build_id=build_id, name=build_name, md5=md5)
    session.add(build)
    session.commit()

    return build.id


#function insert particular result.
def insert_results(session, build_id, lab_id, params_combination_id,
                   time=None, bandwith=0.0, meta=""):
    result = Result(build_id=build_id, lab_id=lab_id, params_combination_id=params_combination_id, time=time,
                    bandwith=bandwith, meta=meta)
    session.add(result)
    session.commit()


#function responsible for adding particular params combination to database
def add_param_comb(session, *params):
    params_names = sorted([s for s in dir(ParamCombination) if s.startswith('param_')])
    d = zip(params_names, params)
    where = ""

    for item in d:
        where = sql.and_(where, getattr(ParamCombination, item[0]) == item[1])

    query = session.query(ParamCombination).filter(where)
    rs = session.execute(query).fetchall()


    if len(rs) == 0:
        param_comb = ParamCombination()

        for p in params_names:
            i = int(p.split('_')[1])
            param_comb.__setattr__('param_' + str(i), params[i - 1])

            param = session.query(Param).filter(Param.id == i).one()
            values = eval(param.type)

            if params[i - 1] not in values:
                values.add(params[i - 1])
                param.type = str(values)

        session.add(param_comb)
        session.commit()
        return param_comb.id
    else:
        return rs[0][0]


def add_lab(session, lab_url, lab_name, ceph_version, fuel_version, data, info):
    result = session.query(Lab).filter(Lab.name == lab_name).all()

    if len(result) != 0:
        return result[0].id
    else:
        lab = Lab(name=lab_name, url=lab_url, ceph_version=ceph_version,
                  fuel_version=fuel_version, lab_general_info=str(data), lab_meta=str(info))
        session.add(lab)
        session.commit()
        return lab.id

#function store list of builds in database
def add_data(data):
    data = json.loads(data)
    session = db.session()
    add_io_params(session)

    for build_data in data:
        build_id = add_build(session,
                             build_data.pop("build_id"),
                             build_data.pop("name"),
                             build_data.pop("type"),
                             build_data.pop("iso_md5"),
                             )
        creds = {"username": build_data.pop("username"),
                 "password": build_data.pop("password"),
                 "tenant_name": build_data.pop("tenant_name")
        }
        lab_url = build_data.pop("lab_url")
        lab_name = build_data.pop("lab_name")
        ceph_version = build_data.pop("ceph_version")
        data = collect_lab_data(lab_url, creds)
        data['name'] = lab_name
        info = total_lab_info(data)
        lab_id = add_lab(session, lab_url=lab_name, lab_name=lab_url,
                ceph_version=ceph_version, fuel_version=data['fuel_version'], data=data, info=info)

        date = build_data.pop("date")
        date = datetime.datetime.strptime(date, "%a %b %d %H:%M:%S %Y")

        for params, [bw, dev] in build_data.items():
            param_comb_id = add_param_comb(session, *params.split(" "))
            result = Result(param_combination_id=param_comb_id, build_id=build_id,
                            bandwith=bw, date=date, lab_id=lab_id)
            session.add(result)
            session.commit()


#function loads data by parametres described in *params tuple.
def load_data(*params):
    session = db.session()
    params_names = sorted([s for s in dir(ParamCombination) if s.startswith('param_')])
    d = zip(params_names, params)
    where = ""

    for item in d:
        where = sql.and_(where, getattr(ParamCombination, item[0]) == item[1])

    query = session.query(ParamCombination).filter(where)
    rs = session.execute(query).fetchall()

    ids = [r[0] for r in rs]

    results = session.query(Result).filter(Result.param_combination_id.in_(ids))
    rs = session.execute(results).fetchall()

    return [r[5] for r in rs]


#load all builds from database
def load_all():
    session = db.session()
    r = session.query(Param).filter(Param.id == 1).all()
    results = session.query(Result, Build, ParamCombination).join(Build).join(ParamCombination).all()

    return results


#function collecting all builds from database and filter it by names
def collect_builds_from_db(*names):
    results = load_all()
    d = {}

    for item in results:
        result_data = item[0]
        build_data = item[1]
        param_combination_data = item[2]

        if build_data.name not in d:
            d[build_data.name] = [build_data, result_data, param_combination_data]
        else:
            d[build_data.name].append(result_data)
            d[build_data.name].append(param_combination_data)

    if len(names) == 0:
        return {k: v for k, v in d.items()}

    return {k: v for k, v in d.items() if k in names}


#function creates measurement from data was extracted from database.
def create_measurement(data):
    build_data = data[0]

    m = Measurement()
    m.build = build_data.build_id
    m.build_type = build_data.type
    m.name = build_data.name
    m.results = {}

    for i in range(1, len(data), 2):
        result = data[i]
        param_combination = data[i + 1]

        if not str(param_combination) in m.results:
            m.results[str(param_combination)] = [result.bandwith]
        else:
            m.results[str(param_combination)] += [result.bandwith]

    for k in m.results.keys():
        m.results[k] = [mean(m.results[k]), stdev(m.results[k])]

    return m


#function preparing data for display plots.
#Format {build_name : Measurement}
def prepare_build_data(build_name):
    session = db.session()
    build = session.query(Build).filter(Build.name == build_name).first()
    names = []

    if build.type == 'GA':
        names = [build_name]
    else:
        res = session.query(Build).filter(Build.type.in_(['GA', 'master', build.type])).all()
        for r in res:
            names.append(r.name)


    d = collect_builds_from_db()
    d = {k: v for k, v in d.items() if k in names}
    results = {}

    for data in d.keys():
        m = create_measurement(d[data])
        results[m.build_type] = m

    return results


#function getting list of all builds available to index page
#returns list of dicts which contains data to display on index page.
def builds_list():
    res = []
    builds = set()
    data = load_all()

    for item in data:
        build = item[1]
        result = item[0]

        if not build.name in builds:
            builds.add(build.name)
            d = {}
            d["type"] = build.type
            d["url"] = build.name
            d["date"] = result.date
            d["name"] = build.name
            res.append(d)

    return res


#Processing data from database.
#List of dicts, where each dict contains build meta info and kev-value measurements.
#key - param combination.
#value - [mean, deviation]
def get_builds_data(names=None):
    d = collect_builds_from_db()

    if not names is None:
        d = {k: v for k, v in d.items() if k in names}
    else:
        d = {k: v for k, v in d.items()}
    output = []

    for key, value in d.items():
        result = {}
        build = value[0]
        result["build_id"] = build.build_id
        result["iso_md5"] = build.md5
        result["type"] = build.type
        result["date"] = "Date must be here"

        for i in range(1, len(value), 2):
            r = value[i]
            param_combination = value[i + 1]

            if not str(param_combination) in result:
                result[str(param_combination)] = [r.bandwith]
            else:
                result[str(param_combination)].append(r.bandwith)

        output.append(result)

    for build in output:
        process_build_data(build)

    return output


#Function for getting result to display table
def get_data_for_table(build_name=""):
    session = db.session()
    build = session.query(Build).filter(Build.name == build_name).one()
    names = []

    #Get names of build that we need.
    if build.type == 'GA':
        names = [build_name]
    else:
        res = session.query(Build).filter(Build.type.in_(['GA', 'master', build.type])).all()
        for r in res:
            names.append(r.name)
    #get data for particular builds.
    return get_builds_data(names)


if __name__ == '__main__':
    # add_build("Some build", "GA", "bla bla")
    cred = {"username": "admin", "password": "admin", "tenant_name": "admin"}
    json_data = '[{\
        "username": "admin",\
        "password": "admin", \
        "tenant_name": "admin",\
        "lab_url": "http://172.16.52.112:8000",\
        "lab_name": "Perf-1-Env",\
        "ceph_version": "v0.80 Firefly",\
        "randwrite a 256k": [16885, 1869],\
        "randwrite s 4k": [79, 2],\
        "read a 64k": [74398, 11618],\
        "write s 1024k": [7490, 193],\
        "randwrite a 64k": [14167, 4665],\
        "build_id": "1",\
        "randread a 1024k": [68683, 8604],\
        "randwrite s 256k": [3277, 146],\
        "write a 1024k": [24069, 660],\
        "type": "GA",\
        "write a 64k": [24555, 1006],\
        "write s 64k": [1285, 57],\
        "write a 256k": [24928, 503],\
        "write s 256k": [4029, 192],\
        "randwrite a 1024k": [23980, 1897],\
        "randread a 64k": [27257, 17268],\
        "randwrite s 1024k": [8504, 238],\
        "randread a 256k": [60868, 2637],\
        "randread a 4k": [3612, 1355],\
        "read a 1024k": [71122, 9217],\
        "date": "Thu Feb 12 19:11:56 2015",\
        "write s 4k": [87, 3],\
        "read a 4k": [88367, 6471],\
        "read a 256k": [80904, 8930],\
        "name": "GA - 6.0 GA",\
        "randwrite s 1k": [20, 0],\
        "randwrite s 64k": [1029, 34],\
        "write s 1k": [21, 0],\
        "iso_md5": "bla bla"\
    },\
    {\
        "username": "admin",\
        "password": "admin", \
        "tenant_name": "admin",\
        "lab_url": "http://172.16.52.112:8000",\
        "ceph_version": "v0.80 Firefly",\
        "lab_name": "Perf-1-Env",\
        "randwrite a 256k": [20212, 5690],\
        "randwrite s 4k": [83, 6],\
        "read a 64k": [89394, 3912],\
        "write s 1024k": [8054, 280],\
        "randwrite a 64k": [14595, 3245],\
        "build_id": "2",\
        "randread a 1024k": [83277, 9310],\
        "randwrite s 256k": [3628, 433],\
        "write a 1024k": [29226, 8624],\
        "type": "master",\
        "write a 64k": [25089, 790],\
        "write s 64k": [1236, 30],\
        "write a 256k": [30327, 9799],\
        "write s 256k": [4049, 172],\
        "randwrite a 1024k": [29000, 9302],\
        "randread a 64k": [26775, 16319],\
        "randwrite s 1024k": [8665, 1457],\
        "randread a 256k": [63608, 16126],\
        "randread a 4k": [3212, 1620],\
        "read a 1024k": [89676, 4401],\
        "date": "Thu Feb 12 19:11:56 2015",\
        "write s 4k": [88, 3],\
        "read a 4k": [92263, 5186],\
        "read a 256k": [94505, 6868],\
        "name": "6.1 Dev",\
        "randwrite s 1k": [22, 3],\
        "randwrite s 64k": [1105, 46],\
        "write s 1k": [22, 0],\
        "iso_md5": "bla bla"\
    },\
    {\
        "username": "admin",\
        "password": "admin", \
        "tenant_name": "admin",\
        "lab_url": "http://172.16.52.112:8000",\
        "ceph_version": "v0.80 Firefly",\
        "lab_name": "Perf-1-Env",\
        "randwrite a 256k": [16885, 1869],\
        "randwrite s 4k": [79, 2],\
        "read a 64k": [74398, 11618],\
        "write s 1024k": [7490, 193],\
        "randwrite a 64k": [14167, 4665],\
        "build_id": "1",\
        "randread a 1024k": [68683, 8604],\
        "randwrite s 256k": [3277, 146],\
        "write a 1024k": [24069, 660],\
        "type": "sometype",\
        "write a 64k": [24555, 1006],\
        "write s 64k": [1285, 57],\
        "write a 256k": [24928, 503],\
        "write s 256k": [4029, 192],\
        "randwrite a 1024k": [23980, 1897],\
        "randread a 64k": [27257, 17268],\
        "randwrite s 1024k": [8504, 238],\
        "randread a 256k": [60868, 2637],\
        "randread a 4k": [3612, 1355],\
        "read a 1024k": [71122, 9217],\
        "date": "Thu Feb 12 19:11:56 2015",\
        "write s 4k": [87, 3],\
        "read a 4k": [88367, 6471],\
        "read a 256k": [80904, 8930],\
        "name": "somedev",\
        "randwrite s 1k": [20, 0],\
        "randwrite s 64k": [1029, 34],\
        "write s 1k": [21, 0],\
        "iso_md5": "bla bla"\
    }]'

    # json_to_db(json_data)
    # print load_data()
    add_data(json_data)

    print collect_builds_from_db()
    print prepare_build_data('6.1 Dev')
    print builds_list()
    print get_data_for_table('somedev')