cocluto/cluster_stats.py

# encoding: utf-8
# import sys
import os
import re
import datetime
import numpy as np
# fix to prevent the following error when run from www-data
# Failed to create /var/www/.matplotlib; consider setting MPLCONFIGDIR to a writable directory for matplotlib configuration data
# root@intranet:~# echo ~www-data
# /var/www
os.environ['MPLCONFIGDIR'] = "/tmp/cluster_stats"

import matplotlib
# fix to TclError at /cluster/ClusterEvolution/
# no display name and no $DISPLAY environment variable
# https://matplotlib.org/tutorials/introductory/usage.html#what-is-a-backend
r = os.system('python -c "import matplotlib.pyplot as plt;plt.figure()"')
if r != 0:
    matplotlib.use('Agg')  # use Anti Grain Geometry backend for non-interactive rendering into pngs, svg, etc...
import matplotlib.pyplot as plt

# import matplotlib.pyplot as plt
import matplotlib.dates
import abc

# from SimpaDbUtil import SqlDatabaseReader, SqlFile
# from inventory import Inventory


def is_cluster_node_name(name):
    return re.match('^simpatix[0-9]+$', name) is not None or re.match('^physix[0-9]+$', name) is not None


def get_investment_over_time(time_value, price, purchase_time):
    percent_decay_per_day = 0.0  # 1.0/(7.0*365.0)
    f1 = (purchase_time - time_value) * percent_decay_per_day + 1.0
    f2 = np.where(f1 < 0.0, 0.0, f1)
    f3 = np.where(time_value < purchase_time, 0.0, f2)

    return f3 * price


def get_flops_over_time(inventory, time_value, computer_id, purchase_time):
    """
    :param Inventory inventory:
    """
    return np.where(time_value < purchase_time, 0.0, inventory.get_computer_dflops(computer_id))


def get_flops_price_over_time(inventory, time_value):
    """
    :param Inventory inventory: the inventory database
    """

    rows = inventory.query("SELECT * FROM machines")

    def get_key(item):
        return item['time']

    flops_prices = []

    for row in rows:
        (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
        is_cluster_node = is_cluster_node_name(name)
        if is_cluster_node:
            purchase_date = inventory.get_machine_purchase_date(name)
            if purchase_date is not None:
                # print(name, price_ex_vat)
                purchase_time = matplotlib.dates.date2num(purchase_date.date())
                computer_flops = inventory.get_computer_dflops(name)
                flops_price = (price_ex_vat - inventory.get_computer_options_price(name)) / computer_flops
                # print ( purchase_date, name, price_ex_vat, computer_flops, flops_price )
                flops_prices.append({'time': purchase_time, 'flops_price': flops_price, 'purchase_date': purchase_date})

    flops_prices = sorted(flops_prices, key=get_key)

    flops_price_over_time = np.where(True, 0.0, 0.0)
    for item in flops_prices:
        # print(item)
        flops_price_over_time = np.where(time_value < item['time'], flops_price_over_time, item['flops_price'])

    return flops_price_over_time


def get_computer_value_over_time(inventory, computer_id, time_value, flops_price_over_time, purchase_time):
    # print('flops_price_over_time = ', flops_price_over_time)
    computer_flops = inventory.get_computer_dflops(computer_id)
    computer_flops_over_time = np.where(time_value < purchase_time, 0.0, computer_flops)
    computer_value_over_time = computer_flops_over_time * flops_price_over_time

    return computer_value_over_time

# def stackplot(ax, x_signal, y_signals):
#     """
#     :param matplotlib.Axes ax:
#     :param numpy.array x_signal:
#     :param dict(str,numpy.array) y_signals:
#     """
#     # matplot 1.1.1 doesn't have the stackplot method in Axes
#     if 'toto_stackplot' in dir(ax):
#         ax.stackplot(x_signal, list(y_signals.itervalues()) )
#         plt.legend(list(y_signals.keys()))
#     else:
#         colors = ['blue', 'orange', 'green', 'purple', 'yellow']
#         # emulating missing Axes.stackplot method
#         y = np.row_stack(list(y_signals.itervalues()))
#         # this call to 'cumsum' (cumulative sum), passing in your y data,
#         # is necessary to avoid having to manually order the datasets
#         y_stack = np.cumsum(y, axis=0)   # a 3x10 array
#         for series_index in range(len(y_signals)):
#             if series_index == 0:
#                 from_signal = 0
#             else:
#                 from_signal = y_stack[series_index-1,:]
#             ax.fill_between(x_signal, from_signal, y_stack[series_index,:], color=colors[series_index], lw=0.0, label=y_signals.keys()[series_index])
#         plt.legend()


def stackplot(ax, x_signal, y_signals):
    """
    :param matplotlib.Axes ax:
    :param numpy.array x_signal:
    :param dict(str,numpy.array) y_signals:
    """
    if 'stackplot' in dir(ax):
        ax.stackplot(x_signal, list(y_signals.values()))
        plt.legend(list(y_signals.keys()))
    else:
        # emulating missing Axes.stackplot method
        colors = ['blue', 'orange', 'green', 'purple', 'yellow']
        y = np.row_stack(list(y_signals.itervalues()))
        # this call to 'cumsum' (cumulative sum), passing in your y data,
        # is necessary to avoid having to manually order the datasets
        y_stack = np.cumsum(y, axis=0)   # a 3x10 array
        for series_index in range(len(y_signals)):
            if series_index == 0:
                from_signal = 0
            else:
                from_signal = y_stack[series_index - 1, :]
            ax.fill_between(x_signal, from_signal, y_stack[series_index, :], color=colors[series_index], lw=0.0)
            p = plt.Rectangle((0, 0), 0, 0, color=colors[series_index])
            ax.add_patch(p)
        plt.legend(list(y_signals.keys()))


def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type):
    time_value = matplotlib.dates.drange(dstart=from_date, dend=to_date, delta=datetime.timedelta(days=1))
    flops_price_over_time = get_flops_price_over_time(inventory, time_value)
    cluster_value = {}

    rows = inventory.query("SELECT * FROM machines")

    for row in rows:
        (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
        is_cluster_node = is_cluster_node_name(name)
        if is_cluster_node:
            purchase_date = inventory.get_machine_purchase_date(name)
            if purchase_date is not None:
                # print(name, price_ex_vat)
                purchase_time = matplotlib.dates.date2num(purchase_date.date())
                item_value_over_time = {
                    'cluster_cost_over_time': get_investment_over_time(time_value, price_ex_vat, purchase_time),
                    'cluster_value_over_time': get_computer_value_over_time(inventory, name, time_value, flops_price_over_time, purchase_time),
                    'cluster_dp_gflops_over_time': get_flops_over_time(inventory, time_value, name, purchase_time)}[graph_type]
                for ownership in inventory.get_item_ownership(name):
                    # print(ownership)
                    # print(ownership['owner'], ownership['owner_ratio'])
                    owner = ownership['owner']
                    owner_dept = owner.split('.')[1]
                    # if owner_dept == 'matnano':
                    # print(name, owner, purchase_date, price_ex_vat)
                    if owner_dept in cluster_value.keys():
                        cluster_value[owner_dept] += item_value_over_time
                    else:
                        cluster_value[owner_dept] = item_value_over_time  # np.zeros_like(time_value)

                # print(purchase_date)
    # print(type(from_date))
    # print(type(to_date))
    # X = np.linspace(-np.pi, np.pi, 256, endpoint=True)
    # C,S = np.cos(X), np.sin(X)

    fig, ax = plt.subplots()
    ax.set_title(graph_type)
    # for dept, cluster_value_for_dept in cluster_value.iteritems():
    #    ax.plot(time_value, cluster_value_for_dept)

    stackplot(ax, time_value, cluster_value)

    plt.xlabel('time')
    plt.ylabel({
        'cluster_cost_over_time': u'cluster investment (€)',
        'cluster_value_over_time': u'cluster value (€)',
        'cluster_dp_gflops_over_time': u'double prec gflops'}[graph_type])

    years = matplotlib.dates.YearLocator()   # every year
    months = matplotlib.dates.MonthLocator()  # every month
    yearsFmt = matplotlib.dates.DateFormatter('%Y')

    # format the ticks
    ax.xaxis.set_major_locator(years)
    ax.xaxis.set_major_formatter(yearsFmt)
    ax.xaxis.set_minor_locator(months)

    datemin = datetime.date(from_date.year, 1, 1)
    datemax = datetime.date(to_date.year + 1, 1, 1)
    ax.set_xlim(datemin, datemax)

    # rotates and right aligns the x labels, and moves the bottom of the
    # axes up to make room for them
    # fig.autofmt_xdate()
    ax.grid(True)

    # plt.plot()
    # plt.plot(X,S)
    return fig


def draw_dp_gflops_price_over_time_over_time_graph(inventory, from_date, to_date):
    """
    :param Inventory inventory: the inventory database
    :param datetime from_time:
    :param datetime to_time:
    """

    time_value = matplotlib.dates.drange(dstart=from_date, dend=to_date, delta=datetime.timedelta(days=1))
    gflops_price_over_time = get_flops_price_over_time(inventory, time_value) * 1.0e9
    fig, ax = plt.subplots()
    ax.set_yscale('log')
    ax.plot(time_value, gflops_price_over_time)
    ax.set_xlabel('time')
    ax.set_ylabel(u'double precision flops price (€/gflops)')
    ax.set_title('gflops_price_over_time')

    years = matplotlib.dates.YearLocator()   # every year
    months = matplotlib.dates.MonthLocator()  # every month
    yearsFmt = matplotlib.dates.DateFormatter('%Y')

    # format the ticks
    ax.xaxis.set_major_locator(years)
    ax.xaxis.set_major_formatter(yearsFmt)
    ax.xaxis.set_minor_locator(months)
    ax.grid(True)
    return fig


def draw_age_pyramid_graph(inventory):
    """
    :param Inventory inventory: the inventory database
    """

    oldest_age = 20
    age_histogram = np.zeros(shape=(oldest_age))

    rows = inventory.query("SELECT * FROM machines")

    for row in rows:
        (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
        is_cluster_node = is_cluster_node_name(name)
        if is_cluster_node:
            purchase_date = inventory.get_machine_purchase_date(name)
            if purchase_date is not None:
                purchase_time = matplotlib.dates.date2num(purchase_date.date())  # noqa: F841
                age = datetime.datetime.now() - purchase_date
                age_histogram[age.days / 365] += 1
                # print(name, age)

    fig, ax = plt.subplots()
    ax.bar(range(oldest_age), age_histogram)
    ax.set_xlabel('age (in years)')
    ax.set_xticks(range(oldest_age))

    ax.set_ylabel(u'number of compute nodes')
    ax.set_title('compute_nodes_age_pyramid')

    # format the ticks
    ax.grid(True)
    return fig


class IFigureHandler(object):
    """
    specifies what to do with generated figures
    """

    @abc.abstractmethod
    def on_figure_ended(self, fig):
        """
        :param matplotlib.Figure fig:
        """
        pass

    @abc.abstractmethod
    def on_finalize(self):
        """
        called after all figures have been created
        """
        pass


class ScreenFigureHandler(IFigureHandler):
    """
    displays figures on screen
    """
    def __init__(self):
        pass

    def on_figure_ended(self, fig):
        pass

    def on_finalize(self):
        plt.show()


class SvgFigureHandler(IFigureHandler):
    """
    saves figures as svg files
    """

    def __init__(self, out_svg_dir_path):
        """
        :param str out_svg_dir_path: where to save the svg files
        """
        self._out_svg_dir_path = out_svg_dir_path

    def on_figure_ended(self, fig):
        fig.savefig(self._out_svg_dir_path + '/' + fig.axes[0].get_title() + '.svg')

    def on_finalize(self):
        pass


def draw_graphs(inventory, from_time, to_time, figure_handler):
    """
    :param Inventory inventory: the inventory database
    :param datetime from_time:
    :param datetime to_time:
    :param IFigureHandler figure_handler:
    """
    fig = draw_cluster_value_over_time_graph(inventory, from_time.date(), to_time.date(), 'cluster_value_over_time')
    figure_handler.on_figure_ended(fig)

    fig = draw_cluster_value_over_time_graph(inventory, from_time.date(), to_time.date(), 'cluster_dp_gflops_over_time')
    figure_handler.on_figure_ended(fig)

    fig = draw_cluster_value_over_time_graph(inventory, from_time.date(), to_time.date(), 'cluster_cost_over_time')
    figure_handler.on_figure_ended(fig)

    fig = draw_dp_gflops_price_over_time_over_time_graph(inventory, from_time.date(), to_time.date())
    figure_handler.on_figure_ended(fig)

    fig = draw_age_pyramid_graph(inventory)
    figure_handler.on_figure_ended(fig)

    figure_handler.on_finalize()