From ad8f6f383bd1e8026763d9e2b7bb17c63c2acf80 Mon Sep 17 00:00:00 2001 From: Guillaume Raffy Date: Mon, 7 Jun 2021 11:42:03 +0000 Subject: [PATCH] Bug 3165 - la page ClusterEvolution ne fonctionne plus (failed to find in table orderings a value for ordering_date where ordering_id is ietr.order20210101) - fixed bug : it was caused by the fact that the database has no information about ietr machines order. Fixed this by ignoring ietr machines because they're only part of ipr cluster temporarily and as such, they shouldn't appear in the stats. also some improvements: - improved the colors of the owners of the machines so that tey're visually close for the same organism (eg ipr owners have close colors). This change was made in june 2020 I think (for the questionnaire about it needs related to eskemmdata) - added a graph to show the age of cores - prevented overlapping of dates in the graph showing the evolution of gflops price --- cluster_stats.py | 82 +++++++++++++++++++++++++++++++++++++++++------- inventory.py | 6 ++++ 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/cluster_stats.py b/cluster_stats.py index 0e5a099..f8b7037 100644 --- a/cluster_stats.py +++ b/cluster_stats.py @@ -32,6 +32,16 @@ def is_cluster_node_name(name): return re.match('^simpatix[0-9]+$', name) is not None or re.match('^physix[0-9]+$', name) is not None +def is_test_machine(name): + return name in [ + # ietr amd machines temporarily in the cluster for tests + 'physix12', + 'physix13', + 'physix14', + 'physix15', + ] + + def get_investment_over_time(time_value, price, purchase_time): percent_decay_per_day = 0.0 # 1.0/(7.0*365.0) f1 = (purchase_time - time_value) * percent_decay_per_day + 1.0 @@ -63,7 +73,7 @@ def get_flops_price_over_time(inventory, time_value): for row in rows: (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row is_cluster_node = is_cluster_node_name(name) - if is_cluster_node: + if is_cluster_node and not is_test_machine(name): purchase_date = inventory.get_machine_purchase_date(name) if purchase_date is not None: # print(name, price_ex_vat) @@ -119,7 +129,7 @@ def get_computer_value_over_time(inventory, computer_id, time_value, flops_price def get_rgb_palette(num_colors, saturation=0.5, value=0.5): hsv_tuples = [(x * 1.0 / num_colors, saturation, value) for x in range(num_colors)] - rgb_tuples = map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples) + rgb_tuples = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)) return rgb_tuples @@ -131,7 +141,7 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'): :param str legend_location: one of the values allowed in loc argument of matplotlib's plt.legend() function, or: 'outside right': outside the graph, at its right """ - if 'stackplot' in dir(ax): + if False: # 'stackplot' in dir(ax): ax.stackplot(x_signal, list(y_signals.values())) plt.legend(list(y_signals.keys())) else: @@ -162,6 +172,7 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'): y_stack = np.cumsum(y, axis=0) # a 3x10 array last_lab = None color_index = 0 + print(colors) for series_index in range(len(y_signals)): if series_index == 0: from_signal = 0 @@ -173,8 +184,10 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'): if lab != last_lab: color_index = (color_index + num_unused_colors_between_labs) % num_colors last_lab = lab - ax.fill_between(x_signal, from_signal, y_stack[series_index, :], color=colors[color_index], lw=0.0) - p = plt.Rectangle((0, 0), 0, 0, color=colors[color_index]) + print(color_index) + color = colors[color_index] + ax.fill_between(x_signal, from_signal, y_stack[series_index, :], color=color, lw=0.0) + p = plt.Rectangle((0, 0), 0, 0, color=color) ax.add_patch(p) if legend_location == 'outside right': @@ -193,7 +206,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type for row in rows: (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row is_cluster_node = is_cluster_node_name(name) - if is_cluster_node: + if is_cluster_node and not is_test_machine(name): purchase_date = inventory.get_machine_purchase_date(name) if purchase_date is not None: # print(name, price_ex_vat) @@ -239,7 +252,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type max_num_years = 8 num_years = (datemax - datemin).days // 365 - year_step = num_years / max_num_years + year_step = num_years // max_num_years years = matplotlib.dates.YearLocator(year_step) # every year_step year months = matplotlib.dates.MonthLocator() # every month @@ -276,7 +289,15 @@ def draw_dp_gflops_price_over_time_over_time_graph(inventory, from_date, to_date ax.set_ylabel(u'double precision flops price (€/gflops)') ax.set_title('gflops_price_over_time') - years = matplotlib.dates.YearLocator() # every year + datemin = datetime.date(from_date.year, 1, 1) + datemax = datetime.date(to_date.year + 1, 1, 1) + ax.set_xlim(datemin, datemax) + + max_num_years = 8 + num_years = (datemax - datemin).days // 365 + year_step = num_years // max_num_years + + years = matplotlib.dates.YearLocator(year_step) # every year_step year months = matplotlib.dates.MonthLocator() # every month yearsFmt = matplotlib.dates.DateFormatter('%Y') @@ -288,7 +309,7 @@ def draw_dp_gflops_price_over_time_over_time_graph(inventory, from_date, to_date return fig -def draw_age_pyramid_graph(inventory): +def draw_machine_age_pyramid_graph(inventory): """ :param Inventory inventory: the inventory database """ @@ -301,12 +322,12 @@ def draw_age_pyramid_graph(inventory): for row in rows: (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row is_cluster_node = is_cluster_node_name(name) - if is_cluster_node: + if is_cluster_node and not is_test_machine(name): purchase_date = inventory.get_machine_purchase_date(name) if purchase_date is not None: purchase_time = matplotlib.dates.date2num(purchase_date.date()) # noqa: F841 age = datetime.datetime.now() - purchase_date - age_histogram[age.days / 365] += 1 + age_histogram[age.days // 365] += 1 # print(name, age) fig, ax = plt.subplots() @@ -322,6 +343,40 @@ def draw_age_pyramid_graph(inventory): return fig +def draw_core_age_pyramid_graph(inventory): + """ + :param Inventory inventory: the inventory database + """ + + oldest_age = 20 + age_histogram = np.zeros(shape=(oldest_age)) + + rows = inventory.query("SELECT * FROM machines") + + for row in rows: + (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row + is_cluster_node = is_cluster_node_name(name) + if is_cluster_node and not is_test_machine(name): + purchase_date = inventory.get_machine_purchase_date(name) + if purchase_date is not None: + purchase_time = matplotlib.dates.date2num(purchase_date.date()) # noqa: F841 + age = datetime.datetime.now() - purchase_date + age_histogram[age.days // 365] += inventory.get_num_cores(name) + # print(name, age) + + fig, ax = plt.subplots() + ax.bar(range(oldest_age), age_histogram) + ax.set_xlabel('age (in years)') + ax.set_xticks(range(oldest_age)) + + ax.set_ylabel(u'number of cores') + ax.set_title('cores_age_pyramid') + + # format the ticks + ax.grid(True) + return fig + + class IFigureHandler(object): """ specifies what to do with generated figures @@ -393,7 +448,10 @@ def draw_graphs(inventory, from_time, to_time, figure_handler): fig = draw_dp_gflops_price_over_time_over_time_graph(inventory, from_time.date(), to_time.date()) figure_handler.on_figure_ended(fig) - fig = draw_age_pyramid_graph(inventory) + fig = draw_machine_age_pyramid_graph(inventory) + figure_handler.on_figure_ended(fig) + + fig = draw_core_age_pyramid_graph(inventory) figure_handler.on_figure_ended(fig) figure_handler.on_finalize() diff --git a/inventory.py b/inventory.py index afe574e..0892b7c 100644 --- a/inventory.py +++ b/inventory.py @@ -162,6 +162,12 @@ class Inventory(object): def get_num_cpus(self, computer_name): return int(self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'num_cpu')) + def get_num_cores(self, computer_name): + num_cpus = int(self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'num_cpu')) + cpu_model = self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'cpu_model') + num_cores_per_cpu = int(self._sql_reader.get_table_attr('cpu_specs', 'cpu_model', cpu_model, 'num_cores')) + return num_cpus * num_cores_per_cpu + def get_cpu_model(self, computer_name): return self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'cpu_model')