Bug 3165 - la page ClusterEvolution ne fonctionne plus (failed to find in table orderings a value for ordering_date where ordering_id is ietr.order20210101)

- fixed bug : it was caused by the fact that the database has no information about ietr machines order. Fixed this by ignoring ietr machines because they're only part of ipr cluster temporarily and as such, they shouldn't appear in the stats.

also some improvements:
- improved the colors of the owners of the machines so that tey're visually close for the same organism (eg ipr owners have close colors). This change was made in june 2020 I think (for the questionnaire about it needs related to eskemmdata)
- added a graph to show the age of cores
- prevented overlapping of dates in the graph showing the evolution of gflops price

cluster_stats.py

@@ -32,6 +32,16 @@ def is_cluster_node_name(name):
     return re.match('^simpatix[0-9]+$', name) is not None or re.match('^physix[0-9]+$', name) is not None
 
 
+def is_test_machine(name):
+    return name in [
+        # ietr amd machines temporarily in the cluster for tests
+        'physix12',
+        'physix13',
+        'physix14',
+        'physix15',
+    ]
+
+
 def get_investment_over_time(time_value, price, purchase_time):
     percent_decay_per_day = 0.0  # 1.0/(7.0*365.0)
     f1 = (purchase_time - time_value) * percent_decay_per_day + 1.0
@@ -63,7 +73,7 @@ def get_flops_price_over_time(inventory, time_value):
     for row in rows:
         (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
         is_cluster_node = is_cluster_node_name(name)
-        if is_cluster_node:
+        if is_cluster_node and not is_test_machine(name):
             purchase_date = inventory.get_machine_purchase_date(name)
             if purchase_date is not None:
                 # print(name, price_ex_vat)
@@ -119,7 +129,7 @@ def get_computer_value_over_time(inventory, computer_id, time_value, flops_price
 
 def get_rgb_palette(num_colors, saturation=0.5, value=0.5):
     hsv_tuples = [(x * 1.0 / num_colors, saturation, value) for x in range(num_colors)]
-    rgb_tuples = map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)
+    rgb_tuples = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
     return rgb_tuples
 
 
@@ -131,7 +141,7 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'):
     :param str legend_location: one of the values allowed in loc argument of matplotlib's plt.legend() function, or:
         'outside right': outside the graph, at its right
     """
-    if 'stackplot' in dir(ax):
+    if False:  # 'stackplot' in dir(ax):
         ax.stackplot(x_signal, list(y_signals.values()))
         plt.legend(list(y_signals.keys()))
     else:
@@ -162,6 +172,7 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'):
         y_stack = np.cumsum(y, axis=0)   # a 3x10 array
         last_lab = None
         color_index = 0
+        print(colors)
         for series_index in range(len(y_signals)):
             if series_index == 0:
                 from_signal = 0
@@ -173,8 +184,10 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'):
             if lab != last_lab:
                 color_index = (color_index + num_unused_colors_between_labs) % num_colors
                 last_lab = lab
-            ax.fill_between(x_signal, from_signal, y_stack[series_index, :], color=colors[color_index], lw=0.0)
+            print(color_index)
-            p = plt.Rectangle((0, 0), 0, 0, color=colors[color_index])
+            color = colors[color_index]
+            ax.fill_between(x_signal, from_signal, y_stack[series_index, :], color=color, lw=0.0)
+            p = plt.Rectangle((0, 0), 0, 0, color=color)
             ax.add_patch(p)
 
         if legend_location == 'outside right':
@@ -193,7 +206,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type
     for row in rows:
         (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
         is_cluster_node = is_cluster_node_name(name)
-        if is_cluster_node:
+        if is_cluster_node and not is_test_machine(name):
             purchase_date = inventory.get_machine_purchase_date(name)
             if purchase_date is not None:
                 # print(name, price_ex_vat)
@@ -239,7 +252,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type
 
     max_num_years = 8
     num_years = (datemax - datemin).days // 365
-    year_step = num_years / max_num_years
+    year_step = num_years // max_num_years
 
     years = matplotlib.dates.YearLocator(year_step)   # every year_step year
     months = matplotlib.dates.MonthLocator()  # every month
@@ -276,7 +289,15 @@ def draw_dp_gflops_price_over_time_over_time_graph(inventory, from_date, to_date
     ax.set_ylabel(u'double precision flops price (€/gflops)')
     ax.set_title('gflops_price_over_time')
 
-    years = matplotlib.dates.YearLocator()   # every year
+    datemin = datetime.date(from_date.year, 1, 1)
+    datemax = datetime.date(to_date.year + 1, 1, 1)
+    ax.set_xlim(datemin, datemax)
+
+    max_num_years = 8
+    num_years = (datemax - datemin).days // 365
+    year_step = num_years // max_num_years
+
+    years = matplotlib.dates.YearLocator(year_step)   # every year_step year
     months = matplotlib.dates.MonthLocator()  # every month
     yearsFmt = matplotlib.dates.DateFormatter('%Y')
 
@@ -288,7 +309,7 @@ def draw_dp_gflops_price_over_time_over_time_graph(inventory, from_date, to_date
     return fig
 
 
-def draw_age_pyramid_graph(inventory):
+def draw_machine_age_pyramid_graph(inventory):
     """
     :param Inventory inventory: the inventory database
     """
@@ -301,12 +322,12 @@ def draw_age_pyramid_graph(inventory):
     for row in rows:
         (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
         is_cluster_node = is_cluster_node_name(name)
-        if is_cluster_node:
+        if is_cluster_node and not is_test_machine(name):
             purchase_date = inventory.get_machine_purchase_date(name)
             if purchase_date is not None:
                 purchase_time = matplotlib.dates.date2num(purchase_date.date())  # noqa: F841
                 age = datetime.datetime.now() - purchase_date
-                age_histogram[age.days / 365] += 1
+                age_histogram[age.days // 365] += 1
                 # print(name, age)
 
     fig, ax = plt.subplots()
@@ -322,6 +343,40 @@ def draw_age_pyramid_graph(inventory):
     return fig
 
 
+def draw_core_age_pyramid_graph(inventory):
+    """
+    :param Inventory inventory: the inventory database
+    """
+
+    oldest_age = 20
+    age_histogram = np.zeros(shape=(oldest_age))
+
+    rows = inventory.query("SELECT * FROM machines")
+
+    for row in rows:
+        (name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
+        is_cluster_node = is_cluster_node_name(name)
+        if is_cluster_node and not is_test_machine(name):
+            purchase_date = inventory.get_machine_purchase_date(name)
+            if purchase_date is not None:
+                purchase_time = matplotlib.dates.date2num(purchase_date.date())  # noqa: F841
+                age = datetime.datetime.now() - purchase_date
+                age_histogram[age.days // 365] += inventory.get_num_cores(name)
+                # print(name, age)
+
+    fig, ax = plt.subplots()
+    ax.bar(range(oldest_age), age_histogram)
+    ax.set_xlabel('age (in years)')
+    ax.set_xticks(range(oldest_age))
+
+    ax.set_ylabel(u'number of cores')
+    ax.set_title('cores_age_pyramid')
+
+    # format the ticks
+    ax.grid(True)
+    return fig
+
+
 class IFigureHandler(object):
     """
     specifies what to do with generated figures
@@ -393,7 +448,10 @@ def draw_graphs(inventory, from_time, to_time, figure_handler):
     fig = draw_dp_gflops_price_over_time_over_time_graph(inventory, from_time.date(), to_time.date())
     figure_handler.on_figure_ended(fig)
 
-    fig = draw_age_pyramid_graph(inventory)
+    fig = draw_machine_age_pyramid_graph(inventory)
+    figure_handler.on_figure_ended(fig)
+
+    fig = draw_core_age_pyramid_graph(inventory)
     figure_handler.on_figure_ended(fig)
 
     figure_handler.on_finalize()

inventory.py

@@ -162,6 +162,12 @@ class Inventory(object):
     def get_num_cpus(self, computer_name):
         return int(self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'num_cpu'))
 
+    def get_num_cores(self, computer_name):
+        num_cpus = int(self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'num_cpu'))
+        cpu_model = self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'cpu_model')
+        num_cores_per_cpu = int(self._sql_reader.get_table_attr('cpu_specs', 'cpu_model', cpu_model, 'num_cores'))
+        return num_cpus * num_cores_per_cpu
+
     def get_cpu_model(self, computer_name):
         return self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'cpu_model')