cellinfo
/
cocluto
2
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Bug 3165 - la page ClusterEvolution ne fonctionne plus (failed to find in table orderings a value for ordering_date where ordering_id is ietr.order20210101) 

- fixed bug : it was caused by the fact that the database has no information about ietr machines order. Fixed this by ignoring ietr machines because they're only part of ipr cluster temporarily and as such, they shouldn't appear in the stats.

also some improvements:
- improved the colors of the owners of the machines so that tey're visually close for the same organism (eg ipr owners have close colors). This change was made in june 2020 I think (for the questionnaire about it needs related to eskemmdata)
- added a graph to show the age of cores
- prevented overlapping of dates in the graph showing the evolution of gflops price
This commit is contained in:
Guillaume Raffy 2021-06-07 11:42:03 +00:00
parent 939ecda69d
commit ad8f6f383b
2 changed files with 76 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
cluster_stats.py
View File

@ -32,6 +32,16 @@ def is_cluster_node_name(name):
return re.match('^simpatix[0-9]+$', name) is not None or re.match('^physix[0-9]+$', name) is not None
def is_test_machine(name):
return name in [
# ietr amd machines temporarily in the cluster for tests
'physix12',
'physix13',
'physix14',
'physix15',
]
def get_investment_over_time(time_value, price, purchase_time):
percent_decay_per_day = 0.0 # 1.0/(7.0*365.0)
f1 = (purchase_time - time_value) * percent_decay_per_day + 1.0
@ -63,7 +73,7 @@ def get_flops_price_over_time(inventory, time_value):
for row in rows:
(name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
is_cluster_node = is_cluster_node_name(name)
if is_cluster_node:
if is_cluster_node and not is_test_machine(name):
purchase_date = inventory.get_machine_purchase_date(name)
if purchase_date is not None:
# print(name, price_ex_vat)
@ -119,7 +129,7 @@ def get_computer_value_over_time(inventory, computer_id, time_value, flops_price
def get_rgb_palette(num_colors, saturation=0.5, value=0.5):
hsv_tuples = [(x * 1.0 / num_colors, saturation, value) for x in range(num_colors)]
rgb_tuples = map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples)
rgb_tuples = list(map(lambda x: colorsys.hsv_to_rgb(*x), hsv_tuples))
return rgb_tuples
@ -131,7 +141,7 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'):
:param str legend_location: one of the values allowed in loc argument of matplotlib's plt.legend() function, or:
'outside right': outside the graph, at its right
"""
if 'stackplot' in dir(ax):
if False: # 'stackplot' in dir(ax):
ax.stackplot(x_signal, list(y_signals.values()))
plt.legend(list(y_signals.keys()))
else:
@ -162,6 +172,7 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'):
y_stack = np.cumsum(y, axis=0) # a 3x10 array
last_lab = None
color_index = 0
print(colors)
for series_index in range(len(y_signals)):
if series_index == 0:
from_signal = 0
@ -173,8 +184,10 @@ def stackplot(ax, x_signal, y_signals, legend_location='best'):
if lab != last_lab:
color_index = (color_index + num_unused_colors_between_labs) % num_colors
last_lab = lab
ax.fill_between(x_signal, from_signal, y_stack[series_index, :], color=colors[color_index], lw=0.0)
p = plt.Rectangle((0, 0), 0, 0, color=colors[color_index])
print(color_index)
color = colors[color_index]
ax.fill_between(x_signal, from_signal, y_stack[series_index, :], color=color, lw=0.0)
p = plt.Rectangle((0, 0), 0, 0, color=color)
ax.add_patch(p)
if legend_location == 'outside right':
@ -193,7 +206,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type
for row in rows:
(name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
is_cluster_node = is_cluster_node_name(name)
if is_cluster_node:
if is_cluster_node and not is_test_machine(name):
purchase_date = inventory.get_machine_purchase_date(name)
if purchase_date is not None:
# print(name, price_ex_vat)
@ -239,7 +252,7 @@ def draw_cluster_value_over_time_graph(inventory, from_date, to_date, graph_type
max_num_years = 8
num_years = (datemax - datemin).days // 365
year_step = num_years / max_num_years
year_step = num_years // max_num_years
years = matplotlib.dates.YearLocator(year_step) # every year_step year
months = matplotlib.dates.MonthLocator() # every month
@ -276,7 +289,15 @@ def draw_dp_gflops_price_over_time_over_time_graph(inventory, from_date, to_date
ax.set_ylabel(u'double precision flops price (€/gflops)')
ax.set_title('gflops_price_over_time')
years = matplotlib.dates.YearLocator() # every year
datemin = datetime.date(from_date.year, 1, 1)
datemax = datetime.date(to_date.year + 1, 1, 1)
ax.set_xlim(datemin, datemax)
max_num_years = 8
num_years = (datemax - datemin).days // 365
year_step = num_years // max_num_years
years = matplotlib.dates.YearLocator(year_step) # every year_step year
months = matplotlib.dates.MonthLocator() # every month
yearsFmt = matplotlib.dates.DateFormatter('%Y')
@ -288,7 +309,7 @@ def draw_dp_gflops_price_over_time_over_time_graph(inventory, from_date, to_date
return fig
def draw_age_pyramid_graph(inventory):
def draw_machine_age_pyramid_graph(inventory):
"""
:param Inventory inventory: the inventory database
"""
@ -301,12 +322,12 @@ def draw_age_pyramid_graph(inventory):
for row in rows:
(name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
is_cluster_node = is_cluster_node_name(name)
if is_cluster_node:
if is_cluster_node and not is_test_machine(name):
purchase_date = inventory.get_machine_purchase_date(name)
if purchase_date is not None:
purchase_time = matplotlib.dates.date2num(purchase_date.date()) # noqa: F841
age = datetime.datetime.now() - purchase_date
age_histogram[age.days / 365] += 1
age_histogram[age.days // 365] += 1
# print(name, age)
fig, ax = plt.subplots()
@ -322,6 +343,40 @@ def draw_age_pyramid_graph(inventory):
return fig
def draw_core_age_pyramid_graph(inventory):
"""
:param Inventory inventory: the inventory database
"""
oldest_age = 20
age_histogram = np.zeros(shape=(oldest_age))
rows = inventory.query("SELECT * FROM machines")
for row in rows:
(name, serial_number, affectation, machine_spec_id, command_id, price_ex_vat, pos_x, pos_y, pos_z, inv_number) = row
is_cluster_node = is_cluster_node_name(name)
if is_cluster_node and not is_test_machine(name):
purchase_date = inventory.get_machine_purchase_date(name)
if purchase_date is not None:
purchase_time = matplotlib.dates.date2num(purchase_date.date()) # noqa: F841
age = datetime.datetime.now() - purchase_date
age_histogram[age.days // 365] += inventory.get_num_cores(name)
# print(name, age)
fig, ax = plt.subplots()
ax.bar(range(oldest_age), age_histogram)
ax.set_xlabel('age (in years)')
ax.set_xticks(range(oldest_age))
ax.set_ylabel(u'number of cores')
ax.set_title('cores_age_pyramid')
# format the ticks
ax.grid(True)
return fig
class IFigureHandler(object):
"""
specifies what to do with generated figures
@ -393,7 +448,10 @@ def draw_graphs(inventory, from_time, to_time, figure_handler):
fig = draw_dp_gflops_price_over_time_over_time_graph(inventory, from_time.date(), to_time.date())
figure_handler.on_figure_ended(fig)
fig = draw_age_pyramid_graph(inventory)
fig = draw_machine_age_pyramid_graph(inventory)
figure_handler.on_figure_ended(fig)
fig = draw_core_age_pyramid_graph(inventory)
figure_handler.on_figure_ended(fig)
figure_handler.on_finalize()

6
inventory.py
View File

@ -162,6 +162,12 @@ class Inventory(object):
def get_num_cpus(self, computer_name):
return int(self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'num_cpu'))
def get_num_cores(self, computer_name):
num_cpus = int(self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'num_cpu'))
cpu_model = self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'cpu_model')
num_cores_per_cpu = int(self._sql_reader.get_table_attr('cpu_specs', 'cpu_model', cpu_model, 'num_cores'))
return num_cpus * num_cores_per_cpu
def get_cpu_model(self, computer_name):
return self._sql_reader.get_table_attr('computer_to_cpu', 'computer_id', computer_name, 'cpu_model')