graffy
/
concho
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

the user can now choose what to plot and for which configs 

Now concho is much more generic, as this modification completes the refactoring initiated last week.
This commit is contained in:
Guillaume Raffy 2020-10-01 19:18:53 +02:00
parent 6b91c0d1ca
commit b82d60b49d
2 changed files with 88 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

183
concho/procs_chooser.py
View File

@ -1,4 +1,5 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
from abc import ABCMeta, abstractmethod
import numpy import numpy
import pylab import pylab
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
@ -67,30 +68,60 @@ def create_unique_marker(config_index):
config_index = config_index // alphabet_size config_index = config_index // alphabet_size
return marker_string return marker_string
def plot_system_efficiency(configs): class ConfigAxisDef():
def __init__(self):
pass
@abstractmethod
def get_axis_label(self):
pass
@abstractmethod
def get_value_for_config(self, config):
pass
class ConfigPrice(ConfigAxisDef):
def get_axis_label(self):
return u'speed/price ratio [core.MHz/€]'
def get_value_for_config(self, config):
return config.get_price()
class ConfigFlops(ConfigAxisDef):
def get_axis_label(self):
return u'num total DP operations per second'
def get_value_for_config(self, config):
return config.get_flops()
class ConfigFlopsPerEuro(ConfigAxisDef):
def get_axis_label(self):
return u'num total DP operations/total cost [€/^-1]'
def get_value_for_config(self, config):
kWHPrice = 0.07 * 1.5
containerLifetime = 7.0 # in years
powerUsageEfficiency = 0.5
powerUsedInLifetime = (config.get_power_consumption() * containerLifetime * 365 * 24) / powerUsageEfficiency
itemTotalCost = config.get_price() + (powerUsedInLifetime / 1000.0 * kWHPrice )
item_total_num_ops = config.get_flops() * containerLifetime * 365 * 24 * 3600
return item_total_num_ops / itemTotalCost
def plot_configs(configs, xaxis_def, yaxis_def, plot_title):
""" """
Args: Args:
configs (list(Config)): the tist of configurations to plot configs (list(Config)): the tist of configurations to plot
""" """
cpuTable = numpy.genfromtxt('cpu_table.dat', dtype=("|U15", float, int, float, float, float), names=True, delimiter='\t')
#cpuTable = numpy.genfromtxt('dell_ivybridge_table.dat', dtype=(('id', "|S10"), ('clock', float), ('num_cores', int), ('price', float, float)), names=None, delimiter='\t')
print(type(cpuTable))
print(cpuTable.dtype)
print(cpuTable)
print(cpuTable['id'])
#cpuTable = numpy.genfromtxt('dell_ivybridge_table.dat', dtype=(('id', "|S10"), ('clock', float), ('num_cores', int), ('price', float, float)), names=None, delimiter='\t')
#for (x, y) in clusters:
def GHzToMHz( frequency ): def GHzToMHz( frequency ):
return frequency * 1000.0 return frequency * 1000.0
kWHPrice = 0.07 * 1.5
containerLifetime = 7.0 # in years
powerUsageEfficiency = 0.5
def getColorCodeFromItemLabel(label): def getColorCodeFromItemLabel(label):
generation=label[-1] generation=label[-1]
(num_servers, model, proc_id, ram_size) = re.split('_', label) (num_servers, model, proc_id, ram_size) = re.split('_', label)
@ -133,99 +164,27 @@ def plot_system_efficiency(configs):
(model, proc_id, ram_size) = re.split('_', label) (model, proc_id, ram_size) = re.split('_', label)
return get_marker(proc_id) return get_marker(proc_id)
markerSize = 50
item_price = numpy.array([]) plt.subplot(1,2,1)
item_power_consumption = numpy.array([])
item_speed = numpy.array([]) markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
item_label = numpy.array([])
config_index = 0
for config in configs: for config in configs:
cpu = config.cpu cpu = config.cpu
config_label = str(config.num_servers) + '_' + config.chassis.uid + '_' + cpu.uid + '_' + str(config.ram_size/config.num_servers) + 'gb' config_label = str(config.num_servers) + '_' + config.chassis.uid + '_' + cpu.uid + '_' + str(config.ram_size/config.num_servers) + 'gb'
item_label = numpy.append( item_label, config_label)
# print('procOptionPrice', procOptionPrice) x1 = xaxis_def.get_value_for_config(config)
# config_price = procOptionPrice y1 = yaxis_def.get_value_for_config(config)
# config_price += config.get_empty_price()
# print('config.get_empty_price()', config.get_empty_price())
# ram_update_price = config.get_ram_update_price(cpu=cpu, ram_per_core=4.0e9)
# # ram_update_price = config.get_ram_update_price(cpu=cpu, ram_per_server=192.0e9)
# # ram_update_price = config.get_ram_update_price(cpu=cpu, ram_per_cpu=96.0e9)
# print('ram_update_price', ram_update_price)
# config_price += ram_update_price
# config_price += config.get_guarantee_price(5)
# print('config.config.get_guarantee_price(5)', config.get_guarantee_price(5))
# config_price += config.get_disk_upgrade_price(2.0e12)
# print('config.get_disk_upgrade_price(2.0e12)', config.get_disk_upgrade_price(2.0e12))
# print(item_label, config.get_price(), config.get_power_consumption())
item_price = numpy.append( item_price, config.get_price() )
item_power_consumption = numpy.append( item_power_consumption, config.get_power_consumption())
# # print(hostTypeId, procId, item_power_consumption[-1])
item_speed = numpy.append( item_speed, config.get_flops())
#pylab.plot(x, y, '+') print(config_label, x1, y1)
#pylab.xlabel('speed/price ratio [core.MHz/euros]')
#pylab.ylabel('speed/power consumption ratio [core.MHz/W]')
#pylab.show() # or savefig(<filename>)
#print("items = ")
#print(item_label)
markerSize = 50
if False:
plt.subplot(1,2,1)
plt.subplots_adjust(bottom = 0.1)
markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
x = item_speed / item_price
y = item_speed / item_power_consumption
for label, x1, y1, power, speed, price, in zip(item_label, x, y, item_power_consumption, item_speed, item_price):
marker = markersCycler.next()
color = getColorCodeFromItemLabel(label)
plt.scatter( x1, y1, color = color, s = markerSize, marker = marker[0], label = label)
#print(x1, y1, color, markerSize, marker[0], label)
if False:
plt.scatter( x, y, marker = 'o')
for label, x1, y1, power, speed, price, in zip(item_label, x, y, item_power_consumption, item_speed, item_price):
#print(label)
plt.annotate( u'%s (%.1f core.GHz, %.0f W, %.0f €)' % (label,speed/1000.0, power, price),
xy = (x1, y1), xytext = (-50, 50),
textcoords = 'offset points', ha = 'right', va = 'bottom',
bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))
plt.xlabel(u'speed/price ratio [core.MHz/€]')
plt.ylabel(u'speed/power consumption ratio [core.MHz/W]')
plt.xlim( xmin = 0.0 )
plt.ylim( ymin = 0.0 )
plt.subplot(1,2,1)
#fig = plt.figure()
#ax = fig.gca()
#ax.set_xticks(numpy.arange(0,1,0.1))
#ax.set_yticks(numpy.arange(0,1.,0.1))
powerUsedInLifetime = (item_power_consumption * containerLifetime * 365 * 24) / powerUsageEfficiency
itemTotalCost = item_price + (powerUsedInLifetime / 1000.0 * kWHPrice )
markersCycler = itertools.cycle(itertools.product(markerTypes, markerColors))
item_flops = item_speed
# print item_flops
item_total_num_ops = item_flops * containerLifetime * 365 * 24 * 3600
print(item_price)
x = item_price
y = item_total_num_ops / itemTotalCost
for i in range(len(item_label)):
print(item_label[i], item_price[i], y[i])
print('itemTotalCost', itemTotalCost[i])
print('flops', item_flops[i])
# print y
config_index = 0
for label, x1, y1, power, speed, price, in zip(item_label, x, y, item_power_consumption, item_speed, item_price):
print(label, x1, y1)
if y1 > 0.0001: if y1 > 0.0001:
color = getColorCodeFromItemLabel(label) color = getColorCodeFromItemLabel(config_label)
# marker = markersCycler.next() # marker = markersCycler.next()
# marker = get_marker_from_label( label ) # marker = get_marker_from_label( label )
#print(x1, y1) #print(x1, y1)
short_label = label.replace('dell-poweredge-','').replace('intel-xeon-','') short_label = config_label.replace('dell-poweredge-','').replace('intel-xeon-','')
plt.scatter( x1, y1, facecolors=color, s=(markerSize * len(create_unique_marker(config_index))) , marker='$\mathrm{%s}$' % create_unique_marker(config_index), label=short_label) plt.scatter( x1, y1, facecolors=color, s=(markerSize * len(create_unique_marker(config_index))) , marker='$\mathrm{%s}$' % create_unique_marker(config_index), label=short_label)
if False: # y1 > 5.0e16: if False: # y1 > 5.0e16:
plt.annotate( u'%s' % short_label, plt.annotate( u'%s' % short_label,
@ -235,9 +194,9 @@ def plot_system_efficiency(configs):
arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0')) arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))
config_index += 1 config_index += 1
plt.xlabel(u'purchase price [€]') plt.xlabel(xaxis_def.get_axis_label())
plt.ylabel(u'num total DP operations/total cost [€/^-1]') plt.ylabel(yaxis_def.get_axis_label())
plt.title(u'total cost including electricity') plt.title(plot_title)
plt.xlim( xmin = 0.0 ) plt.xlim( xmin = 0.0 )
plt.ylim( ymin = 0.0 ) plt.ylim( ymin = 0.0 )
plt.minorticks_on() plt.minorticks_on()
@ -246,24 +205,20 @@ def plot_system_efficiency(configs):
plt.legend(bbox_to_anchor=(1.1, 1.1), ncol=3) plt.legend(bbox_to_anchor=(1.1, 1.1), ncol=3)
plt.draw() plt.draw()
plt.show() plt.show()
#plotCpuPassmark(): def plot_configurators(configurators, ram_per_core, xaxis_def, yaxis_def, plot_title, config_filter=lambda config : True):
def plot_efficiency(configurators, ram_per_core, cpu_filter=lambda cpu : True):
configs = [] configs = []
for configurator in configurators: for configurator in configurators:
for cpu in configurator.get_cpu_options(): for cpu in configurator.get_cpu_options():
if cpu_filter(cpu): config = configurator.create_config()
config = configurator.create_config() config.num_cpu_per_server = config.configurator.chassis.item.num_cpu_slots_per_server
config.num_cpu_per_server = config.configurator.chassis.item.num_cpu_slots_per_server config.set_cpu(cpu)
config.set_cpu(cpu) config.set_ram(ram_per_core=ram_per_core)
config.set_ram(ram_per_core=ram_per_core) if config_filter(config):
configs.append(config) configs.append(config)
plot_system_efficiency(configs) plot_configs(configs, xaxis_def=xaxis_def, yaxis_def=yaxis_def, plot_title=plot_title)
if __name__ == '__main__': if __name__ == '__main__':

34
tests/test1.py
View File

@ -1,7 +1,9 @@
from concho.dell import DellMatinfoCsvConfigurator from concho.dell import DellMatinfoCsvConfigurator
from concho.dell import DellMatinfoConfigurator from concho.dell import DellMatinfoConfigurator
from concho.procs_chooser import plot_system_efficiency from concho.procs_chooser import plot_configurators
from concho.procs_chooser import plot_efficiency from concho.procs_chooser import ConfigPrice
from concho.procs_chooser import ConfigFlops
from concho.procs_chooser import ConfigFlopsPerEuro
def test_all_matinfo_2020_configs(): def test_all_matinfo_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
@ -13,7 +15,7 @@ def test_all_matinfo_2020_configs():
# dell.DellPowerEdgeR940(), # dell.DellPowerEdgeR940(),
] ]
plot_efficiency(configurators=configurators, ram_per_core=4.0e9) plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='total cost including electricity')
def test_credits_2020_configs(): def test_credits_2020_configs():
# configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html') # configurator = DellMatinfoConfigurator('rcrc1406676-4834664 - Cat2 Conf4 PowerEdge R640 - Dell.html')
@ -25,19 +27,21 @@ def test_credits_2020_configs():
# dell.DellPowerEdgeR940(), # dell.DellPowerEdgeR940(),
] ]
cpu_filter = lambda cpu : cpu.uid in [ # config_filter = lambda config : config.cpu.uid in [
'intel-xeon-gold-5222', # 'intel-xeon-gold-5222',
'intel-xeon-gold-6226r', # 'intel-xeon-gold-6226r',
'intel-xeon-gold-6230r', # 'intel-xeon-gold-6230r',
'intel-xeon-gold-6234r', # 'intel-xeon-gold-6234r',
'intel-xeon-gold-6240r', # 'intel-xeon-gold-6240r',
'intel-xeon-gold-6248r', # 'intel-xeon-gold-6248r',
'intel-xeon-gold-6230', # 'intel-xeon-gold-6230',
'intel-xeon-gold-6240', # 'intel-xeon-gold-6240',
] # ]
plot_efficiency(configurators=configurators, ram_per_core=4.0e9, cpu_filter=cpu_filter) config_filter = lambda config : config.get_price() < 15000.0
plot_configurators(configurators=configurators, ram_per_core=4.0e9, xaxis_def=ConfigPrice(), yaxis_def=ConfigFlopsPerEuro(), plot_title='physmol/ts credit 2020 configs', config_filter=config_filter)
if __name__ == '__main__': if __name__ == '__main__':
test_all_matinfo_2020_configs() test_credits_2020_configs()